{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-11-01", "retrieved_at": "2026-11-01T01:45:38.316Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's non-zero-score runs from the newest 90 days: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2241, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.964646, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.09468125757575757, "avg_duration_per_task_ms": 101980.28282828283, "avg_output_tokens_per_task": 8439.414141414141, "last_run_timestamp": "2026-10-14T15:00:03.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-4-argon", "model_slug": "google/gemini-4-argon", "model_name": "Gemini 4 Argon", "provider_name": "auto-routing", "accuracy": 0.959596, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.029260161616161615, "avg_duration_per_task_ms": 40252.82323232323, "avg_output_tokens_per_task": 2870.0252525252527, "last_run_timestamp": "2026-10-23T07:38:37.725Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.95637, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.06822796858638744, "avg_duration_per_task_ms": 132197.8062827225, "avg_output_tokens_per_task": 18049.068062827224, "last_run_timestamp": "2026-10-01T08:31:33.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.954545, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.332555398989899, "avg_duration_per_task_ms": 75446.5505050505, "avg_output_tokens_per_task": 4530.388888888889, "last_run_timestamp": "2026-10-01T08:27:35.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07423223737373738, "avg_duration_per_task_ms": 86748.28787878787, "avg_output_tokens_per_task": 19646.409090909092, "last_run_timestamp": "2026-10-01T08:29:11.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.95202, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20364760606060606, "avg_duration_per_task_ms": 135706.18686868687, "avg_output_tokens_per_task": 16847.588383838385, "last_run_timestamp": "2026-10-01T19:14:26.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.9511785, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.33620659090909094, "avg_duration_per_task_ms": 70544.52272727272, "avg_output_tokens_per_task": 4588.830808080808, "last_run_timestamp": "2026-10-02T23:23:01.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.9511779999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20247136363636364, "avg_duration_per_task_ms": 137578.30555555556, "avg_output_tokens_per_task": 16748.56818181818, "last_run_timestamp": "2026-10-01T19:12:42.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.949495, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.35495546717171716, "avg_duration_per_task_ms": 72102.17676767676, "avg_output_tokens_per_task": 4622.60101010101, "last_run_timestamp": "2026-10-01T08:29:03.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.9482614885496183, "accuracy_std_dev": 0.0012429999999999941, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.07536056743002545, "avg_duration_per_task_ms": 132814.94147582698, "avg_output_tokens_per_task": 18975.51145038168, "last_run_timestamp": "2026-10-15T21:00:04.244Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.947917, "accuracy_std_dev": null, "run_count": 1, "total_questions": 96, "avg_cost_per_task_usd": 0.009331666666666667, "avg_duration_per_task_ms": 232878.71875, "avg_output_tokens_per_task": 10278.75, "last_run_timestamp": "2026-10-14T18:00:04.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.947811, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.021484267676767678, "avg_duration_per_task_ms": 55623.856060606064, "avg_output_tokens_per_task": 1890.328282828283, "last_run_timestamp": "2026-10-30T00:03:33.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9472503333333333, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11162585016835018, "avg_duration_per_task_ms": 48234.978114478115, "avg_output_tokens_per_task": 2076.191919191919, "last_run_timestamp": "2026-10-02T23:16:05.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra", "model_slug": "sakana/fugu-ultra", "model_name": "Fugu Ultra", "provider_name": "auto-routing", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 1.148739303030303, "avg_duration_per_task_ms": 328779.10101010103, "avg_output_tokens_per_task": 33271.606060606064, "last_run_timestamp": "2026-09-15T01:07:47.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.9461276666666666, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1181814292929293, "avg_duration_per_task_ms": 36670.56565656565, "avg_output_tokens_per_task": 2059.811447811448, "last_run_timestamp": "2026-10-30T00:02:55.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20203182828282829, "avg_duration_per_task_ms": 133595.55303030304, "avg_output_tokens_per_task": 16712.858585858587, "last_run_timestamp": "2026-10-01T19:16:14.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11333201010101009, "avg_duration_per_task_ms": 57056.792929292926, "avg_output_tokens_per_task": 2108.121212121212, "last_run_timestamp": "2026-10-01T08:25:16.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.944444, "accuracy_std_dev": 0.0027491439879836154, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020607563973063972, "avg_duration_per_task_ms": 47519.37542087542, "avg_output_tokens_per_task": 1902.2373737373737, "last_run_timestamp": "2026-10-02T23:20:22.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02136650505050505, "avg_duration_per_task_ms": 52411.32323232323, "avg_output_tokens_per_task": 1978.1313131313132, "last_run_timestamp": "2026-10-01T08:24:51.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9431713315363881, "accuracy_std_dev": 0.006147978549897494, "run_count": 4, "total_questions": 742, "avg_cost_per_task_usd": 0.10982969407008086, "avg_duration_per_task_ms": 117613.10242587602, "avg_output_tokens_per_task": 1949.2371967654988, "last_run_timestamp": "2026-10-01T08:26:36.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9427606666666666, "accuracy_std_dev": 0.0036369299629708685, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.029872336700336698, "avg_duration_per_task_ms": 28492.575757575756, "avg_output_tokens_per_task": 7819.474747474747, "last_run_timestamp": "2026-10-01T16:16:03.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.941077, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.15125092424242426, "avg_duration_per_task_ms": 97603.77272727272, "avg_output_tokens_per_task": 11559.585858585859, "last_run_timestamp": "2026-10-01T16:10:33.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.939394, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.31252995454545457, "avg_duration_per_task_ms": 150832.55050505052, "avg_output_tokens_per_task": 10288.712121212122, "last_run_timestamp": "2026-10-01T19:12:11.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.939394, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.32760574494949496, "avg_duration_per_task_ms": 78455.45202020202, "avg_output_tokens_per_task": 11977.780303030304, "last_run_timestamp": "2026-10-01T16:36:19.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.9393935, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.15067959090909092, "avg_duration_per_task_ms": 95544.60353535354, "avg_output_tokens_per_task": 11515.699494949495, "last_run_timestamp": "2026-10-02T23:21:48.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.9391951653543308, "accuracy_std_dev": 0.009884880879403648, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.026313035433070862, "avg_duration_per_task_ms": 56131.107611548556, "avg_output_tokens_per_task": 7729.320209973754, "last_run_timestamp": "2026-10-15T21:01:11.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.938528, "accuracy_std_dev": 0.004357499999999959, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.0293307974025974, "avg_duration_per_task_ms": 59248.22337662338, "avg_output_tokens_per_task": 7683.532467532467, "last_run_timestamp": "2026-10-01T16:17:32.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.9382713333333333, "accuracy_std_dev": 0.004627357405210382, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022108563973063967, "avg_duration_per_task_ms": 32928.50084175084, "avg_output_tokens_per_task": 1914.9974747474748, "last_run_timestamp": "2026-10-01T08:40:50.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.9360266666666668, "accuracy_std_dev": 0.0036369299629708854, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3141816919191919, "avg_duration_per_task_ms": 142474.1195286195, "avg_output_tokens_per_task": 10342.412457912458, "last_run_timestamp": "2026-10-02T23:25:41.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash", "model_slug": "google/gemini-3.8-flash", "model_name": "Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.9343430000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023175040404040403, "avg_duration_per_task_ms": 44381.02525252525, "avg_output_tokens_per_task": 6131.126262626262, "last_run_timestamp": "2026-10-23T07:37:43.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9336215714285715, "accuracy_std_dev": 0.008810277924222438, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.3281882835497835, "avg_duration_per_task_ms": 164668.6948051948, "avg_output_tokens_per_task": 10091.254689754689, "last_run_timestamp": "2026-10-01T19:14:07.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra-v2", "model_slug": "sakana/fugu-ultra-v2", "model_name": "Fugu Ultra V2", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.5025571414141414, "avg_duration_per_task_ms": 194915.41919191918, "avg_output_tokens_per_task": 13806.227272727272, "last_run_timestamp": "2026-09-15T01:07:48.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "typesafe/jev-router", "model_slug": "typesafe/jev-router", "model_name": "TypeSafe: Jev Router", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03490100505050505, "avg_duration_per_task_ms": 57907.71885521885, "avg_output_tokens_per_task": 6070.841750841751, "last_run_timestamp": "2026-10-09T23:23:36.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.9309762, "accuracy_std_dev": 0.005106374267520954, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.057293524242424246, "avg_duration_per_task_ms": 66775.28888888888, "avg_output_tokens_per_task": 11798.8, "last_run_timestamp": "2026-10-01T19:12:42.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.9301345000000001, "accuracy_std_dev": 0.006832688313053163, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.026123152076318747, "avg_duration_per_task_ms": 18695.293490460157, "avg_output_tokens_per_task": 2221.0524691358023, "last_run_timestamp": "2026-10-31T08:07:18.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9301345, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.044492378787878796, "avg_duration_per_task_ms": 69170.04040404041, "avg_output_tokens_per_task": 11715.780303030304, "last_run_timestamp": "2026-10-01T19:12:42.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.929854, "accuracy_std_dev": 0.006348890821762996, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.025367877946127944, "avg_duration_per_task_ms": 20154.14898989899, "avg_output_tokens_per_task": 2304.1734006734005, "last_run_timestamp": "2026-10-31T08:33:04.486Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.9290121666666666, "accuracy_std_dev": 0.0044978590987515684, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024844954545454545, "avg_duration_per_task_ms": 19502.118686868685, "avg_output_tokens_per_task": 2244.3215488215487, "last_run_timestamp": "2026-10-31T09:02:13.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.928451, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.3476244823232323, "avg_duration_per_task_ms": 112460.52777777778, "avg_output_tokens_per_task": 10405.60101010101, "last_run_timestamp": "2026-10-01T19:13:06.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12326839393939394, "avg_duration_per_task_ms": 361758.7878787879, "avg_output_tokens_per_task": 20322.222222222223, "last_run_timestamp": "2026-10-01T16:09:08.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.9282826, "accuracy_std_dev": 0.00626296053316642, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025195250505050507, "avg_duration_per_task_ms": 19633.72121212121, "avg_output_tokens_per_task": 2281.29898989899, "last_run_timestamp": "2026-10-31T08:09:30.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.9276095000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04436604545454545, "avg_duration_per_task_ms": 61670.28282828283, "avg_output_tokens_per_task": 11683.290404040405, "last_run_timestamp": "2026-10-01T19:11:04.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9276091428571428, "accuracy_std_dev": 0.005010400338398178, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10054259307359308, "avg_duration_per_task_ms": 55310.95093795094, "avg_output_tokens_per_task": 3496.1031746031745, "last_run_timestamp": "2026-10-01T19:42:48.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.9275846571787361, "accuracy_std_dev": 0.010407425048021134, "run_count": 13, "total_questions": 2389, "avg_cost_per_task_usd": 0.13203736626203433, "avg_duration_per_task_ms": 128986.54792800335, "avg_output_tokens_per_task": 8711.713269150272, "last_run_timestamp": "2026-10-31T08:39:06.851Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.92672599688196, "accuracy_std_dev": 0.007741573929745956, "run_count": 36, "total_questions": 6735, "avg_cost_per_task_usd": 0.17203167082405343, "avg_duration_per_task_ms": 164862.31818856718, "avg_output_tokens_per_task": 9318.532887899035, "last_run_timestamp": "2026-10-31T08:52:06.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.9264870000000001, "accuracy_std_dev": 0.007815979699734826, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03737200673400674, "avg_duration_per_task_ms": 60770.02693602694, "avg_output_tokens_per_task": 3580.875420875421, "last_run_timestamp": "2026-10-02T23:20:36.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9260940999999999, "accuracy_std_dev": 0.00286181765491794, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.026339350505050502, "avg_duration_per_task_ms": 19433.036868686868, "avg_output_tokens_per_task": 2294.320202020202, "last_run_timestamp": "2026-10-31T08:17:29.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.9259260000000001, "accuracy_std_dev": 0.004610436378478734, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.10788488484848485, "avg_duration_per_task_ms": 187765.80707070706, "avg_output_tokens_per_task": 9499.632323232323, "last_run_timestamp": "2026-10-24T08:37:09.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.9252448891402715, "accuracy_std_dev": 0.0069132275216017836, "run_count": 9, "total_questions": 1768, "avg_cost_per_task_usd": 0.10858752149321266, "avg_duration_per_task_ms": 188184.25339366516, "avg_output_tokens_per_task": 9458.231334841628, "last_run_timestamp": "2026-10-31T09:20:20.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Darkbloom", "accuracy": 0.925214, "accuracy_std_dev": null, "run_count": 1, "total_questions": 78, "avg_cost_per_task_usd": 0.0012695128205128204, "avg_duration_per_task_ms": 237789.78205128206, "avg_output_tokens_per_task": 6844.423076923077, "last_run_timestamp": "2026-10-30T01:38:27.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9246375695652174, "accuracy_std_dev": 0.006863146440316182, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.12292555942028985, "avg_duration_per_task_ms": 340480.1579710145, "avg_output_tokens_per_task": 19662.684057971015, "last_run_timestamp": "2026-10-01T16:09:31.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.924599700084246, "accuracy_std_dev": 0.006819818440072705, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.02751449789385004, "avg_duration_per_task_ms": 24957.139848357205, "avg_output_tokens_per_task": 2420.7127211457455, "last_run_timestamp": "2026-10-01T08:24:52.927Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9245788000000001, "accuracy_std_dev": 0.005259575777569916, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1499325777777778, "avg_duration_per_task_ms": 85124.36363636363, "avg_output_tokens_per_task": 15611.38383838384, "last_run_timestamp": "2026-10-01T19:13:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03863523484848485, "avg_duration_per_task_ms": 63161.229797979795, "avg_output_tokens_per_task": 3708.2954545454545, "last_run_timestamp": "2026-10-01T19:12:28.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "unbiased/pareto-26.10-preview", "model_slug": "unbiased/pareto-26.10-preview", "model_name": "Pareto 26.10 Preview", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0030723434343434344, "avg_duration_per_task_ms": 37807.181818181816, "avg_output_tokens_per_task": 903.9090909090909, "last_run_timestamp": "2026-10-15T14:24:31.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9240912983587339, "accuracy_std_dev": 0.015687164131543977, "run_count": 10, "total_questions": 1706, "avg_cost_per_task_usd": 0.1119299531066823, "avg_duration_per_task_ms": 198230.14243845252, "avg_output_tokens_per_task": 8354.249120750294, "last_run_timestamp": "2026-10-31T08:14:43.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.9237474745083064, "accuracy_std_dev": 0.02621443302538222, "run_count": 27, "total_questions": 5237, "avg_cost_per_task_usd": 0.12266296333778877, "avg_duration_per_task_ms": 159012.23353064733, "avg_output_tokens_per_task": 9966.579912163452, "last_run_timestamp": "2026-10-31T08:44:11.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Near AI", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02620280303030303, "avg_duration_per_task_ms": 592141.2247474748, "avg_output_tokens_per_task": 21737.315656565657, "last_run_timestamp": "2026-10-31T10:06:11.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1436656515151515, "avg_duration_per_task_ms": 89621.648989899, "avg_output_tokens_per_task": 15839.800505050505, "last_run_timestamp": "2026-10-01T19:18:40.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9231607898761836, "accuracy_std_dev": 0.009964755650903951, "run_count": 15, "total_questions": 2746, "avg_cost_per_task_usd": 0.14528911981063364, "avg_duration_per_task_ms": 123061.00109249818, "avg_output_tokens_per_task": 9541.6591405681, "last_run_timestamp": "2026-10-31T08:14:43.086Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DeepInfra", "accuracy": 0.9228069789473685, "accuracy_std_dev": 0.011126928806578514, "run_count": 3, "total_questions": 570, "avg_cost_per_task_usd": 0.00580749649122807, "avg_duration_per_task_ms": 218614.69122807018, "avg_output_tokens_per_task": 19521.97543859649, "last_run_timestamp": "2026-10-31T08:40:34.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "ParetoAlpha", "accuracy": 0.92256925, "accuracy_std_dev": 0.005618592528382903, "run_count": 5, "total_questions": 960, "avg_cost_per_task_usd": 0.148641165625, "avg_duration_per_task_ms": 385822.7, "avg_output_tokens_per_task": 9697.144791666668, "last_run_timestamp": "2026-10-31T08:38:36.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9225589999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14588472727272728, "avg_duration_per_task_ms": 63933.833333333336, "avg_output_tokens_per_task": 16085.368686868687, "last_run_timestamp": "2026-10-01T19:12:44.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.9221626470588234, "accuracy_std_dev": 0.007429213654384402, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.13078954872251933, "avg_duration_per_task_ms": 172274.93434343435, "avg_output_tokens_per_task": 11489.700237670826, "last_run_timestamp": "2026-10-31T09:01:05.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9219976666666666, "accuracy_std_dev": 0.0028614651647170097, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.08309612457912459, "avg_duration_per_task_ms": 49364.952861952865, "avg_output_tokens_per_task": 3748.658249158249, "last_run_timestamp": "2026-10-30T00:00:06.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DigitalOcean", "accuracy": 0.9208753333333334, "accuracy_std_dev": 0.003636621356271357, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.019656516835016836, "avg_duration_per_task_ms": 90437.9696969697, "avg_output_tokens_per_task": 20830.57239057239, "last_run_timestamp": "2026-10-31T08:09:30.885Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9208750000000001, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.026268675084175085, "avg_duration_per_task_ms": 25466.75420875421, "avg_output_tokens_per_task": 2308.39898989899, "last_run_timestamp": "2026-10-30T00:00:05.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.9208749999999999, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025590255892255893, "avg_duration_per_task_ms": 33502.22222222222, "avg_output_tokens_per_task": 2400.506734006734, "last_run_timestamp": "2026-10-02T23:17:42.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.9205386, "accuracy_std_dev": 0.004178401589124734, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.14947429494949496, "avg_duration_per_task_ms": 120588.33434343434, "avg_output_tokens_per_task": 9732.809090909092, "last_run_timestamp": "2026-10-31T08:09:30.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.9199569999999999, "accuracy_std_dev": 0.006554291127331654, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.17177113406795225, "avg_duration_per_task_ms": 299167.57943067036, "avg_output_tokens_per_task": 9764.863636363636, "last_run_timestamp": "2026-10-31T08:33:15.782Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Io Net", "accuracy": 0.9192646019184652, "accuracy_std_dev": 0.016279466419047314, "run_count": 3, "total_questions": 417, "avg_cost_per_task_usd": 0.01652008633093525, "avg_duration_per_task_ms": 447771.20383693045, "avg_output_tokens_per_task": 32878.398081534775, "last_run_timestamp": "2026-10-31T08:35:38.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02528019191919192, "avg_duration_per_task_ms": 32477.28787878788, "avg_output_tokens_per_task": 2369.5, "last_run_timestamp": "2026-10-01T08:24:24.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Azure", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01357689898989899, "avg_duration_per_task_ms": 18304.29292929293, "avg_output_tokens_per_task": 2104.2979797979797, "last_run_timestamp": "2026-10-28T00:00:04.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.9190073210272871, "accuracy_std_dev": 0.01267182538981892, "run_count": 13, "total_questions": 2492, "avg_cost_per_task_usd": 0.1293751532905297, "avg_duration_per_task_ms": 116839.90609951846, "avg_output_tokens_per_task": 8587.111556982343, "last_run_timestamp": "2026-10-31T08:14:19.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.918981375, "accuracy_std_dev": 0.00679620392089402, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12805398863636364, "avg_duration_per_task_ms": 541652.8712121212, "avg_output_tokens_per_task": 37116.541666666664, "last_run_timestamp": "2026-10-03T09:05:48.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "SiliconFlow", "accuracy": 0.9186303333333333, "accuracy_std_dev": 0.010408091158111372, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025292042087542085, "avg_duration_per_task_ms": 160402.3451178451, "avg_output_tokens_per_task": 20957.713804713803, "last_run_timestamp": "2026-10-31T08:50:50.713Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9185276396965866, "accuracy_std_dev": 0.01024254822724797, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.13179984070796458, "avg_duration_per_task_ms": 262587.70375052677, "avg_output_tokens_per_task": 10414.297935103245, "last_run_timestamp": "2026-10-31T08:21:51.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9177144177922683, "accuracy_std_dev": 0.013751154902061724, "run_count": 11, "total_questions": 2147, "avg_cost_per_task_usd": 0.2460820894271076, "avg_duration_per_task_ms": 409070.95482068, "avg_output_tokens_per_task": 37154.30647414998, "last_run_timestamp": "2026-10-31T08:34:22.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.9175082499999999, "accuracy_std_dev": 0.003571399938516188, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.025937152356902356, "avg_duration_per_task_ms": 19174.19191919192, "avg_output_tokens_per_task": 2235.470117845118, "last_run_timestamp": "2026-10-31T08:20:35.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.9165345411718132, "accuracy_std_dev": 0.00728827618680496, "run_count": 13, "total_questions": 2526, "avg_cost_per_task_usd": 0.14153307442596988, "avg_duration_per_task_ms": 283509.3226444972, "avg_output_tokens_per_task": 9267.701108471892, "last_run_timestamp": "2026-10-24T08:44:42.687Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.9162757342365815, "accuracy_std_dev": 0.00785699694603481, "run_count": 10, "total_questions": 1919, "avg_cost_per_task_usd": 0.09128974934861907, "avg_duration_per_task_ms": 645077.3731109953, "avg_output_tokens_per_task": 39631.607087024495, "last_run_timestamp": "2026-10-31T08:24:03.784Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.9159771681985294, "accuracy_std_dev": 0.02535263090623474, "run_count": 22, "total_questions": 4352, "avg_cost_per_task_usd": 0.14850530836397058, "avg_duration_per_task_ms": 399083.8046875, "avg_output_tokens_per_task": 9306.142463235294, "last_run_timestamp": "2026-10-31T09:12:10.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.9158379972375691, "accuracy_std_dev": 0.012441503983078046, "run_count": 13, "total_questions": 1810, "avg_cost_per_task_usd": 0.11735693314917128, "avg_duration_per_task_ms": 179712.15801104973, "avg_output_tokens_per_task": 8118.493370165746, "last_run_timestamp": "2026-10-31T08:07:17.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9149830000000001, "accuracy_std_dev": 0.009335323856192691, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.15949898674242427, "avg_duration_per_task_ms": 579778.1357323233, "avg_output_tokens_per_task": 39960.98863636364, "last_run_timestamp": "2026-10-31T08:49:58.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Baidu", "accuracy": 0.914552902173913, "accuracy_std_dev": 0.010001261731512806, "run_count": 3, "total_questions": 552, "avg_cost_per_task_usd": 0.01860343115942029, "avg_duration_per_task_ms": 136525.615942029, "avg_output_tokens_per_task": 19948.84963768116, "last_run_timestamp": "2026-10-31T08:12:46.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra", "model_slug": "openai/gpt-6.1-terra", "model_name": "GPT-6.1-Terra", "provider_name": "auto-routing", "accuracy": 0.9141410000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0075055656565656565, "avg_duration_per_task_ms": 21128.91414141414, "avg_output_tokens_per_task": 1395.4343434343434, "last_run_timestamp": "2026-10-27T03:32:35.022Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.9129758461538462, "accuracy_std_dev": 0.013360432120177481, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.033474385003885006, "avg_duration_per_task_ms": 223207.8550893551, "avg_output_tokens_per_task": 27728.331002331004, "last_run_timestamp": "2026-10-31T16:05:09.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.9129648851570964, "accuracy_std_dev": 0.008195106272424223, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.0985225175153485, "avg_duration_per_task_ms": 283575.7085590466, "avg_output_tokens_per_task": 25018.89021307331, "last_run_timestamp": "2026-10-31T08:40:32.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.9126954247352496, "accuracy_std_dev": 0.015274129139141904, "run_count": 14, "total_questions": 2644, "avg_cost_per_task_usd": 0.16414042738275345, "avg_duration_per_task_ms": 497069.49243570346, "avg_output_tokens_per_task": 11341.725037821483, "last_run_timestamp": "2026-10-31T08:37:30.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Alibaba", "accuracy": 0.91245775, "accuracy_std_dev": 0.005709247427419856, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.027895760101010105, "avg_duration_per_task_ms": 177535.13888888888, "avg_output_tokens_per_task": 21414.281565656565, "last_run_timestamp": "2026-10-31T08:15:24.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.911926105263158, "accuracy_std_dev": 0.013578898854258508, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.1411968139287613, "avg_duration_per_task_ms": 616719.149122807, "avg_output_tokens_per_task": 40116.86735778841, "last_run_timestamp": "2026-10-31T08:31:11.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Venice", "accuracy": 0.911616, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.024379722222222223, "avg_duration_per_task_ms": 102982.82575757576, "avg_output_tokens_per_task": 20129.815656565657, "last_run_timestamp": "2026-10-31T08:34:49.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "OpenAI", "accuracy": 0.910774, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.011462444444444444, "avg_duration_per_task_ms": 27557.444444444445, "avg_output_tokens_per_task": 2133.969696969697, "last_run_timestamp": "2026-10-28T00:00:04.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.9106891772151898, "accuracy_std_dev": 0.008562817793226714, "run_count": 5, "total_questions": 948, "avg_cost_per_task_usd": 0.14278316033755276, "avg_duration_per_task_ms": 333972.91455696203, "avg_output_tokens_per_task": 37406.36181434599, "last_run_timestamp": "2026-10-31T08:31:11.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "GMICloud", "accuracy": 0.9103534999999999, "accuracy_std_dev": 0.003644869990822727, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.019048060606060607, "avg_duration_per_task_ms": 135272.13636363635, "avg_output_tokens_per_task": 22610.929292929293, "last_run_timestamp": "2026-10-31T08:28:22.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Modal", "accuracy": 0.909632, "accuracy_std_dev": 0.0055275000000000185, "run_count": 2, "total_questions": 308, "avg_cost_per_task_usd": 0.05270355519480519, "avg_duration_per_task_ms": 55285.37662337662, "avg_output_tokens_per_task": 12143.461038961039, "last_run_timestamp": "2026-10-31T08:15:03.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9095115, "accuracy_std_dev": 0.002759842885020819, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0416229191919192, "avg_duration_per_task_ms": 138307.48484848486, "avg_output_tokens_per_task": 27969.503787878788, "last_run_timestamp": "2026-10-01T16:11:23.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.9094272, "accuracy_std_dev": 0.00547072746168187, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025930728282828285, "avg_duration_per_task_ms": 177886.11212121212, "avg_output_tokens_per_task": 28121.050505050505, "last_run_timestamp": "2026-10-03T09:34:35.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StreamLake", "accuracy": 0.9090910000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018701613636363638, "avg_duration_per_task_ms": 137003.63383838383, "avg_output_tokens_per_task": 22059.484848484848, "last_run_timestamp": "2026-10-31T08:36:23.829Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1305062777777778, "avg_duration_per_task_ms": 540279.6767676767, "avg_output_tokens_per_task": 38448.40404040404, "last_run_timestamp": "2026-10-01T09:14:23.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0054125, "avg_duration_per_task_ms": 235839.91414141413, "avg_output_tokens_per_task": 10749.717171717171, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2221867676767677, "avg_duration_per_task_ms": 298883.5202020202, "avg_output_tokens_per_task": 29352.99494949495, "last_run_timestamp": "2026-09-27T00:06:19.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.9090908333333333, "accuracy_std_dev": 0.008189992133017543, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03826549410774411, "avg_duration_per_task_ms": 134585.28703703705, "avg_output_tokens_per_task": 28750.484848484848, "last_run_timestamp": "2026-10-27T00:36:05.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9086699166666665, "accuracy_std_dev": 0.015350820322588266, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03558214772727273, "avg_duration_per_task_ms": 301721.86742424243, "avg_output_tokens_per_task": 29561.46254208754, "last_run_timestamp": "2026-10-31T16:17:15.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.9086667275553827, "accuracy_std_dev": 0.04751984064982348, "run_count": 13, "total_questions": 2573, "avg_cost_per_task_usd": 0.14066514496696458, "avg_duration_per_task_ms": 326890.27283326857, "avg_output_tokens_per_task": 10307.279440342014, "last_run_timestamp": "2026-10-31T09:37:23.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.9083779032258064, "accuracy_std_dev": 0.022205862232752862, "run_count": 12, "total_questions": 2232, "avg_cost_per_task_usd": 0.1474674775985663, "avg_duration_per_task_ms": 555743.3785842294, "avg_output_tokens_per_task": 37926.89829749104, "last_run_timestamp": "2026-10-31T08:44:11.309Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.9081554444444445, "accuracy_std_dev": 0.008876832231665518, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.033613874298540965, "avg_duration_per_task_ms": 210737.89562289562, "avg_output_tokens_per_task": 27019.509539842875, "last_run_timestamp": "2026-10-31T08:40:16.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Mancer 2", "accuracy": 0.9078013574468085, "accuracy_std_dev": 0.012928926018815361, "run_count": 3, "total_questions": 470, "avg_cost_per_task_usd": 0.01807536170212766, "avg_duration_per_task_ms": 357786.7425531915, "avg_output_tokens_per_task": 17947.382978723403, "last_run_timestamp": "2026-10-31T09:05:48.501Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.9076662307692308, "accuracy_std_dev": 0.02122131075178129, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.15455806177156176, "avg_duration_per_task_ms": 600133.6076146077, "avg_output_tokens_per_task": 38964.08663558664, "last_run_timestamp": "2026-10-31T08:28:10.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.9076534680851064, "accuracy_std_dev": 0.005694785441280666, "run_count": 6, "total_questions": 1128, "avg_cost_per_task_usd": 0.025873300531914892, "avg_duration_per_task_ms": 176651.64184397162, "avg_output_tokens_per_task": 28451.522163120568, "last_run_timestamp": "2026-10-31T08:42:09.092Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.9074649821150856, "accuracy_std_dev": 0.007911648334003893, "run_count": 13, "total_questions": 2572, "avg_cost_per_task_usd": 0.030564498833592534, "avg_duration_per_task_ms": 392326.3246500778, "avg_output_tokens_per_task": 31615.309875583203, "last_run_timestamp": "2026-10-31T16:32:36.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cohere", "accuracy": 0.9074074999999998, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014508287878787877, "avg_duration_per_task_ms": 90086.95707070707, "avg_output_tokens_per_task": 21790.61111111111, "last_run_timestamp": "2026-10-31T08:13:11.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Nebius", "accuracy": 0.9074072499999999, "accuracy_std_dev": 0.004610545920766882, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.025605553030303027, "avg_duration_per_task_ms": 96285.60858585859, "avg_output_tokens_per_task": 21160.45580808081, "last_run_timestamp": "2026-10-31T09:42:32.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.9071909310747666, "accuracy_std_dev": 0.03850936704324035, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.1417193566978193, "avg_duration_per_task_ms": 214397.28777258567, "avg_output_tokens_per_task": 9503.626168224298, "last_run_timestamp": "2026-10-31T08:40:21.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.9071887349785409, "accuracy_std_dev": 0.014345789366918764, "run_count": 5, "total_questions": 932, "avg_cost_per_task_usd": 0.12532361909871245, "avg_duration_per_task_ms": 1379721.1190987125, "avg_output_tokens_per_task": 49889.66094420601, "last_run_timestamp": "2026-10-31T08:27:43.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.9071266666666665, "accuracy_std_dev": 0.007235753258799129, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.009577915824915826, "avg_duration_per_task_ms": 164204.16077441076, "avg_output_tokens_per_task": 18450.463804713803, "last_run_timestamp": "2026-10-31T08:24:03.698Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.9070474558880683, "accuracy_std_dev": 0.028977319190080154, "run_count": 13, "total_questions": 2573, "avg_cost_per_task_usd": 0.0985262642829382, "avg_duration_per_task_ms": 564503.555771473, "avg_output_tokens_per_task": 39384.385542168675, "last_run_timestamp": "2026-10-31T08:39:28.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Modal", "accuracy": 0.9070292653061225, "accuracy_std_dev": 0.005723067708842884, "run_count": 3, "total_questions": 588, "avg_cost_per_task_usd": 0.021270954081632654, "avg_duration_per_task_ms": 117754.6887755102, "avg_output_tokens_per_task": 21116.049319727892, "last_run_timestamp": "2026-10-31T08:51:31.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/switchyard", "model_slug": "nvidia/switchyard", "model_name": "NVIDIA: Switchyard", "provider_name": "auto-routing", "accuracy": 0.9067937377877597, "accuracy_std_dev": 0.03179893182614709, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.01609888826501965, "avg_duration_per_task_ms": 58211.32790567097, "avg_output_tokens_per_task": 3297.93037619315, "last_run_timestamp": "2026-10-16T18:32:13.741Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Sail Research", "accuracy": 0.9065654999999998, "accuracy_std_dev": 0.006645834303030217, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0029837954545454543, "avg_duration_per_task_ms": 222248.6893939394, "avg_output_tokens_per_task": 19714.793771043773, "last_run_timestamp": "2026-10-31T08:45:39.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.9062846666666666, "accuracy_std_dev": 0.0031744380430068155, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.011460047138047139, "avg_duration_per_task_ms": 97210.29461279462, "avg_output_tokens_per_task": 21369.54713804714, "last_run_timestamp": "2026-10-31T08:27:43.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Fireworks", "accuracy": 0.9062632644787644, "accuracy_std_dev": 0.007463126815174401, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.024727387387387386, "avg_duration_per_task_ms": 89200.08815958817, "avg_output_tokens_per_task": 20548.897683397685, "last_run_timestamp": "2026-10-31T08:46:47.079Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.9061447499999998, "accuracy_std_dev": 0.005102889738912664, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.007892248737373738, "avg_duration_per_task_ms": 123987.29040404041, "avg_output_tokens_per_task": 16306.166666666666, "last_run_timestamp": "2026-10-31T08:53:36.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.9060041666666666, "accuracy_std_dev": 0.006179339377222198, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.032878036195286196, "avg_duration_per_task_ms": 809018.2592592592, "avg_output_tokens_per_task": 37448.21212121212, "last_run_timestamp": "2026-10-31T08:23:27.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.9059643571428573, "accuracy_std_dev": 0.02530737982319202, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.039628380591630596, "avg_duration_per_task_ms": 275120.5137085137, "avg_output_tokens_per_task": 30935.2702020202, "last_run_timestamp": "2026-10-31T16:23:41.132Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Decart", "accuracy": 0.9059342499999998, "accuracy_std_dev": 0.010910172663874526, "run_count": 3, "total_questions": 528, "avg_cost_per_task_usd": 0.008819143939393941, "avg_duration_per_task_ms": 163195.54924242425, "avg_output_tokens_per_task": 22179.03787878788, "last_run_timestamp": "2026-10-31T08:49:09.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.9057238888888889, "accuracy_std_dev": 0.009356618102764875, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03432713860830528, "avg_duration_per_task_ms": 196757.96408529743, "avg_output_tokens_per_task": 28444.87317620651, "last_run_timestamp": "2026-10-31T08:29:56.453Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9054646923076923, "accuracy_std_dev": 0.020414937929725787, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.036716465034965035, "avg_duration_per_task_ms": 253595.11771561773, "avg_output_tokens_per_task": 30738.896658896658, "last_run_timestamp": "2026-10-31T16:09:19.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Makora", "accuracy": 0.9054373465045593, "accuracy_std_dev": 0.0037554479359991244, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.026207787234042552, "avg_duration_per_task_ms": 127141.36068895644, "avg_output_tokens_per_task": 21741.29989868288, "last_run_timestamp": "2026-10-31T08:31:11.848Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.905376, "accuracy_std_dev": null, "run_count": 1, "total_questions": 155, "avg_cost_per_task_usd": 0.09219292258064517, "avg_duration_per_task_ms": 892432.9419354838, "avg_output_tokens_per_task": 36790.02580645162, "last_run_timestamp": "2026-10-02T06:44:44.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9053629285714285, "accuracy_std_dev": 0.024694616907520228, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.03438339466089466, "avg_duration_per_task_ms": 227033.76984126985, "avg_output_tokens_per_task": 29037.088383838385, "last_run_timestamp": "2026-10-31T16:04:48.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.9053496666666667, "accuracy_std_dev": 0.008612696635394351, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02068693490460157, "avg_duration_per_task_ms": 208793.50953984287, "avg_output_tokens_per_task": 28854.00224466891, "last_run_timestamp": "2026-10-31T09:01:06.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.9052795516491482, "accuracy_std_dev": 0.020697851140189134, "run_count": 14, "total_questions": 2759, "avg_cost_per_task_usd": 0.1211796118158753, "avg_duration_per_task_ms": 328498.6843059079, "avg_output_tokens_per_task": 8296.681406306632, "last_run_timestamp": "2026-10-31T08:25:00.194Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Particle AI", "accuracy": 0.9051626666666668, "accuracy_std_dev": 0.007816314746875412, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017028636363636364, "avg_duration_per_task_ms": 78357.59090909091, "avg_output_tokens_per_task": 16892.26430976431, "last_run_timestamp": "2026-10-31T08:58:04.691Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.9049724513274336, "accuracy_std_dev": 0.008906991898924113, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.020146505689001264, "avg_duration_per_task_ms": 644404.02528445, "avg_output_tokens_per_task": 42703.32616940582, "last_run_timestamp": "2026-10-28T03:59:43.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Wafer", "accuracy": 0.9048818749999998, "accuracy_std_dev": 0.008666358814945001, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008252799242424242, "avg_duration_per_task_ms": 82042.59659090909, "avg_output_tokens_per_task": 16383.162878787878, "last_run_timestamp": "2026-10-31T08:56:01.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Makora", "accuracy": 0.9048634068157615, "accuracy_std_dev": 0.012266915351464668, "run_count": 5, "total_questions": 939, "avg_cost_per_task_usd": 0.06399653567625134, "avg_duration_per_task_ms": 75580.11395101172, "avg_output_tokens_per_task": 14917.831735889244, "last_run_timestamp": "2026-10-31T08:13:57.643Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.9046013333333334, "accuracy_std_dev": 0.009152126540986068, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02685950505050505, "avg_duration_per_task_ms": 136981.2441077441, "avg_output_tokens_per_task": 22236.84175084175, "last_run_timestamp": "2026-10-31T08:31:11.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Ionstream", "accuracy": 0.9046013333333334, "accuracy_std_dev": 0.003967811851498118, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.016546668350168353, "avg_duration_per_task_ms": 140240.16835016836, "avg_output_tokens_per_task": 21954.457912457914, "last_run_timestamp": "2026-10-31T08:27:53.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "AtlasCloud", "accuracy": 0.9046013333333334, "accuracy_std_dev": 0.008285549120138134, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01933321717171717, "avg_duration_per_task_ms": 139538.02861952863, "avg_output_tokens_per_task": 20900.63973063973, "last_run_timestamp": "2026-10-31T08:20:08.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.9045483013645222, "accuracy_std_dev": 0.011375036654859972, "run_count": 13, "total_questions": 2565, "avg_cost_per_task_usd": 0.027518346978557498, "avg_duration_per_task_ms": 307235.2007797271, "avg_output_tokens_per_task": 22778.734892787525, "last_run_timestamp": "2026-10-31T16:12:28.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.9044004522003033, "accuracy_std_dev": 0.00856689234728672, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.014848743045017704, "avg_duration_per_task_ms": 370376.5558927668, "avg_output_tokens_per_task": 28855.582195245323, "last_run_timestamp": "2026-10-31T08:03:09.643Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.9042316703786192, "accuracy_std_dev": 0.0031656583896560934, "run_count": 5, "total_questions": 898, "avg_cost_per_task_usd": 0.22350682405345215, "avg_duration_per_task_ms": 441723.66035634745, "avg_output_tokens_per_task": 42286.14922048998, "last_run_timestamp": "2026-10-31T08:44:55.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.9041981924485772, "accuracy_std_dev": 0.01113877499359892, "run_count": 18, "total_questions": 3549, "avg_cost_per_task_usd": 0.027435362073823615, "avg_duration_per_task_ms": 185052.6246830093, "avg_output_tokens_per_task": 25932.813750352212, "last_run_timestamp": "2026-10-31T08:28:13.141Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.9040405, "accuracy_std_dev": 0.011784499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13174327272727274, "avg_duration_per_task_ms": 249918.9898989899, "avg_output_tokens_per_task": 43791.378787878784, "last_run_timestamp": "2026-10-01T19:18:35.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "Kimi K3.1 Pro", "provider_name": "auto-routing", "accuracy": 0.9040400000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.021051848484848486, "avg_duration_per_task_ms": 58149.92929292929, "avg_output_tokens_per_task": 2035.7424242424242, "last_run_timestamp": "2026-10-26T19:26:14.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.903508802631579, "accuracy_std_dev": 0.0984941779678147, "run_count": 3, "total_questions": 456, "avg_cost_per_task_usd": 0.11285829166666667, "avg_duration_per_task_ms": 423954.7916666667, "avg_output_tokens_per_task": 16846.184210526317, "last_run_timestamp": "2026-10-01T16:04:59.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Mistral", "accuracy": 0.9034793333333332, "accuracy_std_dev": 0.005722799567422317, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025755799663299665, "avg_duration_per_task_ms": 89661.28282828283, "avg_output_tokens_per_task": 21298.74074074074, "last_run_timestamp": "2026-10-31T09:51:08.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimeIntellect", "accuracy": 0.9033668, "accuracy_std_dev": 0.005061737504059227, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.026716864646464646, "avg_duration_per_task_ms": 106368.8292929293, "avg_output_tokens_per_task": 22085.938383838384, "last_run_timestamp": "2026-10-31T08:13:57.548Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.9031985, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14226843434343436, "avg_duration_per_task_ms": 149341.0782828283, "avg_output_tokens_per_task": 9356.287878787878, "last_run_timestamp": "2026-10-01T19:14:55.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.9031337692307693, "accuracy_std_dev": 0.017540530612769802, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.03434532362082362, "avg_duration_per_task_ms": 309648.5489510489, "avg_output_tokens_per_task": 30300.46114996115, "last_run_timestamp": "2026-10-31T16:18:48.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.9030760960809103, "accuracy_std_dev": 0.01701828861577222, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.10602527054361567, "avg_duration_per_task_ms": 544239.2713864307, "avg_output_tokens_per_task": 37272.33965444585, "last_run_timestamp": "2026-10-31T08:11:08.576Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Together", "accuracy": 0.9029179999999999, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.023974107744107742, "avg_duration_per_task_ms": 50011.877104377105, "avg_output_tokens_per_task": 19809.430976430976, "last_run_timestamp": "2026-10-31T08:51:04.442Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9027699539584503, "accuracy_std_dev": 0.006295075818094274, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.028251830432341377, "avg_duration_per_task_ms": 172539.25996631107, "avg_output_tokens_per_task": 28841.043234138124, "last_run_timestamp": "2026-10-31T08:31:10.643Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.902357, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007275146464646465, "avg_duration_per_task_ms": 217667.1313131313, "avg_output_tokens_per_task": 25683.540404040403, "last_run_timestamp": "2026-08-08T09:30:52.456Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.902357, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1437863712121212, "avg_duration_per_task_ms": 149585.63636363635, "avg_output_tokens_per_task": 9456.126262626263, "last_run_timestamp": "2026-10-01T19:14:23.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.9023568, "accuracy_std_dev": 0.003838970664123373, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0695745484848485, "avg_duration_per_task_ms": 344238.6484848485, "avg_output_tokens_per_task": 16158.315151515151, "last_run_timestamp": "2026-10-31T08:52:05.708Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.9020761666666667, "accuracy_std_dev": 0.005271695534223824, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05407119865319866, "avg_duration_per_task_ms": 30482.292087542086, "avg_output_tokens_per_task": 2495.6936026936028, "last_run_timestamp": "2026-10-31T08:27:47.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.9015422819956616, "accuracy_std_dev": 0.037796971040504466, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.09285763702096891, "avg_duration_per_task_ms": 766396.866955893, "avg_output_tokens_per_task": 36238.82212581345, "last_run_timestamp": "2026-10-31T16:24:40.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "auto-routing", "accuracy": 0.9015150000000001, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0736977297979798, "avg_duration_per_task_ms": 138589.0126262626, "avg_output_tokens_per_task": 17125.45707070707, "last_run_timestamp": "2026-10-31T08:09:26.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-1017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "Deepseek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.901515, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006598151515151515, "avg_duration_per_task_ms": 72990.44444444444, "avg_output_tokens_per_task": 11623.267676767677, "last_run_timestamp": "2026-10-23T06:09:54.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "BaseTen", "accuracy": 0.901515, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0720575808080808, "avg_duration_per_task_ms": 100990.45707070707, "avg_output_tokens_per_task": 16730.29797979798, "last_run_timestamp": "2026-10-31T08:17:04.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.901515, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.16319313383838385, "avg_duration_per_task_ms": 88415.74242424243, "avg_output_tokens_per_task": 9762.232323232323, "last_run_timestamp": "2026-10-01T19:12:25.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.901320769230769, "accuracy_std_dev": 0.006035282463007334, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.2067400357420358, "avg_duration_per_task_ms": 98162.20415695416, "avg_output_tokens_per_task": 7860.721445221445, "last_run_timestamp": "2026-10-31T08:46:46.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.9012343333333332, "accuracy_std_dev": 0.0037221826780652403, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03380553956228956, "avg_duration_per_task_ms": 826187.4343434343, "avg_output_tokens_per_task": 38504.42255892256, "last_run_timestamp": "2026-10-31T08:14:20.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.9011541428571429, "accuracy_std_dev": 0.004473489479416358, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.053932944444444436, "avg_duration_per_task_ms": 26662.532467532466, "avg_output_tokens_per_task": 2480.9805194805194, "last_run_timestamp": "2026-10-31T08:25:18.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Crusoe", "accuracy": 0.9010944999999999, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00798360101010101, "avg_duration_per_task_ms": 317918.4797979798, "avg_output_tokens_per_task": 28057.032828282827, "last_run_timestamp": "2026-10-31T16:08:33.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.901078703900709, "accuracy_std_dev": 0.008642828253125116, "run_count": 13, "total_questions": 2256, "avg_cost_per_task_usd": 0.025328547429078016, "avg_duration_per_task_ms": 170551.65514184398, "avg_output_tokens_per_task": 20949.003989361703, "last_run_timestamp": "2026-10-31T16:35:50.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.9009462380396732, "accuracy_std_dev": 0.019684329717769646, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.12647589887203423, "avg_duration_per_task_ms": 518480.37456242705, "avg_output_tokens_per_task": 38463.91637495138, "last_run_timestamp": "2026-10-31T08:17:28.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.9006730000000001, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.010972851010101009, "avg_duration_per_task_ms": 117479.04797979798, "avg_output_tokens_per_task": 19415.13888888889, "last_run_timestamp": "2026-10-31T08:11:46.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.9005650372881355, "accuracy_std_dev": 0.0005644881654108368, "run_count": 3, "total_questions": 590, "avg_cost_per_task_usd": 0.013123520338983051, "avg_duration_per_task_ms": 181977.62711864407, "avg_output_tokens_per_task": 21777.491525423728, "last_run_timestamp": "2026-10-31T08:31:11.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.9005527218243498, "accuracy_std_dev": 0.007008802337352753, "run_count": 14, "total_questions": 2653, "avg_cost_per_task_usd": 0.012335847719562759, "avg_duration_per_task_ms": 174702.59517527328, "avg_output_tokens_per_task": 26019.96117602714, "last_run_timestamp": "2026-10-31T08:07:19.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.9005006153846152, "accuracy_std_dev": 0.0058913432128904816, "run_count": 39, "total_questions": 7722, "avg_cost_per_task_usd": 0.21606604428904433, "avg_duration_per_task_ms": 93452.96775446775, "avg_output_tokens_per_task": 7881.437192437193, "last_run_timestamp": "2026-10-31T08:23:44.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Nebius", "accuracy": 0.9002525, "accuracy_std_dev": 0.004014983343676526, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.014062290404040403, "avg_duration_per_task_ms": 87702.04797979798, "avg_output_tokens_per_task": 13895.555555555555, "last_run_timestamp": "2026-10-31T08:28:22.736Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.9000574515125321, "accuracy_std_dev": 0.11608222219996657, "run_count": 31, "total_questions": 5785, "avg_cost_per_task_usd": 0.16450815436473637, "avg_duration_per_task_ms": 193869.64321521175, "avg_output_tokens_per_task": 10791.89334485739, "last_run_timestamp": "2026-10-31T09:24:16.791Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9000319047619048, "accuracy_std_dev": 0.005076517459315892, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.05824434295334296, "avg_duration_per_task_ms": 35780.01707551708, "avg_output_tokens_per_task": 2517.5358345358345, "last_run_timestamp": "2026-10-31T09:01:04.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.8998963581973581, "accuracy_std_dev": 0.03142747825284958, "run_count": 9, "total_questions": 1287, "avg_cost_per_task_usd": 0.023476212121212123, "avg_duration_per_task_ms": 815676.9502719502, "avg_output_tokens_per_task": 21719.649572649574, "last_run_timestamp": "2026-10-31T16:03:21.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "GMICloud", "accuracy": 0.8998314999999999, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11599434848484849, "avg_duration_per_task_ms": 1589231.9873737374, "avg_output_tokens_per_task": 46211.58585858586, "last_run_timestamp": "2026-10-31T08:31:10.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Cohere", "accuracy": 0.8996625716694773, "accuracy_std_dev": 0.0020759481367959394, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.11324433389544686, "avg_duration_per_task_ms": 858665.3389544688, "avg_output_tokens_per_task": 45077.612141652615, "last_run_timestamp": "2026-10-31T08:28:18.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8995687234401349, "accuracy_std_dev": 0.007447984730328704, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.0197627712197864, "avg_duration_per_task_ms": 382016.3063518831, "avg_output_tokens_per_task": 22859.64811691962, "last_run_timestamp": "2026-10-31T08:33:02.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8994372877637131, "accuracy_std_dev": 0.01977995405637288, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.03544181097046414, "avg_duration_per_task_ms": 152645.21687763714, "avg_output_tokens_per_task": 26597.288607594935, "last_run_timestamp": "2026-10-03T09:28:39.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.8993857647058824, "accuracy_std_dev": 0.018671156495670343, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.07737615121806299, "avg_duration_per_task_ms": 217682.72222222222, "avg_output_tokens_per_task": 30235.886809269163, "last_run_timestamp": "2026-10-31T16:10:09.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8993638888888887, "accuracy_std_dev": 0.004029367042778795, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.030411617283950615, "avg_duration_per_task_ms": 237553.6509539843, "avg_output_tokens_per_task": 26633.1290684624, "last_run_timestamp": "2026-10-03T10:19:34.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.8993264, "accuracy_std_dev": 0.006511664598242117, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13052546969696968, "avg_duration_per_task_ms": 30874.333333333332, "avg_output_tokens_per_task": 2403.130303030303, "last_run_timestamp": "2026-10-01T08:24:30.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.8992302857142856, "accuracy_std_dev": 0.0048260090792136764, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.05735562409812409, "avg_duration_per_task_ms": 25146.343674843676, "avg_output_tokens_per_task": 2479.3174603174602, "last_run_timestamp": "2026-10-31T09:06:56.339Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.8991100000000001, "accuracy_std_dev": 0.005600276243186554, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.05768347799422799, "avg_duration_per_task_ms": 26053.75036075036, "avg_output_tokens_per_task": 2530.0901875901877, "last_run_timestamp": "2026-10-31T08:22:04.658Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.8989897750000003, "accuracy_std_dev": 0.005963928313148534, "run_count": 40, "total_questions": 7920, "avg_cost_per_task_usd": 0.22275342159090905, "avg_duration_per_task_ms": 94239.73522727273, "avg_output_tokens_per_task": 8122.755176767677, "last_run_timestamp": "2026-10-31T08:31:12.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.8987954230769233, "accuracy_std_dev": 0.0076484229655182515, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.12695155264180263, "avg_duration_per_task_ms": 58394.58430458431, "avg_output_tokens_per_task": 4732.304195804196, "last_run_timestamp": "2026-10-31T08:31:13.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.8987492142857141, "accuracy_std_dev": 0.014339465397379814, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.004243736652236653, "avg_duration_per_task_ms": 360908.05663780664, "avg_output_tokens_per_task": 23278.13023088023, "last_run_timestamp": "2026-10-31T08:39:08.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.8987081781963057, "accuracy_std_dev": 0.03249433928983099, "run_count": 14, "total_questions": 2761, "avg_cost_per_task_usd": 0.14709861970300617, "avg_duration_per_task_ms": 823480.3976819993, "avg_output_tokens_per_task": 36846.00507062658, "last_run_timestamp": "2026-10-31T08:15:03.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "BaseTen", "accuracy": 0.8986319552238805, "accuracy_std_dev": 0.009874782731787053, "run_count": 3, "total_questions": 536, "avg_cost_per_task_usd": 0.02413633582089552, "avg_duration_per_task_ms": 93980.86567164179, "avg_output_tokens_per_task": 19966.47014925373, "last_run_timestamp": "2026-10-31T08:27:51.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.8986254025773196, "accuracy_std_dev": 0.041870475079224964, "run_count": 10, "total_questions": 1940, "avg_cost_per_task_usd": 0.047417563402061845, "avg_duration_per_task_ms": 459421.87164948456, "avg_output_tokens_per_task": 15074.60618556701, "last_run_timestamp": "2026-10-31T16:08:56.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna", "model_slug": "openai/gpt-6-luna", "model_name": "GPT-6-Luna", "provider_name": "auto-routing", "accuracy": 0.898477, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.003442197969543147, "avg_duration_per_task_ms": 73312.30456852792, "avg_output_tokens_per_task": 6831.9390862944165, "last_run_timestamp": "2026-10-08T13:29:57.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8981480833333332, "accuracy_std_dev": 0.007606843601415013, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.19857798400673404, "avg_duration_per_task_ms": 95247.20033670033, "avg_output_tokens_per_task": 7731.943181818182, "last_run_timestamp": "2026-10-31T08:27:45.200Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.8981480000000002, "accuracy_std_dev": 0.016043822300187693, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.019599273088023093, "avg_duration_per_task_ms": 202557.83225108226, "avg_output_tokens_per_task": 25548.025613275615, "last_run_timestamp": "2026-10-31T09:18:59.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.898148, "accuracy_std_dev": 0.012012972737836369, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03667170117845118, "avg_duration_per_task_ms": 157797.89814814815, "avg_output_tokens_per_task": 27519.53787878788, "last_run_timestamp": "2026-10-03T09:28:24.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.8981478333333334, "accuracy_std_dev": 0.007386487717823356, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.12543263804713806, "avg_duration_per_task_ms": 965889.989057239, "avg_output_tokens_per_task": 49917.90404040404, "last_run_timestamp": "2026-10-31T08:45:01.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.89814775, "accuracy_std_dev": 0.0055197238778311765, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.11915268055555557, "avg_duration_per_task_ms": 825703.6161616162, "avg_output_tokens_per_task": 47450.94823232323, "last_run_timestamp": "2026-10-31T08:07:17.850Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.8980075833333333, "accuracy_std_dev": 0.004784274352123693, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.23032522264309765, "avg_duration_per_task_ms": 110445.48653198653, "avg_output_tokens_per_task": 8998.862373737375, "last_run_timestamp": "2026-10-31T08:24:37.252Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "StreamLake", "accuracy": 0.8978673333333335, "accuracy_std_dev": 0.007570651065499923, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03724634175084175, "avg_duration_per_task_ms": 745311.5892255892, "avg_output_tokens_per_task": 37370.69696969697, "last_run_timestamp": "2026-10-31T08:12:46.423Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.8977871428571428, "accuracy_std_dev": 0.005599042607665047, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.15678494083694083, "avg_duration_per_task_ms": 138524.9898989899, "avg_output_tokens_per_task": 9631.916305916306, "last_run_timestamp": "2026-10-01T19:18:24.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.8976429, "accuracy_std_dev": 0.019280368027867086, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.024536105555555552, "avg_duration_per_task_ms": 164306.0883838384, "avg_output_tokens_per_task": 28203.131818181817, "last_run_timestamp": "2026-10-31T08:31:11.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8973063333333333, "accuracy_std_dev": 0.004955863351178606, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03703221212121212, "avg_duration_per_task_ms": 137593.69696969696, "avg_output_tokens_per_task": 26012.949494949495, "last_run_timestamp": "2026-10-02T23:26:10.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.8973062307692307, "accuracy_std_dev": 0.017706017439337668, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.030415017871017868, "avg_duration_per_task_ms": 251068.9052059052, "avg_output_tokens_per_task": 27243.771173271172, "last_run_timestamp": "2026-10-31T08:36:32.971Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Crusoe", "accuracy": 0.897306, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.026144708754208754, "avg_duration_per_task_ms": 176560.02693602693, "avg_output_tokens_per_task": 21638.170033670034, "last_run_timestamp": "2026-10-31T08:57:38.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Together", "accuracy": 0.8973059999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07477493686868687, "avg_duration_per_task_ms": 142684.398989899, "avg_output_tokens_per_task": 17437.89393939394, "last_run_timestamp": "2026-10-31T08:49:18.270Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Nebius", "accuracy": 0.8970256666666665, "accuracy_std_dev": 0.00667522615513675, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06776180639730639, "avg_duration_per_task_ms": 222686.2255892256, "avg_output_tokens_per_task": 15778.796296296296, "last_run_timestamp": "2026-10-31T08:27:47.368Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.8970215358404846, "accuracy_std_dev": 0.034154544866046785, "run_count": 21, "total_questions": 3962, "avg_cost_per_task_usd": 0.37220477612317004, "avg_duration_per_task_ms": 491484.80312973243, "avg_output_tokens_per_task": 56849.085562847045, "last_run_timestamp": "2026-10-31T16:18:08.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.8965292307692307, "accuracy_std_dev": 0.011041742442998501, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.20059593162393163, "avg_duration_per_task_ms": 96185.55477855478, "avg_output_tokens_per_task": 7804.96891996892, "last_run_timestamp": "2026-10-31T08:32:06.814Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.8965206666666669, "accuracy_std_dev": 0.0064727425889047965, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.20788648787878788, "avg_duration_per_task_ms": 100622.61919191919, "avg_output_tokens_per_task": 8101.048148148148, "last_run_timestamp": "2026-10-31T08:27:47.283Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.89642245, "accuracy_std_dev": 0.008143539230426761, "run_count": 40, "total_questions": 7920, "avg_cost_per_task_usd": 0.20629075454545454, "avg_duration_per_task_ms": 70772.6904040404, "avg_output_tokens_per_task": 7507.606691919192, "last_run_timestamp": "2026-10-31T08:51:35.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Google", "accuracy": 0.896184, "accuracy_std_dev": 0.007570749896806769, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0237254696969697, "avg_duration_per_task_ms": 319850.3367003367, "avg_output_tokens_per_task": 19569.10606060606, "last_run_timestamp": "2026-10-31T08:20:35.720Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.896157644859813, "accuracy_std_dev": 0.009978064213845336, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.01666022313084112, "avg_duration_per_task_ms": 348918.9147196262, "avg_output_tokens_per_task": 28403.781542056076, "last_run_timestamp": "2026-10-31T09:16:30.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8961037142857144, "accuracy_std_dev": 0.004737329618927418, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.054530834776334774, "avg_duration_per_task_ms": 30476.945887445887, "avg_output_tokens_per_task": 2521.178932178932, "last_run_timestamp": "2026-10-31T08:40:20.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.895623, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03841230555555556, "avg_duration_per_task_ms": 144059.0782828283, "avg_output_tokens_per_task": 27039.489898989897, "last_run_timestamp": "2026-10-01T16:12:34.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.895412375, "accuracy_std_dev": 0.020051295631065218, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02414115151515151, "avg_duration_per_task_ms": 210274.94823232322, "avg_output_tokens_per_task": 27810.135101010103, "last_run_timestamp": "2026-10-31T08:27:49.932Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8952905937499999, "accuracy_std_dev": 0.016969367452854572, "run_count": 8, "total_questions": 1536, "avg_cost_per_task_usd": 0.032397834635416664, "avg_duration_per_task_ms": 236218.794921875, "avg_output_tokens_per_task": 32195.716796875, "last_run_timestamp": "2026-10-31T09:17:17.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.8952341538461538, "accuracy_std_dev": 0.0074752677939800664, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.12034864141414141, "avg_duration_per_task_ms": 58356.32323232323, "avg_output_tokens_per_task": 4592.7913752913755, "last_run_timestamp": "2026-10-31T08:31:11.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.8951904072144289, "accuracy_std_dev": 0.019820245214898276, "run_count": 13, "total_questions": 2495, "avg_cost_per_task_usd": 0.159686151503006, "avg_duration_per_task_ms": 368440.1851703407, "avg_output_tokens_per_task": 40060.66533066132, "last_run_timestamp": "2026-10-31T08:09:30.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.8951417142857143, "accuracy_std_dev": 0.009679824699036726, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008688588023088023, "avg_duration_per_task_ms": 171107.75108225108, "avg_output_tokens_per_task": 30552.396103896102, "last_run_timestamp": "2026-10-31T09:09:17.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8951046923076924, "accuracy_std_dev": 0.009277198784651573, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.23170269114219116, "avg_duration_per_task_ms": 110619.57381507382, "avg_output_tokens_per_task": 9043.770007770008, "last_run_timestamp": "2026-10-31T08:13:10.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "ModelRun", "accuracy": 0.8943087542213883, "accuracy_std_dev": 0.03942204850926279, "run_count": 3, "total_questions": 533, "avg_cost_per_task_usd": 0.019450917448405252, "avg_duration_per_task_ms": 177927.2420262664, "avg_output_tokens_per_task": 28174.943714821762, "last_run_timestamp": "2026-10-28T05:00:14.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.894276, "accuracy_std_dev": 0.008869991905295046, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.11497575656565656, "avg_duration_per_task_ms": 654340.1272727273, "avg_output_tokens_per_task": 45820.55656565657, "last_run_timestamp": "2026-10-31T08:45:35.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.8941497500000002, "accuracy_std_dev": 0.012326495474687834, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.010706986111111112, "avg_duration_per_task_ms": 265829.2607323232, "avg_output_tokens_per_task": 20748.16351010101, "last_run_timestamp": "2026-10-31T08:33:46.211Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "CoreWeave", "accuracy": 0.893939358490566, "accuracy_std_dev": 0.004245539018258487, "run_count": 3, "total_questions": 583, "avg_cost_per_task_usd": 0.023210332761578047, "avg_duration_per_task_ms": 116069.80960548884, "avg_output_tokens_per_task": 13628.193825042881, "last_run_timestamp": "2026-10-31T08:17:29.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Phala", "accuracy": 0.8939393333333333, "accuracy_std_dev": 0.012217692780917704, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02448665488215488, "avg_duration_per_task_ms": 139886.7744107744, "avg_output_tokens_per_task": 20249.15824915825, "last_run_timestamp": "2026-10-31T09:02:19.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8939391999999999, "accuracy_std_dev": 0.01075335131761256, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0050906424242424244, "avg_duration_per_task_ms": 221838.88434343433, "avg_output_tokens_per_task": 27900.379292929294, "last_run_timestamp": "2026-10-31T16:31:13.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.893627599657094, "accuracy_std_dev": 0.01880407170058742, "run_count": 12, "total_questions": 2333, "avg_cost_per_task_usd": 0.118091234462066, "avg_duration_per_task_ms": 525572.4204886412, "avg_output_tokens_per_task": 43317.68838405486, "last_run_timestamp": "2026-10-31T08:55:34.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.8934709999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 194, "avg_cost_per_task_usd": 0.11686864948453608, "avg_duration_per_task_ms": 605687.7474226804, "avg_output_tokens_per_task": 46496.701030927834, "last_run_timestamp": "2026-10-13T03:00:07.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "BaseTen", "accuracy": 0.8933780000000001, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03135734848484849, "avg_duration_per_task_ms": 31339.205387205388, "avg_output_tokens_per_task": 13928.784511784512, "last_run_timestamp": "2026-10-31T08:28:17.755Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "DigitalOcean", "accuracy": 0.8933779999999999, "accuracy_std_dev": 0.0034590473640392213, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01648433501683502, "avg_duration_per_task_ms": 93266.93602693603, "avg_output_tokens_per_task": 13659.838383838383, "last_run_timestamp": "2026-10-31T08:40:23.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "PrimeIntellect", "accuracy": 0.8931621743589744, "accuracy_std_dev": 0.0023230000000000195, "run_count": 2, "total_questions": 390, "avg_cost_per_task_usd": 0.020242015384615383, "avg_duration_per_task_ms": 141496.24102564104, "avg_output_tokens_per_task": 40323.36666666667, "last_run_timestamp": "2026-10-31T08:52:13.905Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8930975, "accuracy_std_dev": 0.010942499999999966, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04334855050505051, "avg_duration_per_task_ms": 40520.42929292929, "avg_output_tokens_per_task": 3151.881313131313, "last_run_timestamp": "2026-10-01T19:11:43.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "AtlasCloud", "accuracy": 0.8930975, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11303334343434343, "avg_duration_per_task_ms": 814053.6843434343, "avg_output_tokens_per_task": 45048.43686868687, "last_run_timestamp": "2026-10-31T08:27:45.248Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.89301948861912, "accuracy_std_dev": 0.023661754900260477, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.06508963581183612, "avg_duration_per_task_ms": 539075.6282245826, "avg_output_tokens_per_task": 36246.24228629236, "last_run_timestamp": "2026-10-31T16:05:14.632Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.8928929372738239, "accuracy_std_dev": 0.015585617718297187, "run_count": 17, "total_questions": 3316, "avg_cost_per_task_usd": 0.014895325693606755, "avg_duration_per_task_ms": 187175.30186972255, "avg_output_tokens_per_task": 21447.96381182147, "last_run_timestamp": "2026-10-31T08:57:07.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8928570861678005, "accuracy_std_dev": 0.015340394005370288, "run_count": 5, "total_questions": 882, "avg_cost_per_task_usd": 0.12683532766439912, "avg_duration_per_task_ms": 285532.1031746032, "avg_output_tokens_per_task": 28366.77097505669, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Novita", "accuracy": 0.892817, "accuracy_std_dev": 0.00915257617650169, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007166784511784513, "avg_duration_per_task_ms": 121588.36531986532, "avg_output_tokens_per_task": 11764.878787878788, "last_run_timestamp": "2026-10-31T08:24:44.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.892817, "accuracy_std_dev": 0.0067805750493597225, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11382627104377104, "avg_duration_per_task_ms": 836209.1372053872, "avg_output_tokens_per_task": 45359.85101010101, "last_run_timestamp": "2026-10-31T09:36:23.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.8923205, "accuracy_std_dev": 0.00915553451470747, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.24410277389277388, "avg_duration_per_task_ms": 85496.22163947164, "avg_output_tokens_per_task": 9073.098096348096, "last_run_timestamp": "2026-10-31T08:12:47.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.8922560000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0070467222222222224, "avg_duration_per_task_ms": 212650.5101010101, "avg_output_tokens_per_task": 24920.08080808081, "last_run_timestamp": "2026-08-08T10:06:09.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8922560000000002, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14220060606060608, "avg_duration_per_task_ms": 188559.2904040404, "avg_output_tokens_per_task": 47276.156565656565, "last_run_timestamp": "2026-10-01T19:12:30.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.8918068666666666, "accuracy_std_dev": 0.011242368859907686, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.20911976700336699, "avg_duration_per_task_ms": 100253.8824915825, "avg_output_tokens_per_task": 8136.723905723906, "last_run_timestamp": "2026-10-31T08:39:17.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.891800684137931, "accuracy_std_dev": 0.016199095521633556, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.14719310068965516, "avg_duration_per_task_ms": 655158.9301149425, "avg_output_tokens_per_task": 37074.59586206896, "last_run_timestamp": "2026-10-31T08:49:49.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8917748571428571, "accuracy_std_dev": 0.009721593908534165, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04607931240981241, "avg_duration_per_task_ms": 43786.40764790765, "avg_output_tokens_per_task": 3453.090909090909, "last_run_timestamp": "2026-10-01T19:13:06.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "OpenInference", "accuracy": 0.8916946666666665, "accuracy_std_dev": 0.009944030347678707, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.011122356902356903, "avg_duration_per_task_ms": 302424.50673400675, "avg_output_tokens_per_task": 25444.905723905726, "last_run_timestamp": "2026-10-31T09:01:06.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.8913204444444445, "accuracy_std_dev": 0.028427403794035355, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.10673178563411896, "avg_duration_per_task_ms": 440642.3956228956, "avg_output_tokens_per_task": 36504.574635241304, "last_run_timestamp": "2026-10-31T16:10:36.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.891312813400857, "accuracy_std_dev": 0.01013985402366827, "run_count": 13, "total_questions": 2567, "avg_cost_per_task_usd": 0.007902175301908844, "avg_duration_per_task_ms": 334153.24970783014, "avg_output_tokens_per_task": 27876.923646279705, "last_run_timestamp": "2026-10-31T08:16:11.302Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8910532857142858, "accuracy_std_dev": 0.008331061289180467, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02299151948051948, "avg_duration_per_task_ms": 246585.4624819625, "avg_output_tokens_per_task": 30738.57647907648, "last_run_timestamp": "2026-10-31T08:09:30.604Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.8907934361904761, "accuracy_std_dev": 0.020177999596946557, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.18863606476190478, "avg_duration_per_task_ms": 1541820.2831746032, "avg_output_tokens_per_task": 55365.57968253968, "last_run_timestamp": "2026-10-24T16:20:08.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8906571072375901, "accuracy_std_dev": 0.009177998565046731, "run_count": 40, "total_questions": 7917, "avg_cost_per_task_usd": 0.13327076518883413, "avg_duration_per_task_ms": 59327.010610079575, "avg_output_tokens_per_task": 4760.107869142352, "last_run_timestamp": "2026-10-31T08:46:09.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008230473063973063, "avg_duration_per_task_ms": 80128.80303030302, "avg_output_tokens_per_task": 12935.998316498317, "last_run_timestamp": "2026-10-02T23:23:43.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.8899710714285712, "accuracy_std_dev": 0.03484950356372031, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.11476333874458873, "avg_duration_per_task_ms": 499182.5613275613, "avg_output_tokens_per_task": 36700.38997113997, "last_run_timestamp": "2026-10-31T08:09:42.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.889876224111283, "accuracy_std_dev": 0.005998909101508626, "run_count": 7, "total_questions": 1294, "avg_cost_per_task_usd": 0.030587522411128287, "avg_duration_per_task_ms": 772887.7256568779, "avg_output_tokens_per_task": 36034.0115919629, "last_run_timestamp": "2026-10-31T08:09:29.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.8895201249999999, "accuracy_std_dev": 0.013120057749849085, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10712794823232324, "avg_duration_per_task_ms": 473198.4646464646, "avg_output_tokens_per_task": 37189.18434343435, "last_run_timestamp": "2026-10-31T08:45:16.520Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8894587960168979, "accuracy_std_dev": 0.02138212178317713, "run_count": 9, "total_questions": 1657, "avg_cost_per_task_usd": 0.15853517501508751, "avg_duration_per_task_ms": 684130.0814725407, "avg_output_tokens_per_task": 50418.83343391672, "last_run_timestamp": "2026-10-31T16:34:03.928Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.02853141346198763, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.136280962962963, "avg_duration_per_task_ms": 40331.19781144781, "avg_output_tokens_per_task": 2516.2390572390573, "last_run_timestamp": "2026-10-01T08:24:17.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.009896244159893304, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008765856902356902, "avg_duration_per_task_ms": 59664.589225589225, "avg_output_tokens_per_task": 12901.691919191919, "last_run_timestamp": "2026-10-01T08:29:53.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8891708512988952, "accuracy_std_dev": 0.008107170901595804, "run_count": 17, "total_questions": 3349, "avg_cost_per_task_usd": 0.015164630934607346, "avg_duration_per_task_ms": 200146.0895789788, "avg_output_tokens_per_task": 27111.550313526426, "last_run_timestamp": "2026-10-31T09:08:32.372Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.8891478461538462, "accuracy_std_dev": 0.008503682775454532, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.20683341996891996, "avg_duration_per_task_ms": 100040.29953379954, "avg_output_tokens_per_task": 8045.402097902098, "last_run_timestamp": "2026-10-31T09:23:04.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.8888890000000002, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0431288872053872, "avg_duration_per_task_ms": 52519.67676767677, "avg_output_tokens_per_task": 3463.8030303030305, "last_run_timestamp": "2026-10-02T23:21:06.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.888889, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1979060606060606, "avg_duration_per_task_ms": 95702.44696969698, "avg_output_tokens_per_task": 7691.454545454545, "last_run_timestamp": "2026-10-01T19:11:18.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8888027820710974, "accuracy_std_dev": 0.021558691528940226, "run_count": 10, "total_questions": 1941, "avg_cost_per_task_usd": 0.019635562596599692, "avg_duration_per_task_ms": 258062.31633178773, "avg_output_tokens_per_task": 30486.462132921173, "last_run_timestamp": "2026-10-31T08:42:01.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8887267478134111, "accuracy_std_dev": 0.05987639223737212, "run_count": 22, "total_questions": 4116, "avg_cost_per_task_usd": 0.008590749028182704, "avg_duration_per_task_ms": 154650.31656948494, "avg_output_tokens_per_task": 24587.18537414966, "last_run_timestamp": "2026-10-31T09:24:59.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.8884226074592075, "accuracy_std_dev": 0.027668835320627423, "run_count": 11, "total_questions": 2145, "avg_cost_per_task_usd": 0.008902693706293706, "avg_duration_per_task_ms": 299088.97715617716, "avg_output_tokens_per_task": 27126.232167832168, "last_run_timestamp": "2026-10-31T08:51:08.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Crusoe", "accuracy": 0.8881691813471503, "accuracy_std_dev": 0.0030120000000000147, "run_count": 2, "total_questions": 386, "avg_cost_per_task_usd": 0.1504010103626943, "avg_duration_per_task_ms": 651195.4274611399, "avg_output_tokens_per_task": 42839.41709844559, "last_run_timestamp": "2026-10-31T16:08:08.539Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8880469999999999, "accuracy_std_dev": 0.011198752698403515, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.025298165945165943, "avg_duration_per_task_ms": 331181.7139249639, "avg_output_tokens_per_task": 26439.20562770563, "last_run_timestamp": "2026-10-31T09:11:16.308Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8880229138638227, "accuracy_std_dev": 0.007263570665748784, "run_count": 7, "total_questions": 1219, "avg_cost_per_task_usd": 0.01426234372436423, "avg_duration_per_task_ms": 205415.2715340443, "avg_output_tokens_per_task": 24591.009844134536, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.8878064999999999, "accuracy_std_dev": 0.018774829060473484, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.028150706349206345, "avg_duration_per_task_ms": 209828.97294372294, "avg_output_tokens_per_task": 25892.321789321788, "last_run_timestamp": "2026-10-31T08:53:51.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.8876718896377422, "accuracy_std_dev": 0.0038308239938977078, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.032938989890480205, "avg_duration_per_task_ms": 750389.2257792755, "avg_output_tokens_per_task": 38026.47177759057, "last_run_timestamp": "2026-10-31T08:03:09.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.8875500348763478, "accuracy_std_dev": 0.013802209238356553, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.1294895707038681, "avg_duration_per_task_ms": 1720933.2999365884, "avg_output_tokens_per_task": 32195.95688015219, "last_run_timestamp": "2026-10-31T16:35:08.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.887537924012158, "accuracy_std_dev": 0.020250680116973837, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.12033997061803446, "avg_duration_per_task_ms": 580755.4113475177, "avg_output_tokens_per_task": 34357.5663627153, "last_run_timestamp": "2026-10-03T16:44:56.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8872051666666666, "accuracy_std_dev": 0.024761493125527695, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1434989882154882, "avg_duration_per_task_ms": 42121.643939393936, "avg_output_tokens_per_task": 2660.465488215488, "last_run_timestamp": "2026-10-01T08:24:34.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.8871015424486148, "accuracy_std_dev": 0.007433711649565692, "run_count": 6, "total_questions": 1119, "avg_cost_per_task_usd": 0.100128890974084, "avg_duration_per_task_ms": 379563.4110813226, "avg_output_tokens_per_task": 36832.25379803396, "last_run_timestamp": "2026-10-31T16:50:08.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.887042338150289, "accuracy_std_dev": 0.010371066166434932, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.010374203757225434, "avg_duration_per_task_ms": 78284.89884393064, "avg_output_tokens_per_task": 7947.398121387283, "last_run_timestamp": "2026-10-01T19:12:42.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.8865318000000001, "accuracy_std_dev": 0.007110736653821441, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.030907244444444443, "avg_duration_per_task_ms": 199105.52626262626, "avg_output_tokens_per_task": 25534.58484848485, "last_run_timestamp": "2026-10-31T08:28:17.984Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.8864282307692305, "accuracy_std_dev": 0.014063708976564978, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.11727622494172495, "avg_duration_per_task_ms": 585593.1717171717, "avg_output_tokens_per_task": 30597.257575757576, "last_run_timestamp": "2026-10-31T16:23:41.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.8863635000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.22157512626262627, "avg_duration_per_task_ms": 106335.13383838384, "avg_output_tokens_per_task": 8638.217171717171, "last_run_timestamp": "2026-10-01T19:10:42.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "BaseTen", "accuracy": 0.886083, "accuracy_std_dev": 0.012320026406898109, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.022550329966329965, "avg_duration_per_task_ms": 149365.32154882155, "avg_output_tokens_per_task": 44803.96127946128, "last_run_timestamp": "2026-10-31T08:13:10.777Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.8856841481205807, "accuracy_std_dev": 0.008667673595090004, "run_count": 14, "total_questions": 2687, "avg_cost_per_task_usd": 0.09660396911053219, "avg_duration_per_task_ms": 451439.94491998514, "avg_output_tokens_per_task": 28785.028656494233, "last_run_timestamp": "2026-10-31T16:12:28.231Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.8856471629778674, "accuracy_std_dev": 0.009013068317726206, "run_count": 8, "total_questions": 1491, "avg_cost_per_task_usd": 0.010023977867203217, "avg_duration_per_task_ms": 270739.96177062375, "avg_output_tokens_per_task": 28220.264922870556, "last_run_timestamp": "2026-10-24T09:01:43.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-max", "model_slug": "sakana/fugu-max", "model_name": "Fugu Max", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07555469191919191, "avg_duration_per_task_ms": 225091.94949494948, "avg_output_tokens_per_task": 12279.550505050505, "last_run_timestamp": "2026-09-15T01:07:47.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02146120202020202, "avg_duration_per_task_ms": 674351.9393939395, "avg_output_tokens_per_task": 45464.17676767677, "last_run_timestamp": "2026-10-01T08:31:57.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Together", "accuracy": 0.8855216666666669, "accuracy_std_dev": 0.0023805928299947096, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.016851747474747474, "avg_duration_per_task_ms": 62951.132996632994, "avg_output_tokens_per_task": 13874.079124579124, "last_run_timestamp": "2026-10-31T08:03:10.178Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.8850689656218401, "accuracy_std_dev": 0.014411932672615443, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.026149528816986855, "avg_duration_per_task_ms": 355797.47320525785, "avg_output_tokens_per_task": 26995.41152679474, "last_run_timestamp": "2026-10-10T08:22:45.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.884890375, "accuracy_std_dev": 0.021787089743340537, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.22976492803030302, "avg_duration_per_task_ms": 949653.6900252525, "avg_output_tokens_per_task": 57582.06123737374, "last_run_timestamp": "2026-10-24T16:07:10.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.88468, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1319599772727273, "avg_duration_per_task_ms": 249775.19949494948, "avg_output_tokens_per_task": 43863.606060606064, "last_run_timestamp": "2026-10-01T19:15:31.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.8846799999999999, "accuracy_std_dev": 0.021044000000000007, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.08896683838383838, "avg_duration_per_task_ms": 416558.4267676768, "avg_output_tokens_per_task": 22204.20202020202, "last_run_timestamp": "2026-10-31T08:57:00.457Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8843169242648614, "accuracy_std_dev": 0.01377803394592024, "run_count": 24, "total_questions": 4727, "avg_cost_per_task_usd": 0.19024729342077426, "avg_duration_per_task_ms": 371375.61032367253, "avg_output_tokens_per_task": 31453.42183202877, "last_run_timestamp": "2026-10-24T16:13:46.475Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.8842123333333334, "accuracy_std_dev": 0.018675407929502712, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.008158511784511784, "avg_duration_per_task_ms": 397047.721661055, "avg_output_tokens_per_task": 28750.622895622895, "last_run_timestamp": "2026-10-31T16:48:29.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.8840456666666667, "accuracy_std_dev": 0.005144110505120299, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.02067068717948718, "avg_duration_per_task_ms": 91684.32991452991, "avg_output_tokens_per_task": 41136.99487179487, "last_run_timestamp": "2026-10-31T08:35:13.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.8840252222222222, "accuracy_std_dev": 0.02805224637777707, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.19950497250280583, "avg_duration_per_task_ms": 1012777.083613917, "avg_output_tokens_per_task": 58973.21717171717, "last_run_timestamp": "2026-10-31T16:24:42.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.8839239383667181, "accuracy_std_dev": 0.011423232412500406, "run_count": 8, "total_questions": 1298, "avg_cost_per_task_usd": 0.00878981278890601, "avg_duration_per_task_ms": 844366.6594761171, "avg_output_tokens_per_task": 29702.687981510015, "last_run_timestamp": "2026-10-31T08:34:49.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04134767424242424, "avg_duration_per_task_ms": 54936.305555555555, "avg_output_tokens_per_task": 3316.282828282828, "last_run_timestamp": "2026-10-01T19:13:07.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008301368686868687, "avg_duration_per_task_ms": 72985.27777777778, "avg_output_tokens_per_task": 13098.969696969696, "last_run_timestamp": "2026-10-01T08:24:38.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014708931818181817, "avg_duration_per_task_ms": 71351.09343434343, "avg_output_tokens_per_task": 13550.636363636364, "last_run_timestamp": "2026-10-31T08:27:52.629Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "StreamLake", "accuracy": 0.8836312915601023, "accuracy_std_dev": 0.003620000000000012, "run_count": 2, "total_questions": 391, "avg_cost_per_task_usd": 0.006277416879795397, "avg_duration_per_task_ms": 104811.05115089515, "avg_output_tokens_per_task": 10905.734015345268, "last_run_timestamp": "2026-10-31T08:31:24.328Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.8832547529880477, "accuracy_std_dev": 0.048459311911738724, "run_count": 14, "total_questions": 2761, "avg_cost_per_task_usd": 0.07450339333574792, "avg_duration_per_task_ms": 577739.1611734879, "avg_output_tokens_per_task": 39363.77290836653, "last_run_timestamp": "2026-10-31T16:51:33.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8831221012658229, "accuracy_std_dev": 0.01342904913771097, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.1860454797468354, "avg_duration_per_task_ms": 320097.15611814347, "avg_output_tokens_per_task": 46697.94894514768, "last_run_timestamp": "2026-10-31T08:25:00.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Minimax", "accuracy": 0.8829965, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006819977272727273, "avg_duration_per_task_ms": 114306.47727272728, "avg_output_tokens_per_task": 11182.35606060606, "last_run_timestamp": "2026-10-31T08:07:19.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.8829290000000001, "accuracy_std_dev": 0.012699110785799135, "run_count": 4, "total_questions": 783, "avg_cost_per_task_usd": 0.027103435504469985, "avg_duration_per_task_ms": 200899.3601532567, "avg_output_tokens_per_task": 30539.469987228607, "last_run_timestamp": "2026-10-31T08:15:03.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8827922637511884, "accuracy_std_dev": 0.009496956336599638, "run_count": 38, "total_questions": 7363, "avg_cost_per_task_usd": 0.1404521245416271, "avg_duration_per_task_ms": 63191.223414369146, "avg_output_tokens_per_task": 5059.606817873149, "last_run_timestamp": "2026-10-31T08:31:11.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.8825948758169935, "accuracy_std_dev": 0.010166708210464918, "run_count": 14, "total_questions": 2754, "avg_cost_per_task_usd": 0.007344207697893974, "avg_duration_per_task_ms": 375236.5925925926, "avg_output_tokens_per_task": 20279.856209150326, "last_run_timestamp": "2026-10-31T08:07:18.580Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.882365125, "accuracy_std_dev": 0.0043279540904883765, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.029606650252525252, "avg_duration_per_task_ms": 78746.81123737374, "avg_output_tokens_per_task": 24519.371843434343, "last_run_timestamp": "2026-10-31T09:02:59.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.8821789256756756, "accuracy_std_dev": 0.01706691482241463, "run_count": 6, "total_questions": 1184, "avg_cost_per_task_usd": 0.04157157263513513, "avg_duration_per_task_ms": 164106.9873310811, "avg_output_tokens_per_task": 34476.61317567567, "last_run_timestamp": "2026-10-31T08:31:11.691Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "ModelRun", "accuracy": 0.8821550000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.026309151515151515, "avg_duration_per_task_ms": 202247.18686868687, "avg_output_tokens_per_task": 39119.5505050505, "last_run_timestamp": "2026-10-29T03:00:03.996Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.8821550000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.129083601010101, "avg_duration_per_task_ms": 388751.6590909091, "avg_output_tokens_per_task": 32112.176767676767, "last_run_timestamp": "2026-10-17T16:07:32.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Reka", "accuracy": 0.881818, "accuracy_std_dev": 0.03925680632450885, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010723755555555554, "avg_duration_per_task_ms": 145330.13333333333, "avg_output_tokens_per_task": 16217.381818181819, "last_run_timestamp": "2026-10-31T09:01:13.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.8817657344384302, "accuracy_std_dev": 0.02198561693228856, "run_count": 15, "total_questions": 2956, "avg_cost_per_task_usd": 0.09460041407307172, "avg_duration_per_task_ms": 453932.54093369417, "avg_output_tokens_per_task": 33735.36434370771, "last_run_timestamp": "2026-10-31T16:08:08.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "GMICloud", "accuracy": 0.8817339999999999, "accuracy_std_dev": 0.016681744617994862, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008588069444444443, "avg_duration_per_task_ms": 150888.15656565657, "avg_output_tokens_per_task": 16172.003787878788, "last_run_timestamp": "2026-10-31T08:07:19.698Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.881655907348243, "accuracy_std_dev": 0.03261050414332772, "run_count": 19, "total_questions": 3756, "avg_cost_per_task_usd": 0.14437988578274757, "avg_duration_per_task_ms": 367673.1634717785, "avg_output_tokens_per_task": 10849.925452609159, "last_run_timestamp": "2026-10-31T09:14:35.378Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.8815936666666665, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002981575757575758, "avg_duration_per_task_ms": 52993.44276094276, "avg_output_tokens_per_task": 5804.632996632997, "last_run_timestamp": "2026-10-02T23:20:31.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.8814814000000001, "accuracy_std_dev": 0.011789297597397391, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007892147474747474, "avg_duration_per_task_ms": 236163.43737373737, "avg_output_tokens_per_task": 27867.079797979797, "last_run_timestamp": "2026-10-03T16:57:09.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.8814812, "accuracy_std_dev": 0.011112816535874249, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.006134084511784513, "avg_duration_per_task_ms": 402598.8686868687, "avg_output_tokens_per_task": 29715.115824915825, "last_run_timestamp": "2026-10-31T08:19:56.454Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8811927142857143, "accuracy_std_dev": 0.007888829783113527, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015891973304473304, "avg_duration_per_task_ms": 279407.10173160175, "avg_output_tokens_per_task": 24276.554834054834, "last_run_timestamp": "2026-10-08T22:18:51.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.8810325, "accuracy_std_dev": 0.016020429392393534, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03953295538720539, "avg_duration_per_task_ms": 203574.52356902356, "avg_output_tokens_per_task": 32800.2398989899, "last_run_timestamp": "2026-10-31T08:47:44.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8809893076923078, "accuracy_std_dev": 0.018410802353236562, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.12893885664335664, "avg_duration_per_task_ms": 63425.74864024864, "avg_output_tokens_per_task": 4924.540792540793, "last_run_timestamp": "2026-10-31T08:31:10.506Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.8808676325443787, "accuracy_std_dev": 0.011642004324486309, "run_count": 9, "total_questions": 1690, "avg_cost_per_task_usd": 0.06728608106508875, "avg_duration_per_task_ms": 317431.9201183432, "avg_output_tokens_per_task": 18592.419526627218, "last_run_timestamp": "2026-10-31T16:03:21.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.880471, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0032475000000000004, "avg_duration_per_task_ms": 58949.292929292926, "avg_output_tokens_per_task": 5746.025252525253, "last_run_timestamp": "2026-09-23T06:09:34.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8801906666666667, "accuracy_std_dev": 0.011226983784119793, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0227881835016835, "avg_duration_per_task_ms": 234700.68434343435, "avg_output_tokens_per_task": 25264.190235690236, "last_run_timestamp": "2026-10-03T09:30:18.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.880186085003036, "accuracy_std_dev": 0.03105021938412785, "run_count": 9, "total_questions": 1647, "avg_cost_per_task_usd": 0.204991802671524, "avg_duration_per_task_ms": 1379722.3017607771, "avg_output_tokens_per_task": 51102.45476624165, "last_run_timestamp": "2026-10-31T16:04:29.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.8800631670886074, "accuracy_std_dev": 0.006743341790240229, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.033778830379746834, "avg_duration_per_task_ms": 152045.89430379748, "avg_output_tokens_per_task": 27961.480379746834, "last_run_timestamp": "2026-10-31T09:42:46.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.8800035555555555, "accuracy_std_dev": 0.060993454403113545, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.13239759820426486, "avg_duration_per_task_ms": 1665151.1414141415, "avg_output_tokens_per_task": 55809.35241301908, "last_run_timestamp": "2026-10-31T16:13:09.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.8799533076923076, "accuracy_std_dev": 0.01785792442666288, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.1271698951048951, "avg_duration_per_task_ms": 62325.81235431235, "avg_output_tokens_per_task": 4863.158508158508, "last_run_timestamp": "2026-10-31T08:12:45.681Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.879872690530519, "accuracy_std_dev": 0.053065810239372686, "run_count": 18, "total_questions": 3506, "avg_cost_per_task_usd": 0.00976165202509983, "avg_duration_per_task_ms": 414485.4175698802, "avg_output_tokens_per_task": 26831.222760981174, "last_run_timestamp": "2026-10-31T08:47:06.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8798644568527919, "accuracy_std_dev": 0.004527248931623837, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.0029938477157360403, "avg_duration_per_task_ms": 316818.90016920475, "avg_output_tokens_per_task": 18321.88663282572, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.87984, "accuracy_std_dev": 0.018052164925570553, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.036141941919191914, "avg_duration_per_task_ms": 139656.41224747474, "avg_output_tokens_per_task": 28081.721590909092, "last_run_timestamp": "2026-10-31T09:14:35.460Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.879749096574958, "accuracy_std_dev": 0.006147652748894318, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.017198152723189218, "avg_duration_per_task_ms": 289110.672094329, "avg_output_tokens_per_task": 31088.430656934306, "last_run_timestamp": "2026-10-31T08:46:18.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto", "model_slug": "openrouter/auto", "model_name": "Auto Router", "provider_name": "auto-routing", "accuracy": 0.8795957999999999, "accuracy_std_dev": 0.026351833738091142, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08917041111111111, "avg_duration_per_task_ms": 196788.44343434344, "avg_output_tokens_per_task": 37445.42121212121, "last_run_timestamp": "2026-09-29T14:28:17.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.879564846153846, "accuracy_std_dev": 0.03143742775944268, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.012390225330225331, "avg_duration_per_task_ms": 350248.7618492618, "avg_output_tokens_per_task": 22401.122377622378, "last_run_timestamp": "2026-10-31T08:24:31.558Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "OpenInference", "accuracy": 0.879386, "accuracy_std_dev": null, "run_count": 1, "total_questions": 76, "avg_cost_per_task_usd": 0.03120878947368421, "avg_duration_per_task_ms": 274583.63157894736, "avg_output_tokens_per_task": 9741.526315789473, "last_run_timestamp": "2026-10-09T00:00:06.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.8793102638453499, "accuracy_std_dev": 0.020773557275777312, "run_count": 10, "total_questions": 1914, "avg_cost_per_task_usd": 0.09333590334378267, "avg_duration_per_task_ms": 520471.578892372, "avg_output_tokens_per_task": 28040.05590386625, "last_run_timestamp": "2026-10-31T08:31:12.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.8791213851308232, "accuracy_std_dev": 0.03905943528044792, "run_count": 20, "total_questions": 3134, "avg_cost_per_task_usd": 0.02867042214422464, "avg_duration_per_task_ms": 357853.9288449266, "avg_output_tokens_per_task": 25473.93746011487, "last_run_timestamp": "2026-10-31T08:10:35.431Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8787880000000001, "accuracy_std_dev": 0.007273551310513108, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.009556249158249158, "avg_duration_per_task_ms": 52070.99494949495, "avg_output_tokens_per_task": 7331.292929292929, "last_run_timestamp": "2026-10-30T00:00:06.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "Deepseek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.010643959595959596, "avg_duration_per_task_ms": 127627.76767676767, "avg_output_tokens_per_task": 14277.949494949495, "last_run_timestamp": "2026-10-08T13:29:57.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Mimo V2.6 Pro", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01249009595959596, "avg_duration_per_task_ms": 315799.5808080808, "avg_output_tokens_per_task": 14226.873737373737, "last_run_timestamp": "2026-10-08T01:55:44.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.043995565656565656, "avg_duration_per_task_ms": 588519.7272727273, "avg_output_tokens_per_task": 34166.82828282828, "last_run_timestamp": "2026-10-01T08:47:14.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.8786261545293074, "accuracy_std_dev": 0.0068412882173039926, "run_count": 6, "total_questions": 1126, "avg_cost_per_task_usd": 0.11316016607460035, "avg_duration_per_task_ms": 476858.84191829484, "avg_output_tokens_per_task": 32318.380994671403, "last_run_timestamp": "2026-10-17T16:07:16.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.8782263333333332, "accuracy_std_dev": 0.004199386807089298, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020176612794612796, "avg_duration_per_task_ms": 120195.73063973064, "avg_output_tokens_per_task": 42971.59090909091, "last_run_timestamp": "2026-10-31T08:09:30.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.8782088436213992, "accuracy_std_dev": 0.01624543331919451, "run_count": 9, "total_questions": 1701, "avg_cost_per_task_usd": 0.18331693709582594, "avg_duration_per_task_ms": 766864.758377425, "avg_output_tokens_per_task": 52771.66372721928, "last_run_timestamp": "2026-10-31T16:35:50.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.8781144000000001, "accuracy_std_dev": 0.016133498487308946, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.023055602525252527, "avg_duration_per_task_ms": 719094.7823232323, "avg_output_tokens_per_task": 35705.98131313131, "last_run_timestamp": "2026-10-31T16:46:00.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8778525555555555, "accuracy_std_dev": 0.02507239829024519, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.13925247306397306, "avg_duration_per_task_ms": 898484.416386083, "avg_output_tokens_per_task": 61943.26487093154, "last_run_timestamp": "2026-10-31T16:04:29.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8777055714285714, "accuracy_std_dev": 0.04371368642773152, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.09160316883116884, "avg_duration_per_task_ms": 609338.455988456, "avg_output_tokens_per_task": 40699.61507936508, "last_run_timestamp": "2026-10-31T16:16:10.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8774956609137057, "accuracy_std_dev": 0.014904254367126186, "run_count": 15, "total_questions": 2955, "avg_cost_per_task_usd": 0.10885685989847717, "avg_duration_per_task_ms": 463191.3489001692, "avg_output_tokens_per_task": 31188.99323181049, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.8774795484247374, "accuracy_std_dev": 0.0070708188238871586, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.11807617502917153, "avg_duration_per_task_ms": 759991.2372617659, "avg_output_tokens_per_task": 30946.45468689226, "last_run_timestamp": "2026-10-31T16:19:28.502Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.8774407999999999, "accuracy_std_dev": 0.00954717737134911, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03921924343434343, "avg_duration_per_task_ms": 242470.04242424242, "avg_output_tokens_per_task": 37354.009090909094, "last_run_timestamp": "2026-10-21T11:23:15.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.8772070529695024, "accuracy_std_dev": 0.03893659390197737, "run_count": 4, "total_questions": 623, "avg_cost_per_task_usd": 0.09425666452648475, "avg_duration_per_task_ms": 461375.2568218299, "avg_output_tokens_per_task": 23922.329052969504, "last_run_timestamp": "2026-09-30T11:37:23.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.8771043333333335, "accuracy_std_dev": 0.007896528133018712, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003143374579124579, "avg_duration_per_task_ms": 35450.33585858586, "avg_output_tokens_per_task": 5733.669191919192, "last_run_timestamp": "2026-10-01T08:25:15.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.877104, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07509321212121212, "avg_duration_per_task_ms": 263175.32828282827, "avg_output_tokens_per_task": 20738.459595959597, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.876893965909091, "accuracy_std_dev": 0.02910034944176192, "run_count": 7, "total_questions": 1232, "avg_cost_per_task_usd": 0.09717019561688313, "avg_duration_per_task_ms": 193376.24837662338, "avg_output_tokens_per_task": 26307.616071428572, "last_run_timestamp": "2026-09-26T09:19:13.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8765966933797908, "accuracy_std_dev": 0.020189124617118887, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.004125730836236933, "avg_duration_per_task_ms": 453615.28397212544, "avg_output_tokens_per_task": 16054.564459930314, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8765431666666668, "accuracy_std_dev": 0.03217293038884639, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11818556060606061, "avg_duration_per_task_ms": 28260.9031986532, "avg_output_tokens_per_task": 2161.995791245791, "last_run_timestamp": "2026-10-01T08:25:56.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.8765007667238421, "accuracy_std_dev": 0.009087011145341243, "run_count": 3, "total_questions": 583, "avg_cost_per_task_usd": 0.09344886277873071, "avg_duration_per_task_ms": 568169.3550600343, "avg_output_tokens_per_task": 27396.61578044597, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.8763216813793104, "accuracy_std_dev": 0.01096372812565248, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.0099918183908046, "avg_duration_per_task_ms": 425649.0579310345, "avg_output_tokens_per_task": 22880.83264367816, "last_run_timestamp": "2026-10-31T08:32:56.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8762625, "accuracy_std_dev": 0.009259500000000032, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014013469696969696, "avg_duration_per_task_ms": 171317.10353535353, "avg_output_tokens_per_task": 26009.79292929293, "last_run_timestamp": "2026-10-31T08:09:25.873Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8762625, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009382047979797978, "avg_duration_per_task_ms": 83118.84595959596, "avg_output_tokens_per_task": 7689.015151515152, "last_run_timestamp": "2026-10-01T19:14:22.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.8762115941845765, "accuracy_std_dev": 0.007231120010715849, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.12366235651074589, "avg_duration_per_task_ms": 903569.9494310999, "avg_output_tokens_per_task": 43400.72819216182, "last_run_timestamp": "2026-10-24T16:19:29.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.8759833172804533, "accuracy_std_dev": 0.02383166285579381, "run_count": 6, "total_questions": 1059, "avg_cost_per_task_usd": 0.006660402266288952, "avg_duration_per_task_ms": 451840.41265344666, "avg_output_tokens_per_task": 36610.579792256845, "last_run_timestamp": "2026-10-31T08:15:03.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Novita", "accuracy": 0.8759819999999999, "accuracy_std_dev": 0.04446362050785635, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.019776964646464647, "avg_duration_per_task_ms": 142117.47643097644, "avg_output_tokens_per_task": 20178.936026936026, "last_run_timestamp": "2026-10-31T09:01:06.393Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.875981888888889, "accuracy_std_dev": 0.0056117777821777124, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.08140571212121211, "avg_duration_per_task_ms": 407227.8625140292, "avg_output_tokens_per_task": 23146.814814814814, "last_run_timestamp": "2026-10-31T16:15:54.664Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.8756311250000001, "accuracy_std_dev": 0.012722348362600958, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03608339141414141, "avg_duration_per_task_ms": 154494.60795454544, "avg_output_tokens_per_task": 29914.79419191919, "last_run_timestamp": "2026-10-31T08:55:44.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.8755709520547945, "accuracy_std_dev": 0.007533567739715949, "run_count": 3, "total_questions": 584, "avg_cost_per_task_usd": 0.031521625, "avg_duration_per_task_ms": 247947.07705479453, "avg_output_tokens_per_task": 62802.890410958906, "last_run_timestamp": "2026-10-31T08:14:43.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Nebius", "accuracy": 0.875421, "accuracy_std_dev": 0.010101000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.05678013888888889, "avg_duration_per_task_ms": 453449.72222222225, "avg_output_tokens_per_task": 62821.7904040404, "last_run_timestamp": "2026-10-31T08:23:57.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Together", "accuracy": 0.8754206666666668, "accuracy_std_dev": 0.004955863351178644, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01787970538720539, "avg_duration_per_task_ms": 181946.53367003368, "avg_output_tokens_per_task": 29504.771043771045, "last_run_timestamp": "2026-10-31T08:09:31.246Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.8754205, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04156198232323233, "avg_duration_per_task_ms": 309570.8888888889, "avg_output_tokens_per_task": 39159.9494949495, "last_run_timestamp": "2026-10-17T08:26:18.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.8753238600311043, "accuracy_std_dev": 0.02970275849959406, "run_count": 13, "total_questions": 2572, "avg_cost_per_task_usd": 0.17156172589424573, "avg_duration_per_task_ms": 364836.6096423017, "avg_output_tokens_per_task": 39108.15202177294, "last_run_timestamp": "2026-10-31T16:32:36.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8753084, "accuracy_std_dev": 0.016963206932652818, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.008280871043771043, "avg_duration_per_task_ms": 217247.1521885522, "avg_output_tokens_per_task": 29293.272727272728, "last_run_timestamp": "2026-10-31T08:31:11.360Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8751679521169355, "accuracy_std_dev": 0.029185533960542576, "run_count": 11, "total_questions": 1984, "avg_cost_per_task_usd": 0.027124963205645166, "avg_duration_per_task_ms": 364845.5393145161, "avg_output_tokens_per_task": 25176.949596774193, "last_run_timestamp": "2026-10-31T16:15:26.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8749014196217493, "accuracy_std_dev": 0.02259480542425625, "run_count": 5, "total_questions": 846, "avg_cost_per_task_usd": 0.12702199408983453, "avg_duration_per_task_ms": 548158.9716312056, "avg_output_tokens_per_task": 48997.07092198582, "last_run_timestamp": "2026-10-03T17:01:24.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.8746724444444444, "accuracy_std_dev": 0.019889553484240645, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.20093631930415262, "avg_duration_per_task_ms": 573135.2710437711, "avg_output_tokens_per_task": 58795.35746352413, "last_run_timestamp": "2026-10-31T16:09:16.278Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.8746087667870037, "accuracy_std_dev": 0.007592482877484693, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.2179415407942238, "avg_duration_per_task_ms": 1239989.4288808664, "avg_output_tokens_per_task": 48485.83321299639, "last_run_timestamp": "2026-10-24T16:18:35.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.874560281359906, "accuracy_std_dev": 0.03491023586403909, "run_count": 13, "total_questions": 2559, "avg_cost_per_task_usd": 0.10398778819851505, "avg_duration_per_task_ms": 633649.7999218445, "avg_output_tokens_per_task": 25844.287612348573, "last_run_timestamp": "2026-10-31T16:08:08.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.8745317312013828, "accuracy_std_dev": 0.011186512454241005, "run_count": 6, "total_questions": 1157, "avg_cost_per_task_usd": 0.0043274416594641305, "avg_duration_per_task_ms": 185028.52031114954, "avg_output_tokens_per_task": 21870.51080380294, "last_run_timestamp": "2026-09-26T08:53:59.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.8744105999999999, "accuracy_std_dev": 0.014149500063253112, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.09050255303030302, "avg_duration_per_task_ms": 196310.64444444445, "avg_output_tokens_per_task": 14775.43686868687, "last_run_timestamp": "2026-10-31T08:07:27.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.8743250351006383, "accuracy_std_dev": 0.026296453097608783, "run_count": 21, "total_questions": 4074, "avg_cost_per_task_usd": 0.03975532817869414, "avg_duration_per_task_ms": 80921.18360333824, "avg_output_tokens_per_task": 25880.549828178693, "last_run_timestamp": "2026-10-31T09:03:20.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "SambaNova", "accuracy": 0.8742983333333334, "accuracy_std_dev": 0.0055553002519115975, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02925406565656566, "avg_duration_per_task_ms": 153006.37373737374, "avg_output_tokens_per_task": 28825.781144781144, "last_run_timestamp": "2026-10-31T08:50:42.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8740494840344654, "accuracy_std_dev": 0.0038017229265163467, "run_count": 10, "total_questions": 1973, "avg_cost_per_task_usd": 0.04434169640141917, "avg_duration_per_task_ms": 202197.7790167258, "avg_output_tokens_per_task": 38209.482513938165, "last_run_timestamp": "2026-10-31T08:10:37.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.8740044607508533, "accuracy_std_dev": 0.007605951280996289, "run_count": 3, "total_questions": 586, "avg_cost_per_task_usd": 0.10401818771331058, "avg_duration_per_task_ms": 459122.319112628, "avg_output_tokens_per_task": 30022.856655290103, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.8737693295358652, "accuracy_std_dev": 0.014113612302391693, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.09247935569620254, "avg_duration_per_task_ms": 203634.65358649788, "avg_output_tokens_per_task": 23372.77805907173, "last_run_timestamp": "2026-10-31T08:40:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Nebius", "accuracy": 0.873737, "accuracy_std_dev": 0.007273551310513108, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01830383333333333, "avg_duration_per_task_ms": 381305.08585858584, "avg_output_tokens_per_task": 30305.045454545456, "last_run_timestamp": "2026-10-31T08:27:46.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0029526111111111116, "avg_duration_per_task_ms": 49722.71717171717, "avg_output_tokens_per_task": 5746.707070707071, "last_run_timestamp": "2026-10-01T08:23:37.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.045554454545454544, "avg_duration_per_task_ms": 605480.6616161616, "avg_output_tokens_per_task": 35385.22727272727, "last_run_timestamp": "2026-10-01T08:40:06.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.873677142857143, "accuracy_std_dev": 0.013379294172377723, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.00609786507936508, "avg_duration_per_task_ms": 261742.29906204907, "avg_output_tokens_per_task": 18465.902236652237, "last_run_timestamp": "2026-10-31T08:36:26.194Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.8728018888888889, "accuracy_std_dev": 0.022448311515045904, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.11960105274971941, "avg_duration_per_task_ms": 614007.7817059484, "avg_output_tokens_per_task": 35342.742424242424, "last_run_timestamp": "2026-10-31T16:25:53.821Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8726149999999999, "accuracy_std_dev": 0.0020998715833752102, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017651676767676767, "avg_duration_per_task_ms": 95564.90235690236, "avg_output_tokens_per_task": 29293.20202020202, "last_run_timestamp": "2026-10-31T08:33:05.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.8722639999999998, "accuracy_std_dev": 0.01391911622553672, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01943295707070707, "avg_duration_per_task_ms": 314686.08207070705, "avg_output_tokens_per_task": 35123.857954545456, "last_run_timestamp": "2026-10-31T08:09:31.159Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8721539397539244, "accuracy_std_dev": 0.008970575607073516, "run_count": 12, "total_questions": 2357, "avg_cost_per_task_usd": 0.008344311412812899, "avg_duration_per_task_ms": 339433.18073822657, "avg_output_tokens_per_task": 29494.21552821383, "last_run_timestamp": "2026-10-17T08:23:51.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.8720325587863464, "accuracy_std_dev": 0.09552669457201043, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.026620932996207335, "avg_duration_per_task_ms": 196837.85040033713, "avg_output_tokens_per_task": 22844.949431099874, "last_run_timestamp": "2026-10-31T16:04:29.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.8720047896405919, "accuracy_std_dev": 0.013430501280667074, "run_count": 10, "total_questions": 1892, "avg_cost_per_task_usd": 0.00839126691331924, "avg_duration_per_task_ms": 234781.10200845666, "avg_output_tokens_per_task": 29482.704545454544, "last_run_timestamp": "2026-10-28T03:34:12.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.8718665555555556, "accuracy_std_dev": 0.016936558486245815, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03607248540965208, "avg_duration_per_task_ms": 502797.54545454547, "avg_output_tokens_per_task": 35101.41638608305, "last_run_timestamp": "2026-10-31T16:06:00.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.8714925000000001, "accuracy_std_dev": 0.007251768353305299, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.007901208754208754, "avg_duration_per_task_ms": 584978.2028619528, "avg_output_tokens_per_task": 44690.53282828283, "last_run_timestamp": "2026-10-31T16:48:05.771Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.8714859264426126, "accuracy_std_dev": 0.00752103815972237, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.06945589727330374, "avg_duration_per_task_ms": 276858.71528218134, "avg_output_tokens_per_task": 11308.629676601142, "last_run_timestamp": "2026-10-10T08:22:14.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8712119166666668, "accuracy_std_dev": 0.038220154043075064, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1817093308080808, "avg_duration_per_task_ms": 45516.91035353536, "avg_output_tokens_per_task": 3430.526515151515, "last_run_timestamp": "2026-10-31T08:20:29.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.8711184444444444, "accuracy_std_dev": 0.016592189722953073, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.04206737822671156, "avg_duration_per_task_ms": 668364.7727272727, "avg_output_tokens_per_task": 35229.93041526375, "last_run_timestamp": "2026-10-31T16:16:55.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.8710116425661915, "accuracy_std_dev": 0.01025225770257459, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.025059682281059068, "avg_duration_per_task_ms": 328330.6405295316, "avg_output_tokens_per_task": 20642.19348268839, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8708191333333333, "accuracy_std_dev": 0.005011630059061523, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.004255378451178452, "avg_duration_per_task_ms": 39486.23198653199, "avg_output_tokens_per_task": 7745.265656565656, "last_run_timestamp": "2026-10-31T08:49:45.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.8708100545730167, "accuracy_std_dev": 0.007904713862626544, "run_count": 10, "total_questions": 1979, "avg_cost_per_task_usd": 0.049469633148054575, "avg_duration_per_task_ms": 293502.65942395147, "avg_output_tokens_per_task": 21011.941384537644, "last_run_timestamp": "2026-10-31T16:14:42.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.8703704, "accuracy_std_dev": 0.00583193054142453, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004018268686868687, "avg_duration_per_task_ms": 37454.91717171717, "avg_output_tokens_per_task": 7796.7444444444445, "last_run_timestamp": "2026-10-31T08:14:20.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.8703605355249205, "accuracy_std_dev": 0.03387763198941155, "run_count": 10, "total_questions": 1886, "avg_cost_per_task_usd": 0.014264545599151641, "avg_duration_per_task_ms": 348340.7725344645, "avg_output_tokens_per_task": 28651.488865323437, "last_run_timestamp": "2026-10-31T08:28:24.326Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.870112333932584, "accuracy_std_dev": 0.07630745338957075, "run_count": 23, "total_questions": 4450, "avg_cost_per_task_usd": 0.21229808741573034, "avg_duration_per_task_ms": 491047.8352808989, "avg_output_tokens_per_task": 61310.42764044944, "last_run_timestamp": "2026-10-31T09:02:18.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.8700336, "accuracy_std_dev": 0.008343089106559986, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.07480730858585859, "avg_duration_per_task_ms": 106906.41464646465, "avg_output_tokens_per_task": 12161.163636363637, "last_run_timestamp": "2026-10-31T08:03:09.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.8699495, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00502325, "avg_duration_per_task_ms": 389434.8611111111, "avg_output_tokens_per_task": 17944.10101010101, "last_run_timestamp": "2026-08-08T10:05:07.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.8699400591341078, "accuracy_std_dev": 0.01681928135474285, "run_count": 10, "total_questions": 1894, "avg_cost_per_task_usd": 0.07171590390707498, "avg_duration_per_task_ms": 408524.5543822598, "avg_output_tokens_per_task": 23628.092397043296, "last_run_timestamp": "2026-10-31T08:45:16.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8697338870630417, "accuracy_std_dev": 0.011260986943785035, "run_count": 17, "total_questions": 3347, "avg_cost_per_task_usd": 0.04833613654018524, "avg_duration_per_task_ms": 200583.7881685091, "avg_output_tokens_per_task": 37254.27158649537, "last_run_timestamp": "2026-10-31T08:35:39.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.8697226153846153, "accuracy_std_dev": 0.03509376134681856, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.18790192113442114, "avg_duration_per_task_ms": 46125.006604506605, "avg_output_tokens_per_task": 3550.8018648018647, "last_run_timestamp": "2026-10-31T08:26:18.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8695517762430939, "accuracy_std_dev": 0.027759107412948937, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.10362995395948434, "avg_duration_per_task_ms": 356439.69521178637, "avg_output_tokens_per_task": 23309.89779005525, "last_run_timestamp": "2026-10-31T08:27:45.609Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10538721464646464, "avg_duration_per_task_ms": 581981.0101010101, "avg_output_tokens_per_task": 34695.7398989899, "last_run_timestamp": "2026-08-08T09:03:54.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8695284999999999, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2202375707070707, "avg_duration_per_task_ms": 264875.98484848486, "avg_output_tokens_per_task": 21926.78787878788, "last_run_timestamp": "2026-10-01T16:12:49.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "ModelRun", "accuracy": 0.8695284999999998, "accuracy_std_dev": 0.01157224020015143, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.018078708333333332, "avg_duration_per_task_ms": 141813.27146464647, "avg_output_tokens_per_task": 29935.99873737374, "last_run_timestamp": "2026-10-31T08:24:37.341Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.8692809647058825, "accuracy_std_dev": 0.009104176033557385, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.0047775359477124175, "avg_duration_per_task_ms": 482494.045751634, "avg_output_tokens_per_task": 26353.58431372549, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.869060888888889, "accuracy_std_dev": 0.00603234318660567, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.008162227272727272, "avg_duration_per_task_ms": 360559.57575757575, "avg_output_tokens_per_task": 28803.3265993266, "last_run_timestamp": "2026-10-31T16:18:48.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8690236000000001, "accuracy_std_dev": 0.0056741455250989255, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004051724242424242, "avg_duration_per_task_ms": 38666.534343434345, "avg_output_tokens_per_task": 7872.538383838384, "last_run_timestamp": "2026-10-31T08:35:55.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.009143975622643609, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.2541581641414141, "avg_duration_per_task_ms": 251149.7840909091, "avg_output_tokens_per_task": 24116.972222222223, "last_run_timestamp": "2026-10-01T16:14:35.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.8686865, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06319552525252525, "avg_duration_per_task_ms": 266969.2752525252, "avg_output_tokens_per_task": 20992.954545454544, "last_run_timestamp": "2026-08-15T15:55:02.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.8686612136752137, "accuracy_std_dev": 0.02320764839922872, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.025878321367521367, "avg_duration_per_task_ms": 786906.9829059829, "avg_output_tokens_per_task": 23783.8, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8684277692307691, "accuracy_std_dev": 0.04024631421662945, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.1798952365967366, "avg_duration_per_task_ms": 44033.45998445999, "avg_output_tokens_per_task": 3389.598679098679, "last_run_timestamp": "2026-10-31T08:13:57.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8682058571428571, "accuracy_std_dev": 0.007673379706651383, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07688844300144301, "avg_duration_per_task_ms": 83582.13492063493, "avg_output_tokens_per_task": 12574.14357864358, "last_run_timestamp": "2026-10-10T08:21:46.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.8681253333333332, "accuracy_std_dev": 0.00965467639137749, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.021851893939393942, "avg_duration_per_task_ms": 524499.1397306398, "avg_output_tokens_per_task": 46276.513468013465, "last_run_timestamp": "2026-10-31T08:03:09.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8679098076923076, "accuracy_std_dev": 0.028085900492293595, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.1592424807692308, "avg_duration_per_task_ms": 153716.81565656565, "avg_output_tokens_per_task": 15358.370240870241, "last_run_timestamp": "2026-10-31T08:57:30.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.867634625, "accuracy_std_dev": 0.007431114989311851, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.025694838383838382, "avg_duration_per_task_ms": 248942.36553030304, "avg_output_tokens_per_task": 21167.008207070707, "last_run_timestamp": "2026-10-08T22:14:16.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8676153636363637, "accuracy_std_dev": 0.004014278793432891, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0702307419651056, "avg_duration_per_task_ms": 231760.75160697888, "avg_output_tokens_per_task": 11437.757575757576, "last_run_timestamp": "2026-10-31T08:17:29.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.8675645333333333, "accuracy_std_dev": 0.00826267276262484, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.004299042424242424, "avg_duration_per_task_ms": 36217.24208754209, "avg_output_tokens_per_task": 7829.657575757576, "last_run_timestamp": "2026-10-31T08:38:41.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.8675644166666667, "accuracy_std_dev": 0.02533032544684445, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0076437706228956225, "avg_duration_per_task_ms": 424115.41877104377, "avg_output_tokens_per_task": 25925.149831649833, "last_run_timestamp": "2026-10-31T08:17:29.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8675643333333334, "accuracy_std_dev": 0.0028614651647170092, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.009448984848484849, "avg_duration_per_task_ms": 79377.02525252526, "avg_output_tokens_per_task": 7744.79797979798, "last_run_timestamp": "2026-10-02T23:19:10.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.8674240000000001, "accuracy_std_dev": 0.0048169925783625705, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004064253787878788, "avg_duration_per_task_ms": 38624.37373737374, "avg_output_tokens_per_task": 7893.444444444444, "last_run_timestamp": "2026-10-31T08:27:45.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.8673413018549746, "accuracy_std_dev": 0.05933095903135433, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.25496933951658235, "avg_duration_per_task_ms": 1215292.3445756042, "avg_output_tokens_per_task": 56493.202360876894, "last_run_timestamp": "2026-10-31T16:10:09.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.8671716000000002, "accuracy_std_dev": 0.006408346388890055, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0041782994949494945, "avg_duration_per_task_ms": 35977.948484848486, "avg_output_tokens_per_task": 7724.74494949495, "last_run_timestamp": "2026-10-31T09:34:38.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.867003, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.22265377272727274, "avg_duration_per_task_ms": 265427.9646464646, "avg_output_tokens_per_task": 22169.171717171717, "last_run_timestamp": "2026-10-01T16:12:54.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.8668163333333333, "accuracy_std_dev": 0.017267918242927966, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.07368100841750841, "avg_duration_per_task_ms": 292389.96184062853, "avg_output_tokens_per_task": 20963.185746352414, "last_run_timestamp": "2026-10-31T16:09:19.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.866566745508982, "accuracy_std_dev": 0.010605013521914997, "run_count": 10, "total_questions": 1670, "avg_cost_per_task_usd": 0.05591402215568862, "avg_duration_per_task_ms": 429070.1862275449, "avg_output_tokens_per_task": 18535.95988023952, "last_run_timestamp": "2026-10-31T16:05:14.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.8659487366153845, "accuracy_std_dev": 0.016174915476337644, "run_count": 9, "total_questions": 1625, "avg_cost_per_task_usd": 0.005155188923076923, "avg_duration_per_task_ms": 688915.9618461538, "avg_output_tokens_per_task": 28341.659692307694, "last_run_timestamp": "2026-10-31T16:22:14.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8658810000000001, "accuracy_std_dev": 0.04916542176842041, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.11255429335016834, "avg_duration_per_task_ms": 478692.68855218857, "avg_output_tokens_per_task": 28354.992845117846, "last_run_timestamp": "2026-10-31T16:01:53.651Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8657292702702702, "accuracy_std_dev": 0.027335158307768007, "run_count": 37, "total_questions": 7326, "avg_cost_per_task_usd": 0.17380805050505052, "avg_duration_per_task_ms": 159066.58531258532, "avg_output_tokens_per_task": 16179.980343980344, "last_run_timestamp": "2026-10-31T08:33:01.888Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.8656005, "accuracy_std_dev": 0.00803956708515244, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02525451262626263, "avg_duration_per_task_ms": 233486.3627946128, "avg_output_tokens_per_task": 20811.0765993266, "last_run_timestamp": "2026-10-01T16:08:04.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Relace", "accuracy": 0.86532, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01495640909090909, "avg_duration_per_task_ms": 120441.0101010101, "avg_output_tokens_per_task": 29795.70707070707, "last_run_timestamp": "2026-10-24T15:00:02.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8653195, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008587179292929294, "avg_duration_per_task_ms": 240899.30303030304, "avg_output_tokens_per_task": 30387.573232323233, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.8650982536651893, "accuracy_std_dev": 0.03856212210008964, "run_count": 15, "total_questions": 2933, "avg_cost_per_task_usd": 0.016069654619843162, "avg_duration_per_task_ms": 435995.1789976134, "avg_output_tokens_per_task": 41080.64609614729, "last_run_timestamp": "2026-10-31T08:51:00.880Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.8650607692307695, "accuracy_std_dev": 0.013738118503771485, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.7501272863247864, "avg_duration_per_task_ms": 502836.99456099456, "avg_output_tokens_per_task": 29840.98679098679, "last_run_timestamp": "2026-10-31T08:23:25.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8649456666666666, "accuracy_std_dev": 0.026980251460816472, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.13166756565656565, "avg_duration_per_task_ms": 468303.6307519641, "avg_output_tokens_per_task": 34458.837261503926, "last_run_timestamp": "2026-10-31T16:30:53.502Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cohere", "accuracy": 0.8647586666666667, "accuracy_std_dev": 0.008943734131906087, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05048964983164983, "avg_duration_per_task_ms": 169537.7323232323, "avg_output_tokens_per_task": 41902.75589225589, "last_run_timestamp": "2026-10-31T08:15:03.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8645427692307693, "accuracy_std_dev": 0.039793064181421686, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.18345110683760685, "avg_duration_per_task_ms": 45870.56643356643, "avg_output_tokens_per_task": 3459.1363636363635, "last_run_timestamp": "2026-10-31T08:09:25.606Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.8644132307692307, "accuracy_std_dev": 0.07186191609570917, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.006970269619269618, "avg_duration_per_task_ms": 569923.0031080031, "avg_output_tokens_per_task": 28743.998834498834, "last_run_timestamp": "2026-10-31T08:07:18.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.8641975, "accuracy_std_dev": 0.018071121259715286, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.09928920664983164, "avg_duration_per_task_ms": 478656.78282828286, "avg_output_tokens_per_task": 26358.21085858586, "last_run_timestamp": "2026-10-31T16:03:21.454Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.8638953076923078, "accuracy_std_dev": 0.009790647408346209, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.8417052486402485, "avg_duration_per_task_ms": 411336.81507381506, "avg_output_tokens_per_task": 33477.120435120436, "last_run_timestamp": "2026-10-31T08:13:57.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.863852179419525, "accuracy_std_dev": 0.02754414592685712, "run_count": 10, "total_questions": 1895, "avg_cost_per_task_usd": 0.012207302902374668, "avg_duration_per_task_ms": 440151.53984168865, "avg_output_tokens_per_task": 28225.067018469657, "last_run_timestamp": "2026-10-31T08:36:30.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.8637468444693992, "accuracy_std_dev": 0.038710484413858166, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.16561091353172375, "avg_duration_per_task_ms": 965884.440763616, "avg_output_tokens_per_task": 55799.21560920831, "last_run_timestamp": "2026-10-31T16:11:41.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.863649682795699, "accuracy_std_dev": 0.01354727385860341, "run_count": 6, "total_questions": 1116, "avg_cost_per_task_usd": 0.01399078853046595, "avg_duration_per_task_ms": 437830.958781362, "avg_output_tokens_per_task": 25123.625448028673, "last_run_timestamp": "2026-10-31T08:16:12.491Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.8636430722945668, "accuracy_std_dev": 0.030671339204155882, "run_count": 12, "total_questions": 2227, "avg_cost_per_task_usd": 0.02386835383924562, "avg_duration_per_task_ms": 460447.3304894477, "avg_output_tokens_per_task": 22718.12303547373, "last_run_timestamp": "2026-10-17T08:18:04.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.8636363333333335, "accuracy_std_dev": 0.008473463964374614, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.056405106902356894, "avg_duration_per_task_ms": 429778.9991582492, "avg_output_tokens_per_task": 55743.61616161616, "last_run_timestamp": "2026-10-31T16:10:09.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8635067692307692, "accuracy_std_dev": 0.008442363578577946, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.7928760839160839, "avg_duration_per_task_ms": 453047.2128982129, "avg_output_tokens_per_task": 31542.781274281275, "last_run_timestamp": "2026-10-31T08:15:24.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.8633403579175706, "accuracy_std_dev": 0.03134350405895737, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.03313610050614606, "avg_duration_per_task_ms": 634565.6637744035, "avg_output_tokens_per_task": 38353.89515545915, "last_run_timestamp": "2026-10-31T08:09:30.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8630751666666665, "accuracy_std_dev": 0.009048483766478349, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0646409191919192, "avg_duration_per_task_ms": 213655.1308922559, "avg_output_tokens_per_task": 10505.142255892257, "last_run_timestamp": "2026-10-31T08:13:57.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.8629839648725213, "accuracy_std_dev": 0.012680294246303573, "run_count": 9, "total_questions": 1765, "avg_cost_per_task_usd": 0.07565230934844192, "avg_duration_per_task_ms": 287267.9807365439, "avg_output_tokens_per_task": 20888.441359773373, "last_run_timestamp": "2026-10-31T16:05:36.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.8624118181818182, "accuracy_std_dev": 0.010959488075454655, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0702179696969697, "avg_duration_per_task_ms": 231211.1772268136, "avg_output_tokens_per_task": 11433.751147842057, "last_run_timestamp": "2026-10-31T08:01:54.465Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.8623781294346978, "accuracy_std_dev": 0.016503860237700806, "run_count": 13, "total_questions": 2565, "avg_cost_per_task_usd": 0.08223287719298245, "avg_duration_per_task_ms": 172990.00506822614, "avg_output_tokens_per_task": 24199.74580896686, "last_run_timestamp": "2026-10-31T08:36:29.328Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.8622376508158508, "accuracy_std_dev": 0.05418494311168491, "run_count": 11, "total_questions": 2145, "avg_cost_per_task_usd": 0.07540476596736596, "avg_duration_per_task_ms": 485054.5156177156, "avg_output_tokens_per_task": 20286.058275058276, "last_run_timestamp": "2026-10-31T08:40:20.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.8621209999999999, "accuracy_std_dev": 0.016724375791042256, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.09178991565656566, "avg_duration_per_task_ms": 180823.8116161616, "avg_output_tokens_per_task": 25366.796464646464, "last_run_timestamp": "2026-10-31T16:04:29.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.8618672673926194, "accuracy_std_dev": 0.052319300007587534, "run_count": 9, "total_questions": 1653, "avg_cost_per_task_usd": 0.18415829764065333, "avg_duration_per_task_ms": 1613453.7084089534, "avg_output_tokens_per_task": 53479.07320024198, "last_run_timestamp": "2026-10-31T16:04:29.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.8617656666666667, "accuracy_std_dev": 0.018346628633681503, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.15345084287317617, "avg_duration_per_task_ms": 965626.3894500561, "avg_output_tokens_per_task": 48791.631313131315, "last_run_timestamp": "2026-10-31T16:05:14.701Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.8617121428571427, "accuracy_std_dev": 0.029687962026136223, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.15537036976911978, "avg_duration_per_task_ms": 152704.13347763347, "avg_output_tokens_per_task": 15319.186507936507, "last_run_timestamp": "2026-10-31T08:27:53.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.04603966955336814, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004525512626262627, "avg_duration_per_task_ms": 565963.2584175084, "avg_output_tokens_per_task": 18978.082491582492, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.8612792, "accuracy_std_dev": 0.03136197207702347, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005120404040404041, "avg_duration_per_task_ms": 530889.794949495, "avg_output_tokens_per_task": 17542.636363636364, "last_run_timestamp": "2026-09-05T09:22:54.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.8611756923076923, "accuracy_std_dev": 0.018869779487458927, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.7569548310023311, "avg_duration_per_task_ms": 510623.6328671329, "avg_output_tokens_per_task": 30145.944444444445, "last_run_timestamp": "2026-10-31T08:30:25.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.861111076923077, "accuracy_std_dev": 0.007633099173604557, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.8831994118104116, "avg_duration_per_task_ms": 414231.13616938615, "avg_output_tokens_per_task": 33454.88888888889, "last_run_timestamp": "2026-10-31T08:24:08.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.8607304246575341, "accuracy_std_dev": 0.01148253802030522, "run_count": 7, "total_questions": 1314, "avg_cost_per_task_usd": 0.02474931582952816, "avg_duration_per_task_ms": 139190.8691019787, "avg_output_tokens_per_task": 37895.824200913245, "last_run_timestamp": "2026-10-31T08:41:38.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.8606433333333333, "accuracy_std_dev": 0.02111334608988141, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.14206713636363638, "avg_duration_per_task_ms": 1276209.3136924803, "avg_output_tokens_per_task": 52392.67283950617, "last_run_timestamp": "2026-10-31T16:08:31.067Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.8606058666666668, "accuracy_std_dev": 0.011601831394894306, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.8426976885521886, "avg_duration_per_task_ms": 457770.45521885523, "avg_output_tokens_per_task": 33527.7531986532, "last_run_timestamp": "2026-10-31T08:07:51.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.860546054237288, "accuracy_std_dev": 0.01522905267434664, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.1431118, "avg_duration_per_task_ms": 301601.77118644066, "avg_output_tokens_per_task": 30086.04463276836, "last_run_timestamp": "2026-10-31T09:06:29.131Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8604851819757365, "accuracy_std_dev": 0.010061527125579174, "run_count": 9, "total_questions": 1731, "avg_cost_per_task_usd": 0.004011181398035817, "avg_duration_per_task_ms": 248472.44656268053, "avg_output_tokens_per_task": 28672.775852108607, "last_run_timestamp": "2026-10-31T16:05:36.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Io Net", "accuracy": 0.860269, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00987439393939394, "avg_duration_per_task_ms": 74831.28787878787, "avg_output_tokens_per_task": 9997.058080808081, "last_run_timestamp": "2026-10-24T08:14:36.264Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.8601396923076924, "accuracy_std_dev": 0.009255270194157684, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.8530733566433567, "avg_duration_per_task_ms": 482253.25835275836, "avg_output_tokens_per_task": 33932.094017094016, "last_run_timestamp": "2026-10-31T08:48:00.920Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Io Net", "accuracy": 0.8596746149425287, "accuracy_std_dev": 0.000831499999999985, "run_count": 2, "total_questions": 348, "avg_cost_per_task_usd": 0.0899417959770115, "avg_duration_per_task_ms": 1024745.4281609196, "avg_output_tokens_per_task": 35825.26724137931, "last_run_timestamp": "2026-10-27T13:57:40.548Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.859429902017291, "accuracy_std_dev": 0.05098775585122179, "run_count": 11, "total_questions": 2082, "avg_cost_per_task_usd": 0.011825536023054757, "avg_duration_per_task_ms": 442610.81796349667, "avg_output_tokens_per_task": 25201.657060518733, "last_run_timestamp": "2026-10-31T08:27:46.434Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8590389615384615, "accuracy_std_dev": 0.017598304950020197, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.8044641944444444, "avg_duration_per_task_ms": 487156.2365967366, "avg_output_tokens_per_task": 30492.046425796427, "last_run_timestamp": "2026-10-31T09:25:22.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.8590346, "accuracy_std_dev": 0.016996610739791626, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.75415548989899, "avg_duration_per_task_ms": 507155.8878787879, "avg_output_tokens_per_task": 30052.987878787877, "last_run_timestamp": "2026-10-31T08:09:30.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.8590258723037101, "accuracy_std_dev": 0.009219054418533492, "run_count": 21, "total_questions": 3477, "avg_cost_per_task_usd": 0.1720266922634455, "avg_duration_per_task_ms": 256422.9841817659, "avg_output_tokens_per_task": 31690.788323267185, "last_run_timestamp": "2026-09-26T08:54:22.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.858959888888889, "accuracy_std_dev": 0.04917250355133622, "run_count": 27, "total_questions": 5346, "avg_cost_per_task_usd": 0.18915268649457537, "avg_duration_per_task_ms": 41450.0987654321, "avg_output_tokens_per_task": 3406.6427235316123, "last_run_timestamp": "2026-10-31T08:27:45.697Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.8588278767123287, "accuracy_std_dev": 0.014585318196353787, "run_count": 3, "total_questions": 438, "avg_cost_per_task_usd": 0.005061143835616439, "avg_duration_per_task_ms": 414471.07305936073, "avg_output_tokens_per_task": 13714.753424657534, "last_run_timestamp": "2026-10-31T16:45:39.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08651329797979797, "avg_duration_per_task_ms": 499241.5707070707, "avg_output_tokens_per_task": 35858.23737373737, "last_run_timestamp": "2026-10-04T03:06:10.317Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8581648333333333, "accuracy_std_dev": 0.025909089369798823, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8336664709595959, "avg_duration_per_task_ms": 439138.7053872054, "avg_output_tokens_per_task": 33233.371632996634, "last_run_timestamp": "2026-10-31T08:35:35.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.857990281303602, "accuracy_std_dev": 0.030501594473235694, "run_count": 12, "total_questions": 2332, "avg_cost_per_task_usd": 0.13072562135506002, "avg_duration_per_task_ms": 510944.2062607204, "avg_output_tokens_per_task": 43400.17238421956, "last_run_timestamp": "2026-10-31T16:16:35.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.8579124, "accuracy_std_dev": 0.017384854495796048, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02057177575757576, "avg_duration_per_task_ms": 445221.4888888889, "avg_output_tokens_per_task": 40903.725252525255, "last_run_timestamp": "2026-10-31T08:25:16.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8575497692307692, "accuracy_std_dev": 0.016422560561869862, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.8395061091686092, "avg_duration_per_task_ms": 459222.6048951049, "avg_output_tokens_per_task": 33403.42074592075, "last_run_timestamp": "2026-10-31T08:40:44.982Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.857418, "accuracy_std_dev": null, "run_count": 1, "total_questions": 173, "avg_cost_per_task_usd": 0.08778269942196533, "avg_duration_per_task_ms": 317630.5202312139, "avg_output_tokens_per_task": 36575.57225433526, "last_run_timestamp": "2026-08-04T03:12:38.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.8571427108013937, "accuracy_std_dev": 0.01011902539224449, "run_count": 3, "total_questions": 574, "avg_cost_per_task_usd": 0.11599589721254357, "avg_duration_per_task_ms": 363022.99128919863, "avg_output_tokens_per_task": 29391.742160278747, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.8570892222222223, "accuracy_std_dev": 0.031137791974732713, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.015986119528619527, "avg_duration_per_task_ms": 333449.29124579125, "avg_output_tokens_per_task": 28587.83613916947, "last_run_timestamp": "2026-10-31T16:04:48.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Wafer", "accuracy": 0.8569025, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018898722222222223, "avg_duration_per_task_ms": 164757.63383838383, "avg_output_tokens_per_task": 31337.035353535353, "last_run_timestamp": "2026-10-24T08:17:56.393Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8569022857142858, "accuracy_std_dev": 0.009351683113815608, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005601821789321789, "avg_duration_per_task_ms": 514036.772005772, "avg_output_tokens_per_task": 30934.699134199134, "last_run_timestamp": "2026-10-31T16:16:55.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8569022500000001, "accuracy_std_dev": 0.013202173483464743, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.007818540404040404, "avg_duration_per_task_ms": 506894.8308080808, "avg_output_tokens_per_task": 28009.079545454544, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.8569019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20989276262626264, "avg_duration_per_task_ms": 263850.72222222225, "avg_output_tokens_per_task": 20893.070707070707, "last_run_timestamp": "2026-10-01T16:31:36.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8568555555555556, "accuracy_std_dev": 0.007583374638878124, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.06372034118967451, "avg_duration_per_task_ms": 605513.0182379349, "avg_output_tokens_per_task": 35143.53956228956, "last_run_timestamp": "2026-10-31T16:08:56.835Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.8568247201068804, "accuracy_std_dev": 0.015059304005690374, "run_count": 9, "total_questions": 1497, "avg_cost_per_task_usd": 0.004656902471609886, "avg_duration_per_task_ms": 413401.8376753507, "avg_output_tokens_per_task": 9906.163660654642, "last_run_timestamp": "2026-10-31T09:01:06.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.856687139561058, "accuracy_std_dev": 0.016735505421958285, "run_count": 9, "total_questions": 1777, "avg_cost_per_task_usd": 0.03746025323579066, "avg_duration_per_task_ms": 607320.918964547, "avg_output_tokens_per_task": 34838.97186268993, "last_run_timestamp": "2026-10-31T16:11:17.542Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.8565023144979739, "accuracy_std_dev": 0.02045561379357996, "run_count": 45, "total_questions": 8884, "avg_cost_per_task_usd": 0.2887550784556506, "avg_duration_per_task_ms": 342908.4242458352, "avg_output_tokens_per_task": 51958.11143628996, "last_run_timestamp": "2026-10-31T16:26:37.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.8558309545454545, "accuracy_std_dev": 0.010665991197502755, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.13619998117539026, "avg_duration_per_task_ms": 429202.50367309456, "avg_output_tokens_per_task": 44305.976124885216, "last_run_timestamp": "2026-10-31T16:26:59.503Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.8557935365853659, "accuracy_std_dev": 0.035827629886346836, "run_count": 41, "total_questions": 8118, "avg_cost_per_task_usd": 0.16502105962059618, "avg_duration_per_task_ms": 151492.22037447648, "avg_output_tokens_per_task": 15265.38950480414, "last_run_timestamp": "2026-10-31T08:51:08.192Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.8556823130287649, "accuracy_std_dev": 0.021169598748854452, "run_count": 12, "total_questions": 2364, "avg_cost_per_task_usd": 0.08525841285956007, "avg_duration_per_task_ms": 295845.3159898477, "avg_output_tokens_per_task": 25006.826565143823, "last_run_timestamp": "2026-10-31T08:34:08.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.8554993333333332, "accuracy_std_dev": 0.021190911552938067, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.043522018518518515, "avg_duration_per_task_ms": 663758.2626262626, "avg_output_tokens_per_task": 36261.223905723906, "last_run_timestamp": "2026-10-31T16:02:14.360Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.8550657272727274, "accuracy_std_dev": 0.012107062717806176, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.02919473599632691, "avg_duration_per_task_ms": 757966.051882461, "avg_output_tokens_per_task": 33302.242883379244, "last_run_timestamp": "2026-10-17T08:19:33.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.8549618244274809, "accuracy_std_dev": 0.005347999999999964, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.22707390839694658, "avg_duration_per_task_ms": 373958.2061068702, "avg_output_tokens_per_task": 50795.681933842236, "last_run_timestamp": "2026-08-15T08:57:45.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.8548688033707866, "accuracy_std_dev": 0.012107174401981674, "run_count": 3, "total_questions": 356, "avg_cost_per_task_usd": 0.00593769382022472, "avg_duration_per_task_ms": 221303.48314606742, "avg_output_tokens_per_task": 23969.63202247191, "last_run_timestamp": "2026-10-31T16:10:36.259Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.85479775, "accuracy_std_dev": 0.009017138608644082, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.09609784722222224, "avg_duration_per_task_ms": 435031.48232323234, "avg_output_tokens_per_task": 30609.608585858587, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8545531474226805, "accuracy_std_dev": 0.018117679996070137, "run_count": 10, "total_questions": 1940, "avg_cost_per_task_usd": 0.08358154742268041, "avg_duration_per_task_ms": 325785.14793814434, "avg_output_tokens_per_task": 28959.811340206186, "last_run_timestamp": "2026-10-31T08:56:36.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Tenstorrent", "accuracy": 0.8540966666666665, "accuracy_std_dev": 0.004199386807089248, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013766887205387206, "avg_duration_per_task_ms": 205590.59595959596, "avg_output_tokens_per_task": 29236.930976430976, "last_run_timestamp": "2026-10-31T08:17:28.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8540819492179129, "accuracy_std_dev": 0.036199470332124874, "run_count": 24, "total_questions": 4667, "avg_cost_per_task_usd": 0.004028577030212127, "avg_duration_per_task_ms": 78708.61088493679, "avg_output_tokens_per_task": 15107.985001071353, "last_run_timestamp": "2026-10-31T08:12:47.827Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.8539094444444444, "accuracy_std_dev": 0.018180043149387204, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.013447226711560045, "avg_duration_per_task_ms": 780881.4635241302, "avg_output_tokens_per_task": 28188.217171717173, "last_run_timestamp": "2026-10-31T16:17:15.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.8537224444444443, "accuracy_std_dev": 0.027781459489343335, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.07644378900112234, "avg_duration_per_task_ms": 480473.94051627384, "avg_output_tokens_per_task": 22998.121212121212, "last_run_timestamp": "2026-10-31T16:12:07.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8537037, "accuracy_std_dev": 0.012630753865466617, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.10325877878787879, "avg_duration_per_task_ms": 455294.6515151515, "avg_output_tokens_per_task": 31795.09494949495, "last_run_timestamp": "2026-10-31T08:16:11.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8536755833333335, "accuracy_std_dev": 0.03297161670148616, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.7094076557239056, "avg_duration_per_task_ms": 485859.81018518517, "avg_output_tokens_per_task": 28361.97053872054, "last_run_timestamp": "2026-10-31T08:09:42.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.853535, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.028549383838383836, "avg_duration_per_task_ms": 582750.7070707071, "avg_output_tokens_per_task": 33383.742424242424, "last_run_timestamp": "2026-08-08T09:24:32.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.8530808309576838, "accuracy_std_dev": 0.0923187678677938, "run_count": 23, "total_questions": 4490, "avg_cost_per_task_usd": 0.007865783073496658, "avg_duration_per_task_ms": 559605.6879732739, "avg_output_tokens_per_task": 27305.323830734967, "last_run_timestamp": "2026-10-31T09:04:13.487Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.8530642646239555, "accuracy_std_dev": 0.022329477337983097, "run_count": 4, "total_questions": 718, "avg_cost_per_task_usd": 0.02860370194986072, "avg_duration_per_task_ms": 460751.2409470752, "avg_output_tokens_per_task": 34791.85933147632, "last_run_timestamp": "2026-10-10T08:22:41.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.852693625, "accuracy_std_dev": 0.006627993756362076, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.05533727904040404, "avg_duration_per_task_ms": 425236.9147727273, "avg_output_tokens_per_task": 28304.02904040404, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.8523164263322883, "accuracy_std_dev": 0.010034068893524688, "run_count": 5, "total_questions": 957, "avg_cost_per_task_usd": 0.08246928213166145, "avg_duration_per_task_ms": 272317.5339602926, "avg_output_tokens_per_task": 27908.60710553814, "last_run_timestamp": "2026-10-31T08:44:39.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8521884, "accuracy_std_dev": 0.01923598183197312, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.030990156565656565, "avg_duration_per_task_ms": 619926.6106060606, "avg_output_tokens_per_task": 34097.76565656566, "last_run_timestamp": "2026-10-31T16:02:14.339Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8521639072512648, "accuracy_std_dev": 0.012873132633608224, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.08519417144463183, "avg_duration_per_task_ms": 288650.1905564924, "avg_output_tokens_per_task": 29819.736368746486, "last_run_timestamp": "2026-10-31T08:22:52.355Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.852137797527048, "accuracy_std_dev": 0.006816033761653469, "run_count": 10, "total_questions": 1941, "avg_cost_per_task_usd": 0.09748740082431735, "avg_duration_per_task_ms": 655650.1257083977, "avg_output_tokens_per_task": 41096.26429675425, "last_run_timestamp": "2026-10-31T16:11:17.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.8521062280219781, "accuracy_std_dev": 0.006055500000000047, "run_count": 2, "total_questions": 364, "avg_cost_per_task_usd": 0.005501717032967033, "avg_duration_per_task_ms": 539158.467032967, "avg_output_tokens_per_task": 15491.851648351649, "last_run_timestamp": "2026-10-17T08:17:58.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.16585097979797978, "avg_duration_per_task_ms": 389107.60101010103, "avg_output_tokens_per_task": 37536.631313131315, "last_run_timestamp": "2026-08-04T03:12:10.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8518517499999999, "accuracy_std_dev": 0.0020620621201845155, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.21063902146464644, "avg_duration_per_task_ms": 251655.29545454544, "avg_output_tokens_per_task": 19971.159090909092, "last_run_timestamp": "2026-10-01T16:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Parasail", "accuracy": 0.8512906666666668, "accuracy_std_dev": 0.00555530025191155, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017986696969696972, "avg_duration_per_task_ms": 129036.91750841751, "avg_output_tokens_per_task": 29778.104377104377, "last_run_timestamp": "2026-10-31T08:56:30.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8510099999999999, "accuracy_std_dev": 0.009995248508666484, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.013658645833333335, "avg_duration_per_task_ms": 646317.4116161616, "avg_output_tokens_per_task": 19001.599116161615, "last_run_timestamp": "2026-10-31T16:05:36.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.8510099999999999, "accuracy_std_dev": 0.037427264881366905, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.12598769823232323, "avg_duration_per_task_ms": 640494.7992424242, "avg_output_tokens_per_task": 42382.61111111111, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.8509165555555555, "accuracy_std_dev": 0.023084461599916624, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.046726610549943884, "avg_duration_per_task_ms": 634694.9528619528, "avg_output_tokens_per_task": 38211.56285072952, "last_run_timestamp": "2026-10-31T16:03:21.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.8508805384615384, "accuracy_std_dev": 0.0098378318641346, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.09800575252525251, "avg_duration_per_task_ms": 565289.1045066045, "avg_output_tokens_per_task": 30479.255633255634, "last_run_timestamp": "2026-10-31T16:01:53.623Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.8507279006896552, "accuracy_std_dev": 0.012037642028375229, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.03107361793103448, "avg_duration_per_task_ms": 729282.3756321839, "avg_output_tokens_per_task": 32410.26482758621, "last_run_timestamp": "2026-10-17T08:19:39.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.8507273166273585, "accuracy_std_dev": 0.011165344551781631, "run_count": 10, "total_questions": 1696, "avg_cost_per_task_usd": 0.006636630896226415, "avg_duration_per_task_ms": 404585.84021226416, "avg_output_tokens_per_task": 16352.901533018869, "last_run_timestamp": "2026-10-31T16:08:33.701Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.8505423333333332, "accuracy_std_dev": 0.03289190627224608, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.010776331649831649, "avg_duration_per_task_ms": 434634.2154882155, "avg_output_tokens_per_task": 38163.07968574635, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8504273076923077, "accuracy_std_dev": 0.019136243013573914, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.12441163325563323, "avg_duration_per_task_ms": 380633.7595182595, "avg_output_tokens_per_task": 38591.44055944056, "last_run_timestamp": "2026-10-31T16:08:08.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.8502976923076923, "accuracy_std_dev": 0.04563619826137448, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.16715858896658897, "avg_duration_per_task_ms": 165688.25796425797, "avg_output_tokens_per_task": 16495.934343434343, "last_run_timestamp": "2026-10-31T08:12:45.468Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.8502042599087539, "accuracy_std_dev": 0.01590746948888509, "run_count": 37, "total_questions": 7014, "avg_cost_per_task_usd": 0.16371538508696884, "avg_duration_per_task_ms": 188635.26946107784, "avg_output_tokens_per_task": 29341.152266894784, "last_run_timestamp": "2026-10-31T09:24:22.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8501748923435677, "accuracy_std_dev": 0.00914970442953917, "run_count": 13, "total_questions": 2573, "avg_cost_per_task_usd": 0.03854331947143411, "avg_duration_per_task_ms": 385001.11737271666, "avg_output_tokens_per_task": 39135.41857753595, "last_run_timestamp": "2026-10-31T16:26:59.494Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.8501685000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20853091414141414, "avg_duration_per_task_ms": 280266.345959596, "avg_output_tokens_per_task": 20755.358585858587, "last_run_timestamp": "2026-10-01T16:20:44.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.8497263442622951, "accuracy_std_dev": 0.01704, "run_count": 2, "total_questions": 366, "avg_cost_per_task_usd": 0.06129340163934426, "avg_duration_per_task_ms": 530265.6393442623, "avg_output_tokens_per_task": 25778.054644808744, "last_run_timestamp": "2026-08-15T08:47:51.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.8497037279483036, "accuracy_std_dev": 0.02211279185187559, "run_count": 16, "total_questions": 3095, "avg_cost_per_task_usd": 0.2026981760904685, "avg_duration_per_task_ms": 259750.08206785138, "avg_output_tokens_per_task": 43394.09111470113, "last_run_timestamp": "2026-10-24T16:29:51.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.8492062857142857, "accuracy_std_dev": 0.009075999999999973, "run_count": 2, "total_questions": 336, "avg_cost_per_task_usd": 0.1515307232142857, "avg_duration_per_task_ms": 1515335.5773809524, "avg_output_tokens_per_task": 50423.354166666664, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8490562452830188, "accuracy_std_dev": 0.004459999999999964, "run_count": 2, "total_questions": 265, "avg_cost_per_task_usd": 0.017200528301886793, "avg_duration_per_task_ms": 165535.98490566039, "avg_output_tokens_per_task": 28515.603773584906, "last_run_timestamp": "2026-10-31T08:20:07.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.84890575, "accuracy_std_dev": 0.05035112340045156, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1629483771043771, "avg_duration_per_task_ms": 162891.44065656565, "avg_output_tokens_per_task": 16117.493265993266, "last_run_timestamp": "2026-10-31T08:45:37.246Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8487436923076923, "accuracy_std_dev": 0.015982537328089897, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.05245943783993783, "avg_duration_per_task_ms": 270756.14801864803, "avg_output_tokens_per_task": 45704.256410256414, "last_run_timestamp": "2026-10-31T16:53:23.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0012582575757575758, "avg_duration_per_task_ms": 88400.0101010101, "avg_output_tokens_per_task": 17501.79797979798, "last_run_timestamp": "2026-10-01T08:22:54.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.8479167041666666, "accuracy_std_dev": 0.00828816902819917, "run_count": 5, "total_questions": 960, "avg_cost_per_task_usd": 0.16138468541666665, "avg_duration_per_task_ms": 277988.090625, "avg_output_tokens_per_task": 24274.81875, "last_run_timestamp": "2026-10-28T05:08:57.249Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.8477888928571429, "accuracy_std_dev": 0.029215151613674717, "run_count": 10, "total_questions": 1960, "avg_cost_per_task_usd": 0.1592395025510204, "avg_duration_per_task_ms": 438119.2229591837, "avg_output_tokens_per_task": 36131.57244897959, "last_run_timestamp": "2026-10-31T08:13:34.182Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.8476458282485876, "accuracy_std_dev": 0.0101614457790704, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.007194387005649717, "avg_duration_per_task_ms": 581669.6435028248, "avg_output_tokens_per_task": 17793.33220338983, "last_run_timestamp": "2026-10-31T16:12:28.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.8475903301204819, "accuracy_std_dev": 0.009100393388336854, "run_count": 13, "total_questions": 2490, "avg_cost_per_task_usd": 0.06626451124497991, "avg_duration_per_task_ms": 191280.34939759035, "avg_output_tokens_per_task": 20610.90562248996, "last_run_timestamp": "2026-10-31T16:08:31.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8473217418810629, "accuracy_std_dev": 0.031381320315484754, "run_count": 12, "total_questions": 2371, "avg_cost_per_task_usd": 0.009025258118937158, "avg_duration_per_task_ms": 479873.75579924084, "avg_output_tokens_per_task": 32127.526360185577, "last_run_timestamp": "2026-10-31T08:31:37.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.84722225, "accuracy_std_dev": 0.04063783312613384, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.09673938762626262, "avg_duration_per_task_ms": 1384478.7525252525, "avg_output_tokens_per_task": 42908.3345959596, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.8469361045003815, "accuracy_std_dev": 0.04231320024956013, "run_count": 8, "total_questions": 1311, "avg_cost_per_task_usd": 0.02445259496567506, "avg_duration_per_task_ms": 437957.52021357743, "avg_output_tokens_per_task": 38826.28070175439, "last_run_timestamp": "2026-10-31T08:55:46.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.8468660769230769, "accuracy_std_dev": 0.017453184397394857, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.12967881662781666, "avg_duration_per_task_ms": 476170.62509712507, "avg_output_tokens_per_task": 48636.332556332556, "last_run_timestamp": "2026-10-31T16:09:19.244Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.8468582586644126, "accuracy_std_dev": 0.013598361278927057, "run_count": 6, "total_questions": 1183, "avg_cost_per_task_usd": 0.26757311665257816, "avg_duration_per_task_ms": 498008.1420118343, "avg_output_tokens_per_task": 60614.17836010144, "last_run_timestamp": "2026-10-31T16:45:45.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.8468012941176469, "accuracy_std_dev": 0.0195477464979672, "run_count": 34, "total_questions": 6732, "avg_cost_per_task_usd": 0.23950519132501485, "avg_duration_per_task_ms": 493273.4841057635, "avg_output_tokens_per_task": 47308.403149138445, "last_run_timestamp": "2026-10-31T17:01:17.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.8466142222222222, "accuracy_std_dev": 0.010836379374616662, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.14184849326599328, "avg_duration_per_task_ms": 144926.00224466893, "avg_output_tokens_per_task": 32057.991582491584, "last_run_timestamp": "2026-10-31T08:40:49.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.8465387902641119, "accuracy_std_dev": 0.025032600265254102, "run_count": 10, "total_questions": 1931, "avg_cost_per_task_usd": 0.10684122164681513, "avg_duration_per_task_ms": 489966.58933195233, "avg_output_tokens_per_task": 34902.65044018643, "last_run_timestamp": "2026-10-31T09:15:06.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.8463805000000001, "accuracy_std_dev": 0.04340338597102762, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09990382196969698, "avg_duration_per_task_ms": 1192685.5157828282, "avg_output_tokens_per_task": 39566.316287878784, "last_run_timestamp": "2026-10-31T16:01:53.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8459293152073732, "accuracy_std_dev": 0.017246738162321286, "run_count": 11, "total_questions": 2170, "avg_cost_per_task_usd": 0.010452895391705068, "avg_duration_per_task_ms": 454110.96359447006, "avg_output_tokens_per_task": 34047.99170506912, "last_run_timestamp": "2026-10-31T08:27:51.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.8456636089524402, "accuracy_std_dev": 0.018418318454142355, "run_count": 17, "total_questions": 3217, "avg_cost_per_task_usd": 0.14255490332608023, "avg_duration_per_task_ms": 266408.3546782717, "avg_output_tokens_per_task": 28803.044140503574, "last_run_timestamp": "2026-10-31T08:33:10.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.8454543999999999, "accuracy_std_dev": 0.04169381837443052, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0038799494949494947, "avg_duration_per_task_ms": 558146.4727272728, "avg_output_tokens_per_task": 40172.065151515155, "last_run_timestamp": "2026-10-31T16:18:02.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.8452858999999998, "accuracy_std_dev": 0.01791875201820708, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.14104453232323233, "avg_duration_per_task_ms": 915068.2414141414, "avg_output_tokens_per_task": 55465.26919191919, "last_run_timestamp": "2026-10-31T16:10:36.259Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.8449505654205607, "accuracy_std_dev": 0.010833165164405542, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.01864547196261682, "avg_duration_per_task_ms": 133412.9898753894, "avg_output_tokens_per_task": 18649.78894080997, "last_run_timestamp": "2026-10-31T16:05:14.635Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8446468343465044, "accuracy_std_dev": 0.01783018725336332, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.0660811013171226, "avg_duration_per_task_ms": 245353.75430597772, "avg_output_tokens_per_task": 20336.286220871327, "last_run_timestamp": "2026-10-31T08:32:53.731Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.844276, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.036391522727272724, "avg_duration_per_task_ms": 53072.40404040404, "avg_output_tokens_per_task": 14468.015151515152, "last_run_timestamp": "2026-10-01T19:12:26.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.8441557142857142, "accuracy_std_dev": 0.035688144801137224, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.004898924242424242, "avg_duration_per_task_ms": 1008847.8311688312, "avg_output_tokens_per_task": 27765.935786435788, "last_run_timestamp": "2026-10-31T16:46:54.774Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8440043168316831, "accuracy_std_dev": 0.006860781750965057, "run_count": 8, "total_questions": 1515, "avg_cost_per_task_usd": 0.05332531353135314, "avg_duration_per_task_ms": 266581.80264026404, "avg_output_tokens_per_task": 22128.052145214522, "last_run_timestamp": "2026-09-26T09:50:39.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.8439077604166666, "accuracy_std_dev": 0.009070260174817954, "run_count": 12, "total_questions": 2112, "avg_cost_per_task_usd": 0.12450989630681818, "avg_duration_per_task_ms": 153233.97348484848, "avg_output_tokens_per_task": 31747.522727272728, "last_run_timestamp": "2026-10-31T08:45:40.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.8438831999999998, "accuracy_std_dev": 0.010796604899689534, "run_count": 30, "total_questions": 5940, "avg_cost_per_task_usd": 0.1486385685185185, "avg_duration_per_task_ms": 173235.29764309764, "avg_output_tokens_per_task": 32642.098316498315, "last_run_timestamp": "2026-10-31T08:24:42.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8435638461538462, "accuracy_std_dev": 0.032311682699906065, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.05297190248640248, "avg_duration_per_task_ms": 206855.22494172494, "avg_output_tokens_per_task": 21970.065268065267, "last_run_timestamp": "2026-10-31T16:28:22.255Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.843457793224299, "accuracy_std_dev": 0.05231442632207949, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.07964885591900311, "avg_duration_per_task_ms": 557670.449376947, "avg_output_tokens_per_task": 43630.647196261685, "last_run_timestamp": "2026-10-31T16:30:10.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.8434345000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.020032184343434345, "avg_duration_per_task_ms": 751978.8939393939, "avg_output_tokens_per_task": 23503.446969696968, "last_run_timestamp": "2026-08-15T14:57:08.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8434174840336134, "accuracy_std_dev": 0.0058964576706039785, "run_count": 7, "total_questions": 1190, "avg_cost_per_task_usd": 0.07875789159663865, "avg_duration_per_task_ms": 236163.5705882353, "avg_output_tokens_per_task": 19324.579831932773, "last_run_timestamp": "2026-10-08T22:17:54.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.8433784397283531, "accuracy_std_dev": 0.003767029717016947, "run_count": 6, "total_questions": 1178, "avg_cost_per_task_usd": 0.1209133650254669, "avg_duration_per_task_ms": 239041.5050933786, "avg_output_tokens_per_task": 32072.49235993209, "last_run_timestamp": "2026-10-31T09:12:37.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.8432182184684685, "accuracy_std_dev": 0.019959068726883127, "run_count": 9, "total_questions": 1332, "avg_cost_per_task_usd": 0.01802810960960961, "avg_duration_per_task_ms": 98985.72822822822, "avg_output_tokens_per_task": 15473.123123123123, "last_run_timestamp": "2026-10-31T16:06:00.067Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8431583644859812, "accuracy_std_dev": 0.027490521207033807, "run_count": 8, "total_questions": 1391, "avg_cost_per_task_usd": 0.08865105607476635, "avg_duration_per_task_ms": 1400795.4744787922, "avg_output_tokens_per_task": 40690.26815240834, "last_run_timestamp": "2026-10-31T16:04:23.355Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8426947486187845, "accuracy_std_dev": 0.027181441039645704, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.01345692311233886, "avg_duration_per_task_ms": 682984.7329650092, "avg_output_tokens_per_task": 42267.71869244936, "last_run_timestamp": "2026-10-31T08:57:00.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8425437868421053, "accuracy_std_dev": 0.00573439306339873, "run_count": 6, "total_questions": 1140, "avg_cost_per_task_usd": 0.08756267807017544, "avg_duration_per_task_ms": 282086.36140350875, "avg_output_tokens_per_task": 21480.616666666665, "last_run_timestamp": "2026-10-01T16:06:12.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.8422337612565445, "accuracy_std_dev": 0.023212455418589382, "run_count": 10, "total_questions": 1910, "avg_cost_per_task_usd": 0.10300291204188483, "avg_duration_per_task_ms": 203070.60157068062, "avg_output_tokens_per_task": 30819.346073298428, "last_run_timestamp": "2026-10-31T08:52:06.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.842080958563536, "accuracy_std_dev": 0.01924201603413352, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.02133896546961326, "avg_duration_per_task_ms": 337269.2900552486, "avg_output_tokens_per_task": 42459.25598526704, "last_run_timestamp": "2026-10-31T08:44:51.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.8419939695817491, "accuracy_std_dev": 0.05773343437894112, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.08744185044359949, "avg_duration_per_task_ms": 697734.2648922687, "avg_output_tokens_per_task": 26121.35234474018, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.841751, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06159810101010101, "avg_duration_per_task_ms": 79946.62794612795, "avg_output_tokens_per_task": 13559.319865319865, "last_run_timestamp": "2026-10-02T23:24:30.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.8416364037363778, "accuracy_std_dev": 0.015589563439410502, "run_count": 11, "total_questions": 1927, "avg_cost_per_task_usd": 0.1375403404255319, "avg_duration_per_task_ms": 272709.34509600414, "avg_output_tokens_per_task": 32850.36170212766, "last_run_timestamp": "2026-10-31T09:42:32.959Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.8414135533290239, "accuracy_std_dev": 0.01226653991581053, "run_count": 9, "total_questions": 1547, "avg_cost_per_task_usd": 0.006907548157724628, "avg_duration_per_task_ms": 661601.5623787977, "avg_output_tokens_per_task": 17049.550096961862, "last_run_timestamp": "2026-10-31T16:34:27.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.8411750606060605, "accuracy_std_dev": 0.008569864104844968, "run_count": 7, "total_questions": 1254, "avg_cost_per_task_usd": 0.01576657097288676, "avg_duration_per_task_ms": 669466.4673046252, "avg_output_tokens_per_task": 16726.208931419456, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8409089999999999, "accuracy_std_dev": 0.010943000000000036, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03708839646464646, "avg_duration_per_task_ms": 47515.843434343435, "avg_output_tokens_per_task": 14746.045454545454, "last_run_timestamp": "2026-10-01T19:13:29.958Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.840891889212828, "accuracy_std_dev": 0.01539818649719325, "run_count": 16, "total_questions": 3087, "avg_cost_per_task_usd": 0.022577127632005183, "avg_duration_per_task_ms": 250189.5425979916, "avg_output_tokens_per_task": 38449.29964366699, "last_run_timestamp": "2026-10-31T08:54:15.638Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.8408359820854386, "accuracy_std_dev": 0.011558364030339736, "run_count": 11, "total_questions": 2177, "avg_cost_per_task_usd": 0.14478948920532844, "avg_duration_per_task_ms": 203069.29581993568, "avg_output_tokens_per_task": 32673.29122645843, "last_run_timestamp": "2026-10-31T09:21:48.528Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.8408155555555555, "accuracy_std_dev": 0.0506301697894998, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0956665359147026, "avg_duration_per_task_ms": 384651.6548821549, "avg_output_tokens_per_task": 21452.003367003366, "last_run_timestamp": "2026-10-31T16:15:29.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8407407, "accuracy_std_dev": 0.024165205569371837, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.013536882323232325, "avg_duration_per_task_ms": 528967.1176767676, "avg_output_tokens_per_task": 40146.823737373736, "last_run_timestamp": "2026-10-31T08:45:25.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8406767200863932, "accuracy_std_dev": 0.03362626844282443, "run_count": 12, "total_questions": 2315, "avg_cost_per_task_usd": 0.030011868250539957, "avg_duration_per_task_ms": 286135.6241900648, "avg_output_tokens_per_task": 9270.458315334774, "last_run_timestamp": "2026-10-31T08:16:12.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8406685714285715, "accuracy_std_dev": 0.01850043854968342, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02123720418470418, "avg_duration_per_task_ms": 436896.4862914863, "avg_output_tokens_per_task": 42244.7886002886, "last_run_timestamp": "2026-10-31T08:09:30.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.8406284444444444, "accuracy_std_dev": 0.012648005025924159, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.08990638720538721, "avg_duration_per_task_ms": 1092332.543771044, "avg_output_tokens_per_task": 45444.60101010101, "last_run_timestamp": "2026-10-31T16:33:40.856Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.840488, "accuracy_std_dev": 0.00819350402758187, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03994661426767677, "avg_duration_per_task_ms": 52435.7619949495, "avg_output_tokens_per_task": 14777.814393939394, "last_run_timestamp": "2026-10-01T19:13:57.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.8404879999999999, "accuracy_std_dev": 0.020003728864889173, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.037026556818181816, "avg_duration_per_task_ms": 113840.68308080808, "avg_output_tokens_per_task": 34919.68308080808, "last_run_timestamp": "2026-10-31T09:18:03.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8404308818757921, "accuracy_std_dev": 0.01259237381870473, "run_count": 20, "total_questions": 3945, "avg_cost_per_task_usd": 0.10827099493029152, "avg_duration_per_task_ms": 326193.6968314322, "avg_output_tokens_per_task": 34498.35411913815, "last_run_timestamp": "2026-10-31T08:48:28.167Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.8401781603650587, "accuracy_std_dev": 0.014388231606073072, "run_count": 16, "total_questions": 3068, "avg_cost_per_task_usd": 0.07506749576271186, "avg_duration_per_task_ms": 388300.5521512386, "avg_output_tokens_per_task": 22304.916558018253, "last_run_timestamp": "2026-10-31T08:43:48.577Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.8400751575262545, "accuracy_std_dev": 0.014386829490645235, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.23145275106962276, "avg_duration_per_task_ms": 284144.37495138077, "avg_output_tokens_per_task": 52451.106184364064, "last_run_timestamp": "2026-10-31T16:57:18.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.8398090817995909, "accuracy_std_dev": 0.059804520823169324, "run_count": 3, "total_questions": 489, "avg_cost_per_task_usd": 0.019161192229038854, "avg_duration_per_task_ms": 228340.47034764825, "avg_output_tokens_per_task": 18072.97955010225, "last_run_timestamp": "2026-10-17T16:05:00.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.8397870000000001, "accuracy_std_dev": 0.06305207943174174, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013501392255892255, "avg_duration_per_task_ms": 269245.1767676768, "avg_output_tokens_per_task": 24293.242424242424, "last_run_timestamp": "2026-10-01T17:44:47.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.8397306000000001, "accuracy_std_dev": 0.024613795376576923, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.16074421262626265, "avg_duration_per_task_ms": 1000006.3535353536, "avg_output_tokens_per_task": 52830.63737373737, "last_run_timestamp": "2026-10-31T16:39:29.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.839693111111111, "accuracy_std_dev": 0.009704287579832906, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0060121301907968576, "avg_duration_per_task_ms": 439670.8260381594, "avg_output_tokens_per_task": 25980.03591470258, "last_run_timestamp": "2026-10-31T16:45:40.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.8391730814863104, "accuracy_std_dev": 0.011927781960856079, "run_count": 31, "total_questions": 6136, "avg_cost_per_task_usd": 0.26252152737940027, "avg_duration_per_task_ms": 275082.67649934813, "avg_output_tokens_per_task": 57033.21740547588, "last_run_timestamp": "2026-10-31T16:16:55.693Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.8387440275454993, "accuracy_std_dev": 0.019690853286246784, "run_count": 11, "total_questions": 2033, "avg_cost_per_task_usd": 0.07400631874077719, "avg_duration_per_task_ms": 403244.14658140676, "avg_output_tokens_per_task": 24425.756025577964, "last_run_timestamp": "2026-10-31T16:18:04.837Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.8386898181818183, "accuracy_std_dev": 0.022320469186247312, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1030642401285583, "avg_duration_per_task_ms": 384608.90220385673, "avg_output_tokens_per_task": 35835.99357208448, "last_run_timestamp": "2026-10-31T08:36:48.545Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.8384273393416928, "accuracy_std_dev": 0.02801308482872427, "run_count": 13, "total_questions": 2552, "avg_cost_per_task_usd": 0.06636876214733543, "avg_duration_per_task_ms": 1061039.8240595611, "avg_output_tokens_per_task": 33105.52742946708, "last_run_timestamp": "2026-10-31T16:05:36.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.8383837142857142, "accuracy_std_dev": 0.012139811903865117, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1527254054834055, "avg_duration_per_task_ms": 220532.05050505052, "avg_output_tokens_per_task": 34470.08658008658, "last_run_timestamp": "2026-10-31T08:45:52.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.8383836923076923, "accuracy_std_dev": 0.06521496654955188, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.20231615501165504, "avg_duration_per_task_ms": 532695.9992229992, "avg_output_tokens_per_task": 45956.61693861694, "last_run_timestamp": "2026-10-31T16:18:48.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.8379036028306059, "accuracy_std_dev": 0.019624931352535135, "run_count": 13, "total_questions": 2261, "avg_cost_per_task_usd": 0.09598150818222025, "avg_duration_per_task_ms": 610315.1362229103, "avg_output_tokens_per_task": 29918.706324635117, "last_run_timestamp": "2026-10-31T16:13:36.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.8378226666666666, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.15828378956228956, "avg_duration_per_task_ms": 446608.7861952862, "avg_output_tokens_per_task": 56298.93097643097, "last_run_timestamp": "2026-10-10T08:30:28.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.8375420000000001, "accuracy_std_dev": 0.005646742468007568, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02810060606060606, "avg_duration_per_task_ms": 530216.4090909091, "avg_output_tokens_per_task": 23231.272727272728, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.837542, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06186689898989899, "avg_duration_per_task_ms": 75089.61363636363, "avg_output_tokens_per_task": 13620.507575757576, "last_run_timestamp": "2026-10-01T19:12:44.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.8374772307692306, "accuracy_std_dev": 0.012060975124207245, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.08510472455322457, "avg_duration_per_task_ms": 722897.0310800311, "avg_output_tokens_per_task": 49108.018648018646, "last_run_timestamp": "2026-10-31T16:46:10.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.8371593636363637, "accuracy_std_dev": 0.013027195241792026, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.141865238292011, "avg_duration_per_task_ms": 181084.47015610652, "avg_output_tokens_per_task": 32015.05968778696, "last_run_timestamp": "2026-10-31T08:16:11.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.836653454183267, "accuracy_std_dev": 0.007833999999999952, "run_count": 2, "total_questions": 251, "avg_cost_per_task_usd": 0.06977503187250997, "avg_duration_per_task_ms": 237256.35059760956, "avg_output_tokens_per_task": 17511.450199203187, "last_run_timestamp": "2026-08-15T08:47:41.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.836139, "accuracy_std_dev": 0.05678065343641852, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005011018518518518, "avg_duration_per_task_ms": 214818.47643097644, "avg_output_tokens_per_task": 17529.722222222223, "last_run_timestamp": "2026-09-26T11:06:41.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.836139, "accuracy_std_dev": 0.0007933738084913017, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05002690909090909, "avg_duration_per_task_ms": 537584.335016835, "avg_output_tokens_per_task": 49932.06734006734, "last_run_timestamp": "2026-09-30T11:44:08.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8361277244542866, "accuracy_std_dev": 0.026614847390527968, "run_count": 16, "total_questions": 3161, "avg_cost_per_task_usd": 0.02298093546346093, "avg_duration_per_task_ms": 307170.6646630813, "avg_output_tokens_per_task": 18491.486871243276, "last_run_timestamp": "2026-10-31T08:31:11.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.8358585, "accuracy_std_dev": 0.01430950000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0011451691919191918, "avg_duration_per_task_ms": 116299.33333333333, "avg_output_tokens_per_task": 18355.219696969696, "last_run_timestamp": "2026-10-01T08:23:48.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.8356642307692309, "accuracy_std_dev": 0.03305920742722029, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.05416566394716394, "avg_duration_per_task_ms": 248261.90637140637, "avg_output_tokens_per_task": 22972.331002331004, "last_run_timestamp": "2026-10-31T16:08:08.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.8351849999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 180, "avg_cost_per_task_usd": 0.0030483999999999997, "avg_duration_per_task_ms": 288802.22222222225, "avg_output_tokens_per_task": 16703.5, "last_run_timestamp": "2026-10-01T08:43:11.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8348581735005084, "accuracy_std_dev": 0.011926219294758369, "run_count": 18, "total_questions": 2951, "avg_cost_per_task_usd": 0.1190305462555066, "avg_duration_per_task_ms": 291312.74483226024, "avg_output_tokens_per_task": 28021.837004405286, "last_run_timestamp": "2026-10-31T08:09:30.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.834789191724138, "accuracy_std_dev": 0.03295145245372225, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.10863842022988504, "avg_duration_per_task_ms": 1052937.216091954, "avg_output_tokens_per_task": 33937.42436781609, "last_run_timestamp": "2026-10-24T16:17:57.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.8346120933503837, "accuracy_std_dev": 0.029860364128538167, "run_count": 6, "total_questions": 782, "avg_cost_per_task_usd": 0.09530405242966751, "avg_duration_per_task_ms": 330065.4283887468, "avg_output_tokens_per_task": 39472.66240409207, "last_run_timestamp": "2026-10-31T16:08:31.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Modal", "accuracy": 0.8344576188870152, "accuracy_std_dev": 0.007843600547934322, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.033416617200674535, "avg_duration_per_task_ms": 208747.37942664418, "avg_output_tokens_per_task": 14517.00168634064, "last_run_timestamp": "2026-10-31T08:47:23.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.8344555555555555, "accuracy_std_dev": 0.01969686689756132, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.09693281088664421, "avg_duration_per_task_ms": 1118597.827721661, "avg_output_tokens_per_task": 44356.06565656565, "last_run_timestamp": "2026-10-31T16:08:56.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8341984152249134, "accuracy_std_dev": 0.029002188381032377, "run_count": 3, "total_questions": 578, "avg_cost_per_task_usd": 0.005492470588235294, "avg_duration_per_task_ms": 956260.6712802768, "avg_output_tokens_per_task": 15404.778546712803, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.8341543388067869, "accuracy_std_dev": 0.008301927538228698, "run_count": 10, "total_questions": 1827, "avg_cost_per_task_usd": 0.0431522140120416, "avg_duration_per_task_ms": 94743.94636015326, "avg_output_tokens_per_task": 15061.821018062397, "last_run_timestamp": "2026-10-31T08:22:53.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.8341254705882353, "accuracy_std_dev": 0.010649185432078013, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.25025737195484254, "avg_duration_per_task_ms": 384709.65567439096, "avg_output_tokens_per_task": 40253.002970885325, "last_run_timestamp": "2026-10-31T08:41:02.403Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.8340364430379746, "accuracy_std_dev": 0.01069546220626094, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0954248565400844, "avg_duration_per_task_ms": 293770.05400843883, "avg_output_tokens_per_task": 23403.932489451476, "last_run_timestamp": "2026-10-01T16:11:05.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.8338420961538461, "accuracy_std_dev": 0.01798642514244685, "run_count": 17, "total_questions": 3276, "avg_cost_per_task_usd": 0.2751184893162393, "avg_duration_per_task_ms": 391372.9496336996, "avg_output_tokens_per_task": 46721.76800976801, "last_run_timestamp": "2026-10-31T16:56:08.871Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8337216923076923, "accuracy_std_dev": 0.02540450493782736, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.13457357070707068, "avg_duration_per_task_ms": 539843.0198135198, "avg_output_tokens_per_task": 43011.44755244755, "last_run_timestamp": "2026-10-31T16:18:28.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.833146111111111, "accuracy_std_dev": 0.01582137855662708, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.13870717059483728, "avg_duration_per_task_ms": 298196.682379349, "avg_output_tokens_per_task": 57171.39337822671, "last_run_timestamp": "2026-10-31T16:45:48.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8330725023474179, "accuracy_std_dev": 0.010168177287347969, "run_count": 7, "total_questions": 1278, "avg_cost_per_task_usd": 0.029710525039123633, "avg_duration_per_task_ms": 655555.5469483568, "avg_output_tokens_per_task": 42323.18544600939, "last_run_timestamp": "2026-10-03T09:40:53.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.8328014100518549, "accuracy_std_dev": 0.010819971665955098, "run_count": 13, "total_questions": 2507, "avg_cost_per_task_usd": 0.23187096051057043, "avg_duration_per_task_ms": 302340.8938970881, "avg_output_tokens_per_task": 52526.70203430395, "last_run_timestamp": "2026-10-31T16:28:00.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.832772, "accuracy_std_dev": 0.0698509274145066, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.024015700336700335, "avg_duration_per_task_ms": 613738.0185185185, "avg_output_tokens_per_task": 59848.74074074074, "last_run_timestamp": "2026-10-31T08:32:40.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8327310906791908, "accuracy_std_dev": 0.020914023298509846, "run_count": 14, "total_questions": 2768, "avg_cost_per_task_usd": 0.02508389053468208, "avg_duration_per_task_ms": 393221.2268786127, "avg_output_tokens_per_task": 46919.84176300578, "last_run_timestamp": "2026-10-31T08:25:10.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.8326857692307692, "accuracy_std_dev": 0.011020358329744859, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.03860720590520591, "avg_duration_per_task_ms": 230762.9052059052, "avg_output_tokens_per_task": 14476.024864024865, "last_run_timestamp": "2026-10-31T08:25:00.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.8325688873598369, "accuracy_std_dev": 0.019237670701256954, "run_count": 10, "total_questions": 1962, "avg_cost_per_task_usd": 0.013576182466870541, "avg_duration_per_task_ms": 438297.6029561672, "avg_output_tokens_per_task": 39848.997961264016, "last_run_timestamp": "2026-10-31T08:26:26.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.8325147922396857, "accuracy_std_dev": 0.029981271466154875, "run_count": 11, "total_questions": 2036, "avg_cost_per_task_usd": 0.011545964145383104, "avg_duration_per_task_ms": 399504.8664047151, "avg_output_tokens_per_task": 22873.502946954814, "last_run_timestamp": "2026-10-31T08:40:04.563Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.8323464262877442, "accuracy_std_dev": 0.027721810831950897, "run_count": 9, "total_questions": 1689, "avg_cost_per_task_usd": 0.006784689757252812, "avg_duration_per_task_ms": 749921.5624629959, "avg_output_tokens_per_task": 18211.90763765542, "last_run_timestamp": "2026-10-31T16:04:29.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.8323081814454126, "accuracy_std_dev": 0.01351251744901741, "run_count": 10, "total_questions": 1951, "avg_cost_per_task_usd": 0.15155813992824194, "avg_duration_per_task_ms": 272697.51101998973, "avg_output_tokens_per_task": 34260.651973347005, "last_run_timestamp": "2026-10-31T08:37:13.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8320106428571429, "accuracy_std_dev": 0.007771735884483182, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.07774830699855699, "avg_duration_per_task_ms": 706637.7438672439, "avg_output_tokens_per_task": 57999.62698412698, "last_run_timestamp": "2026-10-31T16:05:36.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.8319864, "accuracy_std_dev": 0.01402072559605957, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.06441977171717173, "avg_duration_per_task_ms": 117114.99292929293, "avg_output_tokens_per_task": 13401.907070707071, "last_run_timestamp": "2026-10-01T19:14:05.661Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8319222756474104, "accuracy_std_dev": 0.02047048229572281, "run_count": 21, "total_questions": 4016, "avg_cost_per_task_usd": 0.04150572758964145, "avg_duration_per_task_ms": 442057.90986055776, "avg_output_tokens_per_task": 45303.60109561753, "last_run_timestamp": "2026-10-31T16:27:38.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8318478292335116, "accuracy_std_dev": 0.010811745465322452, "run_count": 17, "total_questions": 2805, "avg_cost_per_task_usd": 0.020570678431372547, "avg_duration_per_task_ms": 255956.05668449198, "avg_output_tokens_per_task": 40944.02673796791, "last_run_timestamp": "2026-10-31T08:20:07.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.831769061433447, "accuracy_std_dev": 0.006801018573632123, "run_count": 12, "total_questions": 2344, "avg_cost_per_task_usd": 0.03448967320819113, "avg_duration_per_task_ms": 208520.74530716724, "avg_output_tokens_per_task": 14194.120733788395, "last_run_timestamp": "2026-10-31T08:37:28.112Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.8316496, "accuracy_std_dev": 0.007062535765573171, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.022192562626262628, "avg_duration_per_task_ms": 11557.427272727273, "avg_output_tokens_per_task": 14346.162626262627, "last_run_timestamp": "2026-10-31T08:15:46.337Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "PrimeIntellect", "accuracy": 0.831532, "accuracy_std_dev": null, "run_count": 1, "total_questions": 185, "avg_cost_per_task_usd": 0.01692656216216216, "avg_duration_per_task_ms": 560352.7027027027, "avg_output_tokens_per_task": 13578.437837837839, "last_run_timestamp": "2026-10-31T18:41:15.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8314562888513515, "accuracy_std_dev": 0.012434701966673737, "run_count": 10, "total_questions": 1776, "avg_cost_per_task_usd": 0.21311484403153153, "avg_duration_per_task_ms": 464295.1722972973, "avg_output_tokens_per_task": 48323.44707207207, "last_run_timestamp": "2026-10-31T16:35:08.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.8314150997442455, "accuracy_std_dev": 0.012107134010057875, "run_count": 8, "total_questions": 1564, "avg_cost_per_task_usd": 0.02770090409207161, "avg_duration_per_task_ms": 583377.9558823529, "avg_output_tokens_per_task": 14708.68989769821, "last_run_timestamp": "2026-10-31T08:48:50.088Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.8313539815914488, "accuracy_std_dev": 0.013525888763386546, "run_count": 9, "total_questions": 1684, "avg_cost_per_task_usd": 0.02395000890736342, "avg_duration_per_task_ms": 287524.2090261283, "avg_output_tokens_per_task": 47679.917458432305, "last_run_timestamp": "2026-10-31T08:55:40.953Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.8312629347826086, "accuracy_std_dev": 0.044536486139877485, "run_count": 4, "total_questions": 644, "avg_cost_per_task_usd": 0.005694647515527951, "avg_duration_per_task_ms": 270388.4906832298, "avg_output_tokens_per_task": 13521.194099378881, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.8311777307159355, "accuracy_std_dev": 0.014826610062963942, "run_count": 14, "total_questions": 2165, "avg_cost_per_task_usd": 0.039417619399538105, "avg_duration_per_task_ms": 289131.9612009238, "avg_output_tokens_per_task": 32076.019399538105, "last_run_timestamp": "2026-10-31T16:19:08.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Amazon Bedrock", "accuracy": 0.8310883333333333, "accuracy_std_dev": 0.009654676391377534, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.20155932323232326, "avg_duration_per_task_ms": 266665.86195286195, "avg_output_tokens_per_task": 37910.23063973064, "last_run_timestamp": "2026-10-31T08:09:31.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.8309762, "accuracy_std_dev": 0.004338052761320463, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04722357777777777, "avg_duration_per_task_ms": 258256.24646464645, "avg_output_tokens_per_task": 15298.06111111111, "last_run_timestamp": "2026-10-31T08:20:09.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.8304553494159472, "accuracy_std_dev": 0.047802944677603286, "run_count": 10, "total_questions": 1969, "avg_cost_per_task_usd": 0.194895594718131, "avg_duration_per_task_ms": 603493.3336719146, "avg_output_tokens_per_task": 46389.83697308278, "last_run_timestamp": "2026-10-31T16:25:21.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.8302375046439628, "accuracy_std_dev": 0.02221590376784662, "run_count": 3, "total_questions": 323, "avg_cost_per_task_usd": 0.10860701238390094, "avg_duration_per_task_ms": 289034.8173374613, "avg_output_tokens_per_task": 24728.442724458204, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8302074978687126, "accuracy_std_dev": 0.018798024809619108, "run_count": 12, "total_questions": 2346, "avg_cost_per_task_usd": 0.14303728772378516, "avg_duration_per_task_ms": 291115.02642796247, "avg_output_tokens_per_task": 32292.15643648764, "last_run_timestamp": "2026-10-31T08:14:43.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.8301606153846154, "accuracy_std_dev": 0.025057780433164603, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.038417987567987574, "avg_duration_per_task_ms": 349233.05205905205, "avg_output_tokens_per_task": 39432.61577311577, "last_run_timestamp": "2026-10-24T16:01:48.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8295452499999999, "accuracy_std_dev": 0.02345987124192076, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07824036237373737, "avg_duration_per_task_ms": 700469.6212121212, "avg_output_tokens_per_task": 42273.732323232325, "last_run_timestamp": "2026-10-04T00:07:08.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.8295141534054742, "accuracy_std_dev": 0.11811955384270635, "run_count": 8, "total_questions": 1571, "avg_cost_per_task_usd": 0.12611730617441116, "avg_duration_per_task_ms": 777712.6670910248, "avg_output_tokens_per_task": 38502.726288987906, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8293709188811189, "accuracy_std_dev": 0.009670510440897125, "run_count": 4, "total_questions": 715, "avg_cost_per_task_usd": 0.025556813986013987, "avg_duration_per_task_ms": 118281.40559440559, "avg_output_tokens_per_task": 20315.50909090909, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.8292178888888889, "accuracy_std_dev": 0.011827528589828504, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00797142255892256, "avg_duration_per_task_ms": 508240.4511784512, "avg_output_tokens_per_task": 20513.609988776654, "last_run_timestamp": "2026-10-31T16:01:53.649Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.8288813468715698, "accuracy_std_dev": 0.048221988340541345, "run_count": 14, "total_questions": 2733, "avg_cost_per_task_usd": 0.09670949213318698, "avg_duration_per_task_ms": 542922.2052689352, "avg_output_tokens_per_task": 29030.10171972192, "last_run_timestamp": "2026-10-31T08:36:47.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.828497570984456, "accuracy_std_dev": 0.015230029187102692, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.11257541554404144, "avg_duration_per_task_ms": 438124.0186528497, "avg_output_tokens_per_task": 31261.672538860104, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.82828275, "accuracy_std_dev": 0.004610363346581274, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0223316047979798, "avg_duration_per_task_ms": 615131.3409090909, "avg_output_tokens_per_task": 56159.15404040404, "last_run_timestamp": "2026-10-17T08:18:07.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8280963467299964, "accuracy_std_dev": 0.019705580037059044, "run_count": 14, "total_questions": 2737, "avg_cost_per_task_usd": 0.2455083120204604, "avg_duration_per_task_ms": 398401.90317866276, "avg_output_tokens_per_task": 55634.735842162954, "last_run_timestamp": "2026-10-31T16:06:46.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.8280958888888889, "accuracy_std_dev": 0.00754749308555785, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.047590010101010105, "avg_duration_per_task_ms": 453658.93883277214, "avg_output_tokens_per_task": 26356.35521885522, "last_run_timestamp": "2026-10-31T16:10:31.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.8280885384615385, "accuracy_std_dev": 0.053874268721241306, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.05164174397824397, "avg_duration_per_task_ms": 528859.5505050505, "avg_output_tokens_per_task": 24724.757575757576, "last_run_timestamp": "2026-10-31T16:01:53.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8278389154691463, "accuracy_std_dev": 0.01931102964125829, "run_count": 6, "total_questions": 1183, "avg_cost_per_task_usd": 0.20861228233305154, "avg_duration_per_task_ms": 311401.96280642436, "avg_output_tokens_per_task": 47465.20540997464, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8272467692307692, "accuracy_std_dev": 0.013669555786794399, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.06361650932400933, "avg_duration_per_task_ms": 543058.7459207459, "avg_output_tokens_per_task": 50077.30147630148, "last_run_timestamp": "2026-10-31T16:08:56.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8270583636363636, "accuracy_std_dev": 0.00941702237510474, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.04288225068870524, "avg_duration_per_task_ms": 229942.66437098256, "avg_output_tokens_per_task": 14178.259871441689, "last_run_timestamp": "2026-10-31T08:36:47.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.8269358, "accuracy_std_dev": 0.006767606146932625, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.04578402121212121, "avg_duration_per_task_ms": 60291.652525252524, "avg_output_tokens_per_task": 11285.754545454545, "last_run_timestamp": "2026-09-26T09:19:03.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.8268397142857142, "accuracy_std_dev": 0.013924120672049498, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09914931746031745, "avg_duration_per_task_ms": 361736.2164502164, "avg_output_tokens_per_task": 45492.343434343435, "last_run_timestamp": "2026-10-17T16:06:16.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.8266562135355892, "accuracy_std_dev": 0.04220692579963176, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.08716451886425515, "avg_duration_per_task_ms": 756672.2042007002, "avg_output_tokens_per_task": 31524.158304161803, "last_run_timestamp": "2026-10-31T16:01:53.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8266521780172412, "accuracy_std_dev": 0.022372414615840108, "run_count": 12, "total_questions": 2320, "avg_cost_per_task_usd": 0.02234062025862069, "avg_duration_per_task_ms": 235753.16637931034, "avg_output_tokens_per_task": 44456.286637931036, "last_run_timestamp": "2026-10-31T08:28:04.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.82659925, "accuracy_std_dev": 0.019009280711996968, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02540684343434343, "avg_duration_per_task_ms": 237042.44823232322, "avg_output_tokens_per_task": 21223.366161616163, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.826459, "accuracy_std_dev": 0.009410002267091476, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.14383780639730645, "avg_duration_per_task_ms": 198190.3564814815, "avg_output_tokens_per_task": 35703.72769360269, "last_run_timestamp": "2026-10-31T08:32:01.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Darkbloom", "accuracy": 0.826417643153527, "accuracy_std_dev": 0.01028202813164795, "run_count": 4, "total_questions": 723, "avg_cost_per_task_usd": 0.010974759336099586, "avg_duration_per_task_ms": 1541635.907330567, "avg_output_tokens_per_task": 31233.395573997233, "last_run_timestamp": "2026-10-31T08:24:48.465Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.8264159143968872, "accuracy_std_dev": 0.0040531378816294926, "run_count": 4, "total_questions": 771, "avg_cost_per_task_usd": 0.13822659403372245, "avg_duration_per_task_ms": 272716.0363164721, "avg_output_tokens_per_task": 34382.090791180286, "last_run_timestamp": "2026-10-31T08:38:15.143Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.8264111325878595, "accuracy_std_dev": 0.01425677691801343, "run_count": 10, "total_questions": 1878, "avg_cost_per_task_usd": 0.02454355484558041, "avg_duration_per_task_ms": 264967.17305644305, "avg_output_tokens_per_task": 50263.019169329076, "last_run_timestamp": "2026-10-31T08:17:29.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8263836795903439, "accuracy_std_dev": 0.029166866420929173, "run_count": 7, "total_questions": 1367, "avg_cost_per_task_usd": 0.07172207900512069, "avg_duration_per_task_ms": 248028.12801755668, "avg_output_tokens_per_task": 21803.532553035846, "last_run_timestamp": "2026-10-31T09:17:12.929Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.826225, "accuracy_std_dev": 0.028929060222090415, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.09852584455667786, "avg_duration_per_task_ms": 1409812.7817059483, "avg_output_tokens_per_task": 45348.74074074074, "last_run_timestamp": "2026-10-31T16:08:33.697Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8259871818181819, "accuracy_std_dev": 0.010774204132752242, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03615010422405877, "avg_duration_per_task_ms": 305458.8122130395, "avg_output_tokens_per_task": 15557.78971533517, "last_run_timestamp": "2026-10-31T08:32:33.985Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8258563532084044, "accuracy_std_dev": 0.010360732650818109, "run_count": 9, "total_questions": 1761, "avg_cost_per_task_usd": 0.008003823395797843, "avg_duration_per_task_ms": 338944.8080636002, "avg_output_tokens_per_task": 19222.395797842135, "last_run_timestamp": "2026-10-31T16:08:31.066Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.8258509999999999, "accuracy_std_dev": 0.01267571617086958, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.194836923681257, "avg_duration_per_task_ms": 926868.8086419753, "avg_output_tokens_per_task": 51862.713243546576, "last_run_timestamp": "2026-10-31T16:50:02.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.82575775, "accuracy_std_dev": 0.010065865980505628, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.14380135795454543, "avg_duration_per_task_ms": 418612.6521464646, "avg_output_tokens_per_task": 28784.420454545456, "last_run_timestamp": "2026-10-31T08:40:11.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.8257574999999999, "accuracy_std_dev": 0.015993499999999994, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2609003181818182, "avg_duration_per_task_ms": 231557.84848484848, "avg_output_tokens_per_task": 25997.717171717173, "last_run_timestamp": "2026-10-01T19:12:42.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.825389385046729, "accuracy_std_dev": 0.014184335849948446, "run_count": 11, "total_questions": 2140, "avg_cost_per_task_usd": 0.03383954813084113, "avg_duration_per_task_ms": 407090.0654205608, "avg_output_tokens_per_task": 47991.5476635514, "last_run_timestamp": "2026-10-31T08:48:01.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8244948000000002, "accuracy_std_dev": 0.0601779527199123, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.028505976262626263, "avg_duration_per_task_ms": 700158.9757575757, "avg_output_tokens_per_task": 29700.529292929292, "last_run_timestamp": "2026-10-17T08:18:04.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.8243546666666667, "accuracy_std_dev": 0.03111308863734931, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.25956171212121215, "avg_duration_per_task_ms": 233681.0909090909, "avg_output_tokens_per_task": 25863.887205387204, "last_run_timestamp": "2026-10-01T19:17:25.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.8242303292875989, "accuracy_std_dev": 0.016434035763013298, "run_count": 20, "total_questions": 3790, "avg_cost_per_task_usd": 0.030382969920844324, "avg_duration_per_task_ms": 266171.0530343008, "avg_output_tokens_per_task": 14053.306068601583, "last_run_timestamp": "2026-10-31T08:10:37.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8239975454545455, "accuracy_std_dev": 0.022862830634275574, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.010785916437098255, "avg_duration_per_task_ms": 717852.0280073462, "avg_output_tokens_per_task": 38282.08126721763, "last_run_timestamp": "2026-10-17T08:02:36.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8238164069389231, "accuracy_std_dev": 0.03209164605620812, "run_count": 14, "total_questions": 2767, "avg_cost_per_task_usd": 0.08185260065052402, "avg_duration_per_task_ms": 342308.28370075894, "avg_output_tokens_per_task": 20334.039392844235, "last_run_timestamp": "2026-10-31T16:02:14.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.8236529999999999, "accuracy_std_dev": 0.0034448118235979193, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.018703443181818182, "avg_duration_per_task_ms": 518705.38636363635, "avg_output_tokens_per_task": 18559.679292929293, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.823232, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025912994949494946, "avg_duration_per_task_ms": 162312.898989899, "avg_output_tokens_per_task": 9036.257575757576, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.82281125, "accuracy_std_dev": 0.014500513393928511, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.024772287878787876, "avg_duration_per_task_ms": 511120.71338383836, "avg_output_tokens_per_task": 23142.588383838385, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.8225588000000001, "accuracy_std_dev": 0.009908448060115158, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02341788282828283, "avg_duration_per_task_ms": 349498.4303030303, "avg_output_tokens_per_task": 20884.227272727272, "last_run_timestamp": "2026-10-04T00:17:30.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.8225450489150423, "accuracy_std_dev": 0.023532795309234442, "run_count": 14, "total_questions": 2719, "avg_cost_per_task_usd": 0.026525477749172497, "avg_duration_per_task_ms": 850874.7859507172, "avg_output_tokens_per_task": 49891.477013607946, "last_run_timestamp": "2026-10-31T08:42:18.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.0244105, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2415952904040404, "avg_duration_per_task_ms": 181953.0101010101, "avg_output_tokens_per_task": 24066.49494949495, "last_run_timestamp": "2026-10-01T19:12:31.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.8223905, "accuracy_std_dev": 0.019262412208322532, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03192426473063974, "avg_duration_per_task_ms": 260830.34259259258, "avg_output_tokens_per_task": 14583.287878787878, "last_run_timestamp": "2026-10-31T08:17:29.187Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.8223140000000001, "accuracy_std_dev": 0.020822165248512558, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.012249293847566574, "avg_duration_per_task_ms": 720758.7061524334, "avg_output_tokens_per_task": 36235.862258953166, "last_run_timestamp": "2026-10-17T08:04:21.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.8221762091333679, "accuracy_std_dev": 0.01082457395235492, "run_count": 10, "total_questions": 1927, "avg_cost_per_task_usd": 0.024801258432797096, "avg_duration_per_task_ms": 452703.2304099637, "avg_output_tokens_per_task": 24752.417747794498, "last_run_timestamp": "2026-10-31T16:42:08.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8221660379266751, "accuracy_std_dev": 0.02808252331410946, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.20924120101137803, "avg_duration_per_task_ms": 851157.8482932997, "avg_output_tokens_per_task": 47928.662452591656, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.8216674158517757, "accuracy_std_dev": 0.01474539844154779, "run_count": 10, "total_questions": 1943, "avg_cost_per_task_usd": 0.018114979413278436, "avg_duration_per_task_ms": 598455.0849202265, "avg_output_tokens_per_task": 20142.77406073083, "last_run_timestamp": "2026-10-31T16:13:07.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.8216317349778621, "accuracy_std_dev": 0.09289881485376644, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.09027165528146741, "avg_duration_per_task_ms": 645885.798228969, "avg_output_tokens_per_task": 25622.579380139152, "last_run_timestamp": "2026-10-31T16:16:30.410Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cohere", "accuracy": 0.8215486666666668, "accuracy_std_dev": 0.015611992961253275, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02833054377104377, "avg_duration_per_task_ms": 228888.5606060606, "avg_output_tokens_per_task": 56421.28451178451, "last_run_timestamp": "2026-10-31T08:03:09.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.8214198161925602, "accuracy_std_dev": 0.04954945097072217, "run_count": 8, "total_questions": 1371, "avg_cost_per_task_usd": 0.04645164551422319, "avg_duration_per_task_ms": 608936.6484318016, "avg_output_tokens_per_task": 28638.388767323122, "last_run_timestamp": "2026-09-26T09:56:00.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.8210525603238865, "accuracy_std_dev": 0.07599126782917703, "run_count": 7, "total_questions": 1235, "avg_cost_per_task_usd": 0.007044106882591094, "avg_duration_per_task_ms": 525854.9595141701, "avg_output_tokens_per_task": 14201.662348178137, "last_run_timestamp": "2026-10-03T10:22:02.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8207070000000001, "accuracy_std_dev": 0.024411000000000016, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04063828030303031, "avg_duration_per_task_ms": 83296.42424242424, "avg_output_tokens_per_task": 26969.131313131315, "last_run_timestamp": "2026-10-01T19:23:01.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.8203833076923077, "accuracy_std_dev": 0.018076297388844354, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.042996285159285164, "avg_duration_per_task_ms": 241748.8038073038, "avg_output_tokens_per_task": 14318.661227661227, "last_run_timestamp": "2026-10-31T08:09:30.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8202748856394587, "accuracy_std_dev": 0.060149001089135595, "run_count": 24, "total_questions": 4582, "avg_cost_per_task_usd": 0.0076442278481012645, "avg_duration_per_task_ms": 496763.3814927979, "avg_output_tokens_per_task": 17412.37341772152, "last_run_timestamp": "2026-10-31T16:45:45.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.8202743410372041, "accuracy_std_dev": 0.011484132116496031, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.09063781341600902, "avg_duration_per_task_ms": 168767.6234498309, "avg_output_tokens_per_task": 22236.227170236754, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.8200275393258427, "accuracy_std_dev": 0.011794769394173345, "run_count": 9, "total_questions": 1691, "avg_cost_per_task_usd": 0.03818077646363098, "avg_duration_per_task_ms": 215799.96629213484, "avg_output_tokens_per_task": 14692.68894145476, "last_run_timestamp": "2026-10-31T08:55:11.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.8194060909090909, "accuracy_std_dev": 0.013884218840464785, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.033594433884297524, "avg_duration_per_task_ms": 211994.46005509642, "avg_output_tokens_per_task": 13897.44214876033, "last_run_timestamp": "2026-10-31T08:09:30.951Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.819304, "accuracy_std_dev": 0.02899661507831561, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.08773566049382715, "avg_duration_per_task_ms": 1029441.7283950617, "avg_output_tokens_per_task": 43316.07407407407, "last_run_timestamp": "2026-10-31T16:08:08.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.8192149207920791, "accuracy_std_dev": 0.018823182524040232, "run_count": 6, "total_questions": 909, "avg_cost_per_task_usd": 0.11224348184818483, "avg_duration_per_task_ms": 401079.6094609461, "avg_output_tokens_per_task": 46532.80198019802, "last_run_timestamp": "2026-10-17T16:06:46.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.8191759532044761, "accuracy_std_dev": 0.09354392281067757, "run_count": 10, "total_questions": 1966, "avg_cost_per_task_usd": 0.06251466022380467, "avg_duration_per_task_ms": 1731771.3148524924, "avg_output_tokens_per_task": 19617.884028484234, "last_run_timestamp": "2026-10-31T16:13:09.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.8191437142857143, "accuracy_std_dev": 0.03986796895724072, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04288789971139971, "avg_duration_per_task_ms": 992008.3816738817, "avg_output_tokens_per_task": 26956.146464646463, "last_run_timestamp": "2026-10-17T08:20:47.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.8184407692307691, "accuracy_std_dev": 0.012119050255716884, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.3141125089355089, "avg_duration_per_task_ms": 289618.7264957265, "avg_output_tokens_per_task": 20750.078088578088, "last_run_timestamp": "2026-10-31T08:15:24.404Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.81818175, "accuracy_std_dev": 0.008828586845441324, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06985739772727273, "avg_duration_per_task_ms": 843652.7916666666, "avg_output_tokens_per_task": 40039.878787878784, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.8181817499999999, "accuracy_std_dev": 0.021788216475597543, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02338400883838384, "avg_duration_per_task_ms": 497693.29671717173, "avg_output_tokens_per_task": 22657.377525252527, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.8179862262773724, "accuracy_std_dev": 0.005388830217320935, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.026104914093206064, "avg_duration_per_task_ms": 325164.1538461539, "avg_output_tokens_per_task": 26027.919708029196, "last_run_timestamp": "2026-10-31T16:04:48.178Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.8178727706422019, "accuracy_std_dev": 0.03174962566960436, "run_count": 5, "total_questions": 981, "avg_cost_per_task_usd": 0.023857419979612642, "avg_duration_per_task_ms": 166236.06523955148, "avg_output_tokens_per_task": 14523.781855249745, "last_run_timestamp": "2026-10-31T08:07:51.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.8178077777777777, "accuracy_std_dev": 0.009999885885768644, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.024910318742985414, "avg_duration_per_task_ms": 190389.1571268238, "avg_output_tokens_per_task": 16514.1734006734, "last_run_timestamp": "2026-10-03T10:10:35.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.8177376144306652, "accuracy_std_dev": 0.03151916867911054, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.013423886696730551, "avg_duration_per_task_ms": 855848.4718151072, "avg_output_tokens_per_task": 39679.556933483655, "last_run_timestamp": "2026-10-17T08:19:03.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.8176205555555557, "accuracy_std_dev": 0.01985605337372578, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.006378395061728395, "avg_duration_per_task_ms": 440890.9601571268, "avg_output_tokens_per_task": 18699.455667789, "last_run_timestamp": "2026-10-31T16:12:28.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.8174803333333334, "accuracy_std_dev": 0.015915170704568507, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01320759595959596, "avg_duration_per_task_ms": 508912.26851851854, "avg_output_tokens_per_task": 12721.0867003367, "last_run_timestamp": "2026-10-31T08:36:26.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.8173077377622378, "accuracy_std_dev": 0.04183386114142466, "run_count": 3, "total_questions": 572, "avg_cost_per_task_usd": 0.00918055944055944, "avg_duration_per_task_ms": 556054.2482517483, "avg_output_tokens_per_task": 20397.216783216783, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.8172424786585366, "accuracy_std_dev": 0.032381379178472315, "run_count": 8, "total_questions": 984, "avg_cost_per_task_usd": 0.0567359319105691, "avg_duration_per_task_ms": 328244.918699187, "avg_output_tokens_per_task": 21710.94918699187, "last_run_timestamp": "2026-10-31T16:26:17.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.8171817336142323, "accuracy_std_dev": 0.04982830145510509, "run_count": 11, "total_questions": 2136, "avg_cost_per_task_usd": 0.018343408239700373, "avg_duration_per_task_ms": 793559.7439138577, "avg_output_tokens_per_task": 48776.84925093633, "last_run_timestamp": "2026-10-31T08:31:53.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.8171795172255397, "accuracy_std_dev": 0.008683223783407119, "run_count": 11, "total_questions": 2177, "avg_cost_per_task_usd": 0.03297541662838769, "avg_duration_per_task_ms": 545292.2503445108, "avg_output_tokens_per_task": 15758.780431786863, "last_run_timestamp": "2026-10-31T08:29:48.679Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.8170594444444444, "accuracy_std_dev": 0.01614747979036502, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.007438095959595959, "avg_duration_per_task_ms": 496093.94051627384, "avg_output_tokens_per_task": 18405.01571268238, "last_run_timestamp": "2026-10-31T16:13:09.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.816967559819413, "accuracy_std_dev": 0.022532849378432182, "run_count": 9, "total_questions": 1772, "avg_cost_per_task_usd": 0.021389000564334085, "avg_duration_per_task_ms": 297135.54796839727, "avg_output_tokens_per_task": 17946.336343115123, "last_run_timestamp": "2026-10-03T09:25:02.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.8169574545454545, "accuracy_std_dev": 0.03032911546941362, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.029461516069788796, "avg_duration_per_task_ms": 587955.8774104683, "avg_output_tokens_per_task": 53445.840679522495, "last_run_timestamp": "2026-10-31T08:57:28.755Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.8168723333333333, "accuracy_std_dev": 0.007519404837559345, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.026859203703703705, "avg_duration_per_task_ms": 171179.37822671156, "avg_output_tokens_per_task": 23402.503928170594, "last_run_timestamp": "2026-10-31T16:09:44.821Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8165543999999997, "accuracy_std_dev": 0.010562628756138299, "run_count": 30, "total_questions": 5940, "avg_cost_per_task_usd": 0.33263593888888887, "avg_duration_per_task_ms": 295154.08097643097, "avg_output_tokens_per_task": 21001.168855218853, "last_run_timestamp": "2026-10-31T08:27:57.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.8161616, "accuracy_std_dev": 0.060936149095918435, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.09838498181818181, "avg_duration_per_task_ms": 774603.698989899, "avg_output_tokens_per_task": 39333.56565656565, "last_run_timestamp": "2026-10-03T17:58:22.995Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.81607725, "accuracy_std_dev": 0.011564661524986357, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.018877026515151513, "avg_duration_per_task_ms": 531080.7954545454, "avg_output_tokens_per_task": 18678.97095959596, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.815937111111111, "accuracy_std_dev": 0.004627552783646224, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.011142754769921435, "avg_duration_per_task_ms": 816705.024691358, "avg_output_tokens_per_task": 18593.037598204264, "last_run_timestamp": "2026-10-31T16:13:09.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.815563, "accuracy_std_dev": 0.04477470762482865, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.20474397923681256, "avg_duration_per_task_ms": 873869.0656565657, "avg_output_tokens_per_task": 51153.14253647587, "last_run_timestamp": "2026-10-31T16:08:08.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.815286480254777, "accuracy_std_dev": 0.006665997576319691, "run_count": 4, "total_questions": 785, "avg_cost_per_task_usd": 0.08723442165605096, "avg_duration_per_task_ms": 185900.4152866242, "avg_output_tokens_per_task": 24915.47388535032, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.8152402988505746, "accuracy_std_dev": 0.029604949040447602, "run_count": 4, "total_questions": 783, "avg_cost_per_task_usd": 0.28207722477650066, "avg_duration_per_task_ms": 913648.006385696, "avg_output_tokens_per_task": 58474.00127713921, "last_run_timestamp": "2026-10-31T16:12:07.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.81523575, "accuracy_std_dev": 0.04077706865920479, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.14845152777777776, "avg_duration_per_task_ms": 1663278.9867424243, "avg_output_tokens_per_task": 48562.03977272727, "last_run_timestamp": "2026-10-31T16:08:56.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.8151218824415977, "accuracy_std_dev": 0.008978671073432733, "run_count": 7, "total_questions": 1327, "avg_cost_per_task_usd": 0.04307039035418237, "avg_duration_per_task_ms": 180891.98040693294, "avg_output_tokens_per_task": 17051.95177091183, "last_run_timestamp": "2026-10-31T08:27:47.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.8148975767263426, "accuracy_std_dev": 0.10918305406308805, "run_count": 9, "total_questions": 1564, "avg_cost_per_task_usd": 0.041120762148337583, "avg_duration_per_task_ms": 136841.38746803068, "avg_output_tokens_per_task": 28739.403452685423, "last_run_timestamp": "2026-10-31T16:08:08.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.81481475, "accuracy_std_dev": 0.005455009183081147, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006102087121212121, "avg_duration_per_task_ms": 532828.8005050505, "avg_output_tokens_per_task": 18519.08712121212, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8147523703703704, "accuracy_std_dev": 0.01034211180579509, "run_count": 27, "total_questions": 5346, "avg_cost_per_task_usd": 0.3634217265245044, "avg_duration_per_task_ms": 358529.47362514026, "avg_output_tokens_per_task": 22930.12757201646, "last_run_timestamp": "2026-10-31T08:33:07.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8143338571428571, "accuracy_std_dev": 0.014248444761532711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026212479076479073, "avg_duration_per_task_ms": 329590.34992784995, "avg_output_tokens_per_task": 17511.52380952381, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.813973, "accuracy_std_dev": 0.01262599999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04186610353535354, "avg_duration_per_task_ms": 76135.87878787878, "avg_output_tokens_per_task": 27786.689393939392, "last_run_timestamp": "2026-10-01T19:16:58.595Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.8136272825651302, "accuracy_std_dev": 0.008773473238499465, "run_count": 3, "total_questions": 499, "avg_cost_per_task_usd": 0.0068851422845691386, "avg_duration_per_task_ms": 1090136, "avg_output_tokens_per_task": 17404.21643286573, "last_run_timestamp": "2026-08-15T10:05:32.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.813051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.00766168253968254, "avg_duration_per_task_ms": 356938.3703703704, "avg_output_tokens_per_task": 33904.37566137566, "last_run_timestamp": "2026-10-14T18:00:03.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.8127104999999999, "accuracy_std_dev": 0.020528377888912718, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.10173348611111112, "avg_duration_per_task_ms": 574816.9229797979, "avg_output_tokens_per_task": 37826.392676767675, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.8125701666666667, "accuracy_std_dev": 0.050032071865343405, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.20077214393939394, "avg_duration_per_task_ms": 1584443.372053872, "avg_output_tokens_per_task": 47804.85858585859, "last_run_timestamp": "2026-10-31T16:31:54.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.8122894999999999, "accuracy_std_dev": 0.02853281357051911, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.026932902777777778, "avg_duration_per_task_ms": 507868.57196969696, "avg_output_tokens_per_task": 24489.684343434343, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.8120790625, "accuracy_std_dev": 0.01643967442830283, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.3404591770833333, "avg_duration_per_task_ms": 353918.67329545453, "avg_output_tokens_per_task": 22524.086174242424, "last_run_timestamp": "2026-10-31T08:46:10.648Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.812079, "accuracy_std_dev": 0.1279844797709863, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.15244413194444442, "avg_duration_per_task_ms": 1980931.956439394, "avg_output_tokens_per_task": 55523.080808080806, "last_run_timestamp": "2026-10-24T16:14:24.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.812016, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.14135672674418603, "avg_duration_per_task_ms": 305120.88372093026, "avg_output_tokens_per_task": 39843.74418604651, "last_run_timestamp": "2026-08-04T03:11:59.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.8118685, "accuracy_std_dev": 0.0618254994258033, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.042041156565656566, "avg_duration_per_task_ms": 1127766.4457070706, "avg_output_tokens_per_task": 24710.378787878788, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.8116347777777777, "accuracy_std_dev": 0.0244922962662765, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.008957320426487093, "avg_duration_per_task_ms": 563804.7289562289, "avg_output_tokens_per_task": 19164.750841750843, "last_run_timestamp": "2026-10-31T16:08:08.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.810606, "accuracy_std_dev": 0.010546997511140292, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.044063582702020196, "avg_duration_per_task_ms": 62743.707702020205, "avg_output_tokens_per_task": 15119.022095959595, "last_run_timestamp": "2026-10-31T08:43:59.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.8105124444444445, "accuracy_std_dev": 0.0452643157786968, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.23235617452300789, "avg_duration_per_task_ms": 1158864.2744107745, "avg_output_tokens_per_task": 53109.03591470258, "last_run_timestamp": "2026-10-31T16:30:07.021Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.8103538311386522, "accuracy_std_dev": 0.10032075765294651, "run_count": 14, "total_questions": 2582, "avg_cost_per_task_usd": 0.041116779628195196, "avg_duration_per_task_ms": 153458.49147947328, "avg_output_tokens_per_task": 19676.14329976762, "last_run_timestamp": "2026-10-31T16:17:39.770Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.8102452857142856, "accuracy_std_dev": 0.09184708177837814, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.09330289826839828, "avg_duration_per_task_ms": 515911.21789321786, "avg_output_tokens_per_task": 34419.12373737374, "last_run_timestamp": "2026-10-31T16:05:36.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8102317777777777, "accuracy_std_dev": 0.01704457789821722, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.22739129012345682, "avg_duration_per_task_ms": 1030818.7098765433, "avg_output_tokens_per_task": 61856.86588103255, "last_run_timestamp": "2026-10-31T16:12:28.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.8097645, "accuracy_std_dev": 0.021885500000000002, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.044968992424242425, "avg_duration_per_task_ms": 208495.61616161617, "avg_output_tokens_per_task": 22388.290404040403, "last_run_timestamp": "2026-10-01T19:19:28.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Particle AI", "accuracy": 0.809764, "accuracy_std_dev": 0.007273551310513057, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.027726367003367003, "avg_duration_per_task_ms": 248023.59595959596, "avg_output_tokens_per_task": 55228.13468013468, "last_run_timestamp": "2026-10-31T08:07:51.513Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "PrimeIntellect", "accuracy": 0.8096608426103648, "accuracy_std_dev": 0.012817931849127972, "run_count": 3, "total_questions": 521, "avg_cost_per_task_usd": 0.0079749443378119, "avg_duration_per_task_ms": 289711.4932821497, "avg_output_tokens_per_task": 31380.54894433781, "last_run_timestamp": "2026-10-31T16:04:23.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.8095958000000001, "accuracy_std_dev": 0.023247585009200426, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.11788618585858586, "avg_duration_per_task_ms": 776373.9363636364, "avg_output_tokens_per_task": 61313.756565656564, "last_run_timestamp": "2026-10-31T16:10:09.226Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.8094875524100994, "accuracy_std_dev": 0.03366403756751318, "run_count": 7, "total_questions": 1307, "avg_cost_per_task_usd": 0.028843539403213467, "avg_duration_per_task_ms": 492278.9755164499, "avg_output_tokens_per_task": 17132.139250191278, "last_run_timestamp": "2026-10-17T16:03:44.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.8091630714285715, "accuracy_std_dev": 0.01971105683716007, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.34238485173160177, "avg_duration_per_task_ms": 356759.47510822513, "avg_output_tokens_per_task": 22659.93614718615, "last_run_timestamp": "2026-10-31T08:17:29.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8089225, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.15927496717171716, "avg_duration_per_task_ms": 42973.2904040404, "avg_output_tokens_per_task": 2992.459595959596, "last_run_timestamp": "2026-08-15T09:15:35.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8080809999999999, "accuracy_std_dev": 0.032382769067823704, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.026336486111111113, "avg_duration_per_task_ms": 464443.5934343434, "avg_output_tokens_per_task": 21920.722222222223, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.807532574733096, "accuracy_std_dev": 0.023704551349195916, "run_count": 6, "total_questions": 1124, "avg_cost_per_task_usd": 0.039211227758007124, "avg_duration_per_task_ms": 252786.0587188612, "avg_output_tokens_per_task": 39158.109430604985, "last_run_timestamp": "2026-10-31T16:09:44.822Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.8068179999999999, "accuracy_std_dev": 0.027363478050131, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.046258910353535354, "avg_duration_per_task_ms": 196125.59974747474, "avg_output_tokens_per_task": 21946.10606060606, "last_run_timestamp": "2026-10-01T19:34:33.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.8063973333333333, "accuracy_std_dev": 0.024015690667191354, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.2794591790123457, "avg_duration_per_task_ms": 997160.2497194164, "avg_output_tokens_per_task": 63521.387205387204, "last_run_timestamp": "2026-10-31T16:12:28.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.8063972, "accuracy_std_dev": 0.10239770597117887, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.05410009898989898, "avg_duration_per_task_ms": 1231927.9055555556, "avg_output_tokens_per_task": 29021.94191919192, "last_run_timestamp": "2026-10-31T16:05:14.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.806397, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04417995707070707, "avg_duration_per_task_ms": 210130.46464646465, "avg_output_tokens_per_task": 21993.772727272728, "last_run_timestamp": "2026-10-01T19:26:25.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.8059763749999999, "accuracy_std_dev": 0.01367028747811745, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.022531703914141412, "avg_duration_per_task_ms": 251092.985479798, "avg_output_tokens_per_task": 17458.415404040403, "last_run_timestamp": "2026-10-03T09:13:36.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.8047136666666667, "accuracy_std_dev": 0.0407068763642377, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02657082996632996, "avg_duration_per_task_ms": 316645.5987654321, "avg_output_tokens_per_task": 25739.34006734007, "last_run_timestamp": "2026-10-31T16:05:36.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.8042929999999999, "accuracy_std_dev": 0.028405059672530177, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04403506628787878, "avg_duration_per_task_ms": 84875.49242424243, "avg_output_tokens_per_task": 27183.9375, "last_run_timestamp": "2026-10-01T19:25:58.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.8040673640873017, "accuracy_std_dev": 0.02506085284963421, "run_count": 7, "total_questions": 1008, "avg_cost_per_task_usd": 0.0705810753968254, "avg_duration_per_task_ms": 1609832.9444444445, "avg_output_tokens_per_task": 40293.83829365079, "last_run_timestamp": "2026-10-31T16:05:14.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.8040581192136969, "accuracy_std_dev": 0.06940046297396005, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.19658499112238428, "avg_duration_per_task_ms": 1692732.626506024, "avg_output_tokens_per_task": 45873.13633481294, "last_run_timestamp": "2026-10-31T16:22:14.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8038719166666666, "accuracy_std_dev": 0.01983021612530708, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.3387011611952862, "avg_duration_per_task_ms": 355211.85016835015, "avg_output_tokens_per_task": 22440.643097643097, "last_run_timestamp": "2026-10-31T08:53:23.022Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.8036315714285714, "accuracy_std_dev": 0.0246877462424299, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0294750468975469, "avg_duration_per_task_ms": 317654.27489177487, "avg_output_tokens_per_task": 19508.803030303032, "last_run_timestamp": "2026-10-17T16:09:25.569Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.8028182609489051, "accuracy_std_dev": 0.07654548992882891, "run_count": 9, "total_questions": 1644, "avg_cost_per_task_usd": 0.019102108272506083, "avg_duration_per_task_ms": 309069.2311435523, "avg_output_tokens_per_task": 40420.16605839416, "last_run_timestamp": "2026-10-31T08:33:53.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Mistral", "accuracy": 0.8006073724696356, "accuracy_std_dev": 0.018676167604730915, "run_count": 5, "total_questions": 988, "avg_cost_per_task_usd": 0.026427639676113363, "avg_duration_per_task_ms": 228773.67307692306, "avg_output_tokens_per_task": 52620.16093117409, "last_run_timestamp": "2026-10-31T08:22:47.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.8, "accuracy_std_dev": 0.004689918080308032, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.013948154545454547, "avg_duration_per_task_ms": 321232.3868686869, "avg_output_tokens_per_task": 14312.276767676767, "last_run_timestamp": "2026-10-04T00:00:36.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.7990709066202091, "accuracy_std_dev": 0.036071343734006646, "run_count": 11, "total_questions": 1435, "avg_cost_per_task_usd": 0.00675192613240418, "avg_duration_per_task_ms": 310615.30383275263, "avg_output_tokens_per_task": 26833.981184668988, "last_run_timestamp": "2026-10-31T16:04:29.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7981739230769231, "accuracy_std_dev": 0.03994574297037949, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.043110703574203574, "avg_duration_per_task_ms": 487860.4627039627, "avg_output_tokens_per_task": 42588.88306138306, "last_run_timestamp": "2026-10-31T16:13:09.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Io Net", "accuracy": 0.7976190000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 98, "avg_cost_per_task_usd": 0.012265704081632654, "avg_duration_per_task_ms": 285854.5612244898, "avg_output_tokens_per_task": 10545.989795918367, "last_run_timestamp": "2026-10-03T17:16:45.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7975339581632652, "accuracy_std_dev": 0.14314198310659945, "run_count": 10, "total_questions": 1960, "avg_cost_per_task_usd": 0.01657647346938776, "avg_duration_per_task_ms": 547902.756122449, "avg_output_tokens_per_task": 26479.933673469386, "last_run_timestamp": "2026-10-17T08:10:43.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.7971601298174443, "accuracy_std_dev": 0.023536053284742144, "run_count": 3, "total_questions": 493, "avg_cost_per_task_usd": 0.04863367951318459, "avg_duration_per_task_ms": 714404.5334685599, "avg_output_tokens_per_task": 21075.50304259635, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.79629625, "accuracy_std_dev": 0.01565761636353056, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.17376329545454547, "avg_duration_per_task_ms": 832021.8737373737, "avg_output_tokens_per_task": 55481.57828282828, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.7962405172932331, "accuracy_std_dev": 0.005792281993642719, "run_count": 7, "total_questions": 1330, "avg_cost_per_task_usd": 0.050177010526315784, "avg_duration_per_task_ms": 1098597.4812030075, "avg_output_tokens_per_task": 22847.669172932332, "last_run_timestamp": "2026-10-17T16:06:10.469Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.7958755, "accuracy_std_dev": 0.00853230152127785, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.014628411616161617, "avg_duration_per_task_ms": 291373.6123737374, "avg_output_tokens_per_task": 13989.849747474747, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.795735, "accuracy_std_dev": 0.01152797383180008, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0539772760942761, "avg_duration_per_task_ms": 405752.09764309763, "avg_output_tokens_per_task": 23428.74074074074, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.7955478888888888, "accuracy_std_dev": 0.030696558538356792, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.039506446689113364, "avg_duration_per_task_ms": 192749.99270482603, "avg_output_tokens_per_task": 21847.699214365883, "last_run_timestamp": "2026-10-31T16:08:31.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7955151838886546, "accuracy_std_dev": 0.021167952217358864, "run_count": 12, "total_questions": 2371, "avg_cost_per_task_usd": 0.0645802066638549, "avg_duration_per_task_ms": 194462.45592576973, "avg_output_tokens_per_task": 20212.99072121468, "last_run_timestamp": "2026-10-31T08:22:42.983Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.7933882727272729, "accuracy_std_dev": 0.01128387116915042, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0045943420569329665, "avg_duration_per_task_ms": 250886.0945821855, "avg_output_tokens_per_task": 30080.480257116622, "last_run_timestamp": "2026-10-17T16:03:34.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.79335, "accuracy_std_dev": 0.008448975529613064, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013479858585858586, "avg_duration_per_task_ms": 277514.2765151515, "avg_output_tokens_per_task": 14053.487373737375, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.792742111111111, "accuracy_std_dev": 0.011968486588114497, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.017912929292929293, "avg_duration_per_task_ms": 62394.14758698092, "avg_output_tokens_per_task": 5816.072390572391, "last_run_timestamp": "2026-10-31T16:10:36.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7925423333333335, "accuracy_std_dev": 0.014817180897974928, "run_count": 13, "total_questions": 2427, "avg_cost_per_task_usd": 0.004677955088586733, "avg_duration_per_task_ms": 1415043.368355995, "avg_output_tokens_per_task": 30720.006180469714, "last_run_timestamp": "2026-10-31T16:08:08.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7923681111111112, "accuracy_std_dev": 0.042678175464605506, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1784357115600449, "avg_duration_per_task_ms": 555057.9292929292, "avg_output_tokens_per_task": 55554.39898989899, "last_run_timestamp": "2026-10-31T16:15:54.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7920900579268293, "accuracy_std_dev": 0.011512971285033247, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.01824498475609756, "avg_duration_per_task_ms": 181543.2469512195, "avg_output_tokens_per_task": 15051.83943089431, "last_run_timestamp": "2026-10-08T22:29:11.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.7916666329787234, "accuracy_std_dev": 0.015009500000000009, "run_count": 2, "total_questions": 188, "avg_cost_per_task_usd": 0.0925983085106383, "avg_duration_per_task_ms": 133806.4414893617, "avg_output_tokens_per_task": 38480.17021276596, "last_run_timestamp": "2026-08-15T14:54:40.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.7916664999999999, "accuracy_std_dev": 0.0223775773990394, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006407165404040405, "avg_duration_per_task_ms": 353276.53661616164, "avg_output_tokens_per_task": 15555.522727272728, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.7912072841391009, "accuracy_std_dev": 0.007680330288036654, "run_count": 6, "total_questions": 1179, "avg_cost_per_task_usd": 0.012160957591178965, "avg_duration_per_task_ms": 774485.8049194232, "avg_output_tokens_per_task": 40209.361323155215, "last_run_timestamp": "2026-10-31T16:11:17.542Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7893355839160839, "accuracy_std_dev": 0.10239971274711347, "run_count": 9, "total_questions": 1716, "avg_cost_per_task_usd": 0.07681546794871794, "avg_duration_per_task_ms": 733179.0168997669, "avg_output_tokens_per_task": 27895.818764568765, "last_run_timestamp": "2026-10-10T16:29:40.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.7888407857142856, "accuracy_std_dev": 0.03143577289644615, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.10358753607503608, "avg_duration_per_task_ms": 373041.1764069264, "avg_output_tokens_per_task": 27977.449855699855, "last_run_timestamp": "2026-10-31T16:09:41.959Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.7876364502435629, "accuracy_std_dev": 0.019910444432115912, "run_count": 8, "total_questions": 1437, "avg_cost_per_task_usd": 0.04065875156576201, "avg_duration_per_task_ms": 745848.8225469729, "avg_output_tokens_per_task": 18787.501043841337, "last_run_timestamp": "2026-10-17T16:06:15.029Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7874268851655088, "accuracy_std_dev": 0.015558291548790305, "run_count": 13, "total_questions": 2447, "avg_cost_per_task_usd": 0.008662264814058032, "avg_duration_per_task_ms": 377262.1029832448, "avg_output_tokens_per_task": 34212.00081732734, "last_run_timestamp": "2026-10-31T16:15:26.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "PrimeIntellect", "accuracy": 0.7872267663934426, "accuracy_std_dev": 0.013963506030602306, "run_count": 3, "total_questions": 488, "avg_cost_per_task_usd": 0.008632290983606556, "avg_duration_per_task_ms": 586832.4200819673, "avg_output_tokens_per_task": 37091.375, "last_run_timestamp": "2026-10-31T16:20:33.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.787037, "accuracy_std_dev": 0.014506239226622442, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.028991700757575758, "avg_duration_per_task_ms": 504207.9595959596, "avg_output_tokens_per_task": 24268.680555555555, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.7865693333333335, "accuracy_std_dev": 0.009968547225036253, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02101957912457912, "avg_duration_per_task_ms": 246528.65207631874, "avg_output_tokens_per_task": 12493.957351290685, "last_run_timestamp": "2026-10-31T16:10:36.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.786195, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2643429696969697, "avg_duration_per_task_ms": 180778.88888888888, "avg_output_tokens_per_task": 17428.555555555555, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7854672791234141, "accuracy_std_dev": 0.12208478471357521, "run_count": 5, "total_questions": 867, "avg_cost_per_task_usd": 0.023352209919261825, "avg_duration_per_task_ms": 590708.8500576701, "avg_output_tokens_per_task": 19635.643598615916, "last_run_timestamp": "2026-10-17T08:18:07.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.7853535, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01384800505050505, "avg_duration_per_task_ms": 544619.0151515151, "avg_output_tokens_per_task": 14426.747474747475, "last_run_timestamp": "2026-08-15T14:56:55.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.7853107987288136, "accuracy_std_dev": 0.021945448081549836, "run_count": 4, "total_questions": 472, "avg_cost_per_task_usd": 0.03487327542372881, "avg_duration_per_task_ms": 120089.44067796611, "avg_output_tokens_per_task": 36221.95338983051, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Nebius", "accuracy": 0.78493275, "accuracy_std_dev": 0.010934561842501992, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03407076388888889, "avg_duration_per_task_ms": 483005.77398989897, "avg_output_tokens_per_task": 67906.58964646465, "last_run_timestamp": "2026-10-31T08:31:13.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.7848860000000001, "accuracy_std_dev": 0.020676163312697362, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.31222273961840624, "avg_duration_per_task_ms": 978633.652637486, "avg_output_tokens_per_task": 78687.65488215488, "last_run_timestamp": "2026-10-31T16:04:02.366Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.7837654143365532, "accuracy_std_dev": 0.02906493883169204, "run_count": 10, "total_questions": 1967, "avg_cost_per_task_usd": 0.011243548551093035, "avg_duration_per_task_ms": 599323.9059481444, "avg_output_tokens_per_task": 37234.770716827654, "last_run_timestamp": "2026-10-31T16:03:21.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.7834595000000001, "accuracy_std_dev": 0.025125138984690213, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0537799494949495, "avg_duration_per_task_ms": 455539.1212121212, "avg_output_tokens_per_task": 23376.137626262625, "last_run_timestamp": "2026-10-17T16:03:34.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7832240326797386, "accuracy_std_dev": 0.07801749999999996, "run_count": 2, "total_questions": 153, "avg_cost_per_task_usd": 0.07358333333333333, "avg_duration_per_task_ms": 500220.6274509804, "avg_output_tokens_per_task": 27733.385620915033, "last_run_timestamp": "2026-08-15T15:24:07.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7814084898477156, "accuracy_std_dev": 0.060666856050972155, "run_count": 8, "total_questions": 1576, "avg_cost_per_task_usd": 0.007266644670050761, "avg_duration_per_task_ms": 867252.3737309645, "avg_output_tokens_per_task": 18458.295050761422, "last_run_timestamp": "2026-10-24T16:21:54.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7803449165517241, "accuracy_std_dev": 0.016592666104035245, "run_count": 10, "total_questions": 1450, "avg_cost_per_task_usd": 0.04745455310344828, "avg_duration_per_task_ms": 981137.0944827587, "avg_output_tokens_per_task": 24061.69103448276, "last_run_timestamp": "2026-10-31T16:04:29.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.780302875, "accuracy_std_dev": 0.044334001808537145, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.012861900883838383, "avg_duration_per_task_ms": 776646.4172979798, "avg_output_tokens_per_task": 38634.71654040404, "last_run_timestamp": "2026-10-17T08:18:06.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.779785, "accuracy_std_dev": 0.12136469148745921, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.017734758352758354, "avg_duration_per_task_ms": 640150.5850815851, "avg_output_tokens_per_task": 32813.10295260295, "last_run_timestamp": "2026-10-31T08:48:05.270Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7792768229342326, "accuracy_std_dev": 0.05369266686739757, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.159092970207982, "avg_duration_per_task_ms": 948891.7240022485, "avg_output_tokens_per_task": 66199.51714446319, "last_run_timestamp": "2026-10-31T16:04:51.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.77925075, "accuracy_std_dev": 0.01980242038710168, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02103724936868687, "avg_duration_per_task_ms": 475607.264520202, "avg_output_tokens_per_task": 27969.91224747475, "last_run_timestamp": "2026-10-31T16:16:35.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7786195454545455, "accuracy_std_dev": 0.04555204390657037, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.21966554178145087, "avg_duration_per_task_ms": 867448.9150596878, "avg_output_tokens_per_task": 73058.02662993572, "last_run_timestamp": "2026-10-17T16:06:46.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.7784476091370558, "accuracy_std_dev": 0.018688411382040342, "run_count": 8, "total_questions": 1576, "avg_cost_per_task_usd": 0.022516517131979694, "avg_duration_per_task_ms": 871719.5628172589, "avg_output_tokens_per_task": 38655.122461928935, "last_run_timestamp": "2026-10-24T16:37:21.981Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.776936, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013052893939393939, "avg_duration_per_task_ms": 64108.87373737374, "avg_output_tokens_per_task": 10319.823232323231, "last_run_timestamp": "2026-10-01T19:12:45.924Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7763531538461537, "accuracy_std_dev": 0.018975981644198205, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.005880460761460762, "avg_duration_per_task_ms": 664693.4401709401, "avg_output_tokens_per_task": 36578.56293706294, "last_run_timestamp": "2026-10-31T16:03:21.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.7760940000000002, "accuracy_std_dev": 0.012598160421267844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013424281144781143, "avg_duration_per_task_ms": 64995.4595959596, "avg_output_tokens_per_task": 10615.195286195287, "last_run_timestamp": "2026-10-02T23:20:38.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.7759257999999999, "accuracy_std_dev": 0.12807761483865945, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.007387053030303032, "avg_duration_per_task_ms": 370638.1904040404, "avg_output_tokens_per_task": 23199.052525252526, "last_run_timestamp": "2026-10-31T16:19:28.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7754896779268858, "accuracy_std_dev": 0.1073162914081702, "run_count": 11, "total_questions": 2161, "avg_cost_per_task_usd": 0.1401641702915317, "avg_duration_per_task_ms": 835967.7473391949, "avg_output_tokens_per_task": 38050.62887552059, "last_run_timestamp": "2026-10-31T17:00:44.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7752525, "accuracy_std_dev": 0.02576620138055538, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.011368148148148148, "avg_duration_per_task_ms": 910071.1422558923, "avg_output_tokens_per_task": 43361.55303030303, "last_run_timestamp": "2026-10-31T09:00:40.586Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.7744107142857142, "accuracy_std_dev": 0.007947545348170354, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005179176046176046, "avg_duration_per_task_ms": 801836.0988455989, "avg_output_tokens_per_task": 28610.891053391053, "last_run_timestamp": "2026-10-31T16:17:18.501Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7741524627118646, "accuracy_std_dev": 0.024920321732780762, "run_count": 7, "total_questions": 1180, "avg_cost_per_task_usd": 0.010626239830508475, "avg_duration_per_task_ms": 1250036.8076271187, "avg_output_tokens_per_task": 40411.0593220339, "last_run_timestamp": "2026-10-31T08:17:43.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7741139703903096, "accuracy_std_dev": 0.09718478875928833, "run_count": 4, "total_questions": 743, "avg_cost_per_task_usd": 0.004475604306864065, "avg_duration_per_task_ms": 312870.95827725437, "avg_output_tokens_per_task": 22926.07267833109, "last_run_timestamp": "2026-10-08T22:27:01.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7736031459537571, "accuracy_std_dev": 0.07146005981537049, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.01837689523121387, "avg_duration_per_task_ms": 1270075.3273121386, "avg_output_tokens_per_task": 17747.31430635838, "last_run_timestamp": "2026-10-31T16:45:39.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.7734006, "accuracy_std_dev": 0.005280843894681953, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.014113778787878787, "avg_duration_per_task_ms": 123713.42222222222, "avg_output_tokens_per_task": 10542.545454545454, "last_run_timestamp": "2026-10-01T19:20:04.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.7732982293193716, "accuracy_std_dev": 0.013698592432801259, "run_count": 5, "total_questions": 955, "avg_cost_per_task_usd": 0.00232973612565445, "avg_duration_per_task_ms": 792784.167539267, "avg_output_tokens_per_task": 15348.367539267016, "last_run_timestamp": "2026-10-31T16:10:36.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7732351365494685, "accuracy_std_dev": 0.025060485373108846, "run_count": 9, "total_questions": 1223, "avg_cost_per_task_usd": 0.013341370400654133, "avg_duration_per_task_ms": 54198.84873262469, "avg_output_tokens_per_task": 17681.87653311529, "last_run_timestamp": "2026-10-31T16:39:49.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7732082857142858, "accuracy_std_dev": 0.007069241982283623, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.020289174603174608, "avg_duration_per_task_ms": 740766.9199134199, "avg_output_tokens_per_task": 50513.87373737374, "last_run_timestamp": "2026-10-31T16:23:19.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.7722054594731511, "accuracy_std_dev": 0.007712250974261645, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.011376429078014186, "avg_duration_per_task_ms": 917720.4442755826, "avg_output_tokens_per_task": 11827.906788247214, "last_run_timestamp": "2026-10-31T16:02:14.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "CoreWeave", "accuracy": 0.7714860192771085, "accuracy_std_dev": 0.016470310033005864, "run_count": 6, "total_questions": 830, "avg_cost_per_task_usd": 0.009284253012048193, "avg_duration_per_task_ms": 720483.7698795181, "avg_output_tokens_per_task": 30820.519277108433, "last_run_timestamp": "2026-10-31T16:24:40.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.7713804000000001, "accuracy_std_dev": 0.01747606822600553, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.014499158585858585, "avg_duration_per_task_ms": 1080917.518181818, "avg_output_tokens_per_task": 51478.64141414141, "last_run_timestamp": "2026-10-31T08:09:31.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.7704826666666666, "accuracy_std_dev": 0.023421454841424542, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0568129494949495, "avg_duration_per_task_ms": 614174.2895622896, "avg_output_tokens_per_task": 22780.829966329966, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.7704203757292881, "accuracy_std_dev": 0.015785967573985243, "run_count": 9, "total_questions": 1714, "avg_cost_per_task_usd": 0.02183358868144691, "avg_duration_per_task_ms": 237188.9049008168, "avg_output_tokens_per_task": 29025.875729288215, "last_run_timestamp": "2026-10-31T16:18:04.835Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.7702020714285713, "accuracy_std_dev": 0.010195756445448158, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.01475207287157287, "avg_duration_per_task_ms": 141322.34632034632, "avg_output_tokens_per_task": 29381.38961038961, "last_run_timestamp": "2026-10-31T16:51:23.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Sail Research", "accuracy": 0.7698409107142858, "accuracy_std_dev": 0.0035135000000000027, "run_count": 2, "total_questions": 336, "avg_cost_per_task_usd": 0.012509422619047618, "avg_duration_per_task_ms": 211684.1994047619, "avg_output_tokens_per_task": 31185.72619047619, "last_run_timestamp": "2026-10-31T16:53:46.692Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.7694625413724936, "accuracy_std_dev": 0.015728262890591427, "run_count": 18, "total_questions": 3541, "avg_cost_per_task_usd": 0.006914428410053658, "avg_duration_per_task_ms": 433264.68483479245, "avg_output_tokens_per_task": 28218.570742728043, "last_run_timestamp": "2026-10-31T16:02:14.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.7692009872315728, "accuracy_std_dev": 0.009952898819619341, "run_count": 9, "total_questions": 1723, "avg_cost_per_task_usd": 0.024908178757980265, "avg_duration_per_task_ms": 73252.19210679048, "avg_output_tokens_per_task": 26085.17933836332, "last_run_timestamp": "2026-10-31T16:06:00.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.7686387142857142, "accuracy_std_dev": 0.0359817938793428, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.05187446248196248, "avg_duration_per_task_ms": 873111.9227994228, "avg_output_tokens_per_task": 23166.116883116883, "last_run_timestamp": "2026-10-17T16:03:24.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7683982261904762, "accuracy_std_dev": 0.037461158230359067, "run_count": 10, "total_questions": 1848, "avg_cost_per_task_usd": 0.0074100124458874456, "avg_duration_per_task_ms": 907022.0817099568, "avg_output_tokens_per_task": 19100.03896103896, "last_run_timestamp": "2026-10-31T16:04:29.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.7669798106936416, "accuracy_std_dev": 0.011686708686296891, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.019905706647398845, "avg_duration_per_task_ms": 1154492.3439306358, "avg_output_tokens_per_task": 60181.387283236996, "last_run_timestamp": "2026-10-31T16:03:21.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7669488305084746, "accuracy_std_dev": 0.016365499999999977, "run_count": 2, "total_questions": 354, "avg_cost_per_task_usd": 0.21194037853107345, "avg_duration_per_task_ms": 159764.62146892655, "avg_output_tokens_per_task": 13982.706214689266, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.057140755050505046, "avg_duration_per_task_ms": 101024.92676767676, "avg_output_tokens_per_task": 22766.989898989897, "last_run_timestamp": "2026-10-01T19:11:42.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7659344612403101, "accuracy_std_dev": 0.01074604027072297, "run_count": 4, "total_questions": 774, "avg_cost_per_task_usd": 0.004193177002583979, "avg_duration_per_task_ms": 4103.266149870801, "avg_output_tokens_per_task": 2323.702842377261, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.7657183855579869, "accuracy_std_dev": 0.13627692230884958, "run_count": 12, "total_questions": 2285, "avg_cost_per_task_usd": 0.10443865426695843, "avg_duration_per_task_ms": 680524.7737417943, "avg_output_tokens_per_task": 29927.162800875274, "last_run_timestamp": "2026-10-31T16:40:50.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.7656566, "accuracy_std_dev": 0.0131009148474448, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.01855774343434343, "avg_duration_per_task_ms": 128486.40656565657, "avg_output_tokens_per_task": 30699.099494949496, "last_run_timestamp": "2026-10-31T16:52:10.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7653769053571428, "accuracy_std_dev": 0.011846747415441054, "run_count": 17, "total_questions": 3360, "avg_cost_per_task_usd": 0.016902593452380953, "avg_duration_per_task_ms": 110624.59017857142, "avg_output_tokens_per_task": 5988.1654761904765, "last_run_timestamp": "2026-10-31T09:06:38.813Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.76536225, "accuracy_std_dev": 0.00536920838555365, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004458252525252525, "avg_duration_per_task_ms": 487032.047979798, "avg_output_tokens_per_task": 29595.234848484848, "last_run_timestamp": "2026-10-31T16:16:55.698Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7647332578268877, "accuracy_std_dev": 0.03506906370263671, "run_count": 7, "total_questions": 1086, "avg_cost_per_task_usd": 0.06476073848987109, "avg_duration_per_task_ms": 751502.6731123389, "avg_output_tokens_per_task": 52466.43278084714, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7645579698795182, "accuracy_std_dev": 0.052301214506824685, "run_count": 3, "total_questions": 332, "avg_cost_per_task_usd": 0.11985862349397591, "avg_duration_per_task_ms": 1173733.4337349397, "avg_output_tokens_per_task": 34138.882530120485, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7643096666666668, "accuracy_std_dev": 0.011195306675368749, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.021048479236812575, "avg_duration_per_task_ms": 1070472.4147025815, "avg_output_tokens_per_task": 52516.912457912455, "last_run_timestamp": "2026-10-31T16:51:21.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7636362999999998, "accuracy_std_dev": 0.129902922031069, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.029436740404040405, "avg_duration_per_task_ms": 136141.02676767675, "avg_output_tokens_per_task": 22545.73888888889, "last_run_timestamp": "2026-10-31T16:08:08.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.7636309629005059, "accuracy_std_dev": 0.012408560333719428, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.02061922428330523, "avg_duration_per_task_ms": 132670.10623946038, "avg_output_tokens_per_task": 11913.527824620573, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.7632998, "accuracy_std_dev": 0.007030464092789305, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007469111111111111, "avg_duration_per_task_ms": 123178.52121212121, "avg_output_tokens_per_task": 29681.70909090909, "last_run_timestamp": "2026-10-31T16:38:46.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7629069999999999, "accuracy_std_dev": 0.00819966182229484, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2718667558922559, "avg_duration_per_task_ms": 264601.48148148146, "avg_output_tokens_per_task": 17934.003367003366, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.7627229007633587, "accuracy_std_dev": 0.011904999999999999, "run_count": 2, "total_questions": 262, "avg_cost_per_task_usd": 0.003081797709923664, "avg_duration_per_task_ms": 72385.01526717558, "avg_output_tokens_per_task": 3744.9809160305344, "last_run_timestamp": "2026-08-15T14:54:48.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.7626260000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022027883838383836, "avg_duration_per_task_ms": 90904.42929292929, "avg_output_tokens_per_task": 14522.767676767677, "last_run_timestamp": "2026-10-04T03:02:47.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7626259999999999, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01717197474747475, "avg_duration_per_task_ms": 622307.4974747475, "avg_output_tokens_per_task": 61038.34090909091, "last_run_timestamp": "2026-10-03T10:05:09.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.7622521111111111, "accuracy_std_dev": 0.011328776902153444, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.01744060213243547, "avg_duration_per_task_ms": 72362.74859708193, "avg_output_tokens_per_task": 28828.154320987655, "last_run_timestamp": "2026-10-31T16:23:59.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7620651111111112, "accuracy_std_dev": 0.013326932508974654, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.018258098765432097, "avg_duration_per_task_ms": 148659.74102132436, "avg_output_tokens_per_task": 30202.418630751963, "last_run_timestamp": "2026-10-31T16:08:33.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.76094275, "accuracy_std_dev": 0.01636554188493312, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05637325126262625, "avg_duration_per_task_ms": 580765.970959596, "avg_output_tokens_per_task": 26018.467171717173, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7605754720812183, "accuracy_std_dev": 0.010447569992586773, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.0025078527918781725, "avg_duration_per_task_ms": 100160.06345177664, "avg_output_tokens_per_task": 31388.34771573604, "last_run_timestamp": "2026-10-01T16:04:09.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.760101, "accuracy_std_dev": 0.012052078264764132, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.016351531565656566, "avg_duration_per_task_ms": 481999.97601010103, "avg_output_tokens_per_task": 16387.108585858587, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7585687491525424, "accuracy_std_dev": 0.0142815403075283, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.016968081355932207, "avg_duration_per_task_ms": 256703.14350282485, "avg_output_tokens_per_task": 28488.8406779661, "last_run_timestamp": "2026-10-31T16:22:37.563Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.7584460168918918, "accuracy_std_dev": 0.03053004550509561, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.00475450731981982, "avg_duration_per_task_ms": 646469.7742117117, "avg_output_tokens_per_task": 28382.398085585584, "last_run_timestamp": "2026-10-31T16:20:12.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7574711434482759, "accuracy_std_dev": 0.05835625158448523, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.014400611954022986, "avg_duration_per_task_ms": 105760.68091954023, "avg_output_tokens_per_task": 6123.435402298851, "last_run_timestamp": "2026-10-31T08:19:36.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.7572390000000001, "accuracy_std_dev": 0.007014128456194694, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.022558349494949493, "avg_duration_per_task_ms": 96018.14343434344, "avg_output_tokens_per_task": 14878.855555555556, "last_run_timestamp": "2026-10-03T16:56:57.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7564585415688118, "accuracy_std_dev": 0.10021722464025354, "run_count": 11, "total_questions": 2129, "avg_cost_per_task_usd": 0.006281726632221701, "avg_duration_per_task_ms": 548745.0742132457, "avg_output_tokens_per_task": 26811.618130577735, "last_run_timestamp": "2026-10-27T02:37:18.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7563452081218275, "accuracy_std_dev": 0.0358022938821275, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.014391308798646361, "avg_duration_per_task_ms": 1040556.6945854484, "avg_output_tokens_per_task": 51071.41201353638, "last_run_timestamp": "2026-10-31T08:41:02.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7558922222222222, "accuracy_std_dev": 0.040404060185904256, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.05619586251402918, "avg_duration_per_task_ms": 1048664.5718294051, "avg_output_tokens_per_task": 28923.416386083052, "last_run_timestamp": "2026-10-31T16:08:31.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.755892, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018171858585858586, "avg_duration_per_task_ms": 285760.0353535354, "avg_output_tokens_per_task": 24027.954545454544, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7547698333333334, "accuracy_std_dev": 0.0766049855914889, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004288400673400673, "avg_duration_per_task_ms": 447675.2971380471, "avg_output_tokens_per_task": 15291.021885521886, "last_run_timestamp": "2026-10-17T08:26:03.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7539042317380352, "accuracy_std_dev": 0.03122595132646972, "run_count": 12, "total_questions": 1985, "avg_cost_per_task_usd": 0.045778590428211585, "avg_duration_per_task_ms": 263292.8277078086, "avg_output_tokens_per_task": 17254.131486146096, "last_run_timestamp": "2026-10-31T08:39:08.315Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.7525250000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008515843434343435, "avg_duration_per_task_ms": 110884.25757575757, "avg_output_tokens_per_task": 8312.732323232323, "last_run_timestamp": "2026-10-04T03:06:11.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7517599999999999, "accuracy_std_dev": 0.013833343768524714, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.01652851698806244, "avg_duration_per_task_ms": 1314711.4719926538, "avg_output_tokens_per_task": 54869.09182736455, "last_run_timestamp": "2026-10-31T16:03:21.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7514029999999999, "accuracy_std_dev": 0.013279676389129373, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015289352693602695, "avg_duration_per_task_ms": 350945.7154882155, "avg_output_tokens_per_task": 62534.38636363636, "last_run_timestamp": "2026-10-31T16:38:47.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.7510100999999999, "accuracy_std_dev": 0.032456701001334076, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.3156345363636364, "avg_duration_per_task_ms": 917451.5242424243, "avg_output_tokens_per_task": 80246.15656565657, "last_run_timestamp": "2026-10-31T16:39:49.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7508415000000002, "accuracy_std_dev": 0.0622895, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06288509848484848, "avg_duration_per_task_ms": 493754.3712121212, "avg_output_tokens_per_task": 26228.825757575756, "last_run_timestamp": "2026-08-15T10:07:37.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.7502806666666667, "accuracy_std_dev": 0.02200747420511694, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.07882791077441077, "avg_duration_per_task_ms": 825090.7154882154, "avg_output_tokens_per_task": 22562.79292929293, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.75, "accuracy_std_dev": 0.00985231011827516, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013619438552188553, "avg_duration_per_task_ms": 95832.97558922559, "avg_output_tokens_per_task": 14970.221380471381, "last_run_timestamp": "2026-10-03T16:54:39.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7495323333333332, "accuracy_std_dev": 0.02683967111903158, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1896214567901235, "avg_duration_per_task_ms": 1077261.9915824917, "avg_output_tokens_per_task": 85826.17845117845, "last_run_timestamp": "2026-10-31T16:01:53.649Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.7487311624365482, "accuracy_std_dev": 0.019042500000000018, "run_count": 2, "total_questions": 394, "avg_cost_per_task_usd": 0.00260698730964467, "avg_duration_per_task_ms": 166191.14467005077, "avg_output_tokens_per_task": 30980.81725888325, "last_run_timestamp": "2026-10-01T16:04:24.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.7481013645569621, "accuracy_std_dev": 0.004003499999999993, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.010130237974683544, "avg_duration_per_task_ms": 298779.6734177215, "avg_output_tokens_per_task": 60721.07341772152, "last_run_timestamp": "2026-10-10T08:39:12.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7469697, "accuracy_std_dev": 0.09552459944438396, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.13509163080808081, "avg_duration_per_task_ms": 945862.1585858585, "avg_output_tokens_per_task": 71261.06515151515, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7465941264367815, "accuracy_std_dev": 0.10532480903970348, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.04816395977011494, "avg_duration_per_task_ms": 468882.4540229885, "avg_output_tokens_per_task": 36470.09642401022, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7465043071718538, "accuracy_std_dev": 0.027768881506803414, "run_count": 6, "total_questions": 739, "avg_cost_per_task_usd": 0.053211952638700946, "avg_duration_per_task_ms": 424838.5182679296, "avg_output_tokens_per_task": 15554.012178619756, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.7464647999999999, "accuracy_std_dev": 0.010410614859843804, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.014634281818181818, "avg_duration_per_task_ms": 108043.59494949495, "avg_output_tokens_per_task": 14531.531313131312, "last_run_timestamp": "2026-10-03T16:43:33.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7462993119730186, "accuracy_std_dev": 0.0800254867207247, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.00489628442945475, "avg_duration_per_task_ms": 596052.9263631253, "avg_output_tokens_per_task": 32164.360314783586, "last_run_timestamp": "2026-10-17T16:03:21.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.7457911666666666, "accuracy_std_dev": 0.008528929132012367, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006685603535353536, "avg_duration_per_task_ms": 166915.6186868687, "avg_output_tokens_per_task": 7768.992424242424, "last_run_timestamp": "2026-10-03T16:58:32.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7450601862113403, "accuracy_std_dev": 0.02284683246884292, "run_count": 8, "total_questions": 1552, "avg_cost_per_task_usd": 0.016163423324742267, "avg_duration_per_task_ms": 377921.4826030928, "avg_output_tokens_per_task": 53593.9587628866, "last_run_timestamp": "2026-10-31T08:07:17.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.7443482857142857, "accuracy_std_dev": 0.006065167898142083, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005301684704184704, "avg_duration_per_task_ms": 153567.91919191918, "avg_output_tokens_per_task": 8565.327561327562, "last_run_timestamp": "2026-10-17T16:06:15.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7437335555555555, "accuracy_std_dev": 0.04255083833646293, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.30186382940516276, "avg_duration_per_task_ms": 1012019.6812570146, "avg_output_tokens_per_task": 89968.04769921437, "last_run_timestamp": "2026-10-31T16:11:15.423Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.7430977999999999, "accuracy_std_dev": 0.008356521127837845, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0232193797979798, "avg_duration_per_task_ms": 98352.36666666667, "avg_output_tokens_per_task": 15371.260606060607, "last_run_timestamp": "2026-10-03T16:58:08.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.7420804924406049, "accuracy_std_dev": 0.0038769124828915004, "run_count": 5, "total_questions": 926, "avg_cost_per_task_usd": 0.008837730021598272, "avg_duration_per_task_ms": 138969.186825054, "avg_output_tokens_per_task": 11241.735421166306, "last_run_timestamp": "2026-10-03T17:03:40.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7413018888888888, "accuracy_std_dev": 0.01318452125658845, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.11432362065095399, "avg_duration_per_task_ms": 753653.6470258137, "avg_output_tokens_per_task": 44622.36812570146, "last_run_timestamp": "2026-10-31T16:04:48.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.7410596900662253, "accuracy_std_dev": 0.04968796474861267, "run_count": 8, "total_questions": 1510, "avg_cost_per_task_usd": 0.017916279470198675, "avg_duration_per_task_ms": 948164.3721854305, "avg_output_tokens_per_task": 45200.233774834436, "last_run_timestamp": "2026-10-31T16:47:37.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.7384959999999999, "accuracy_std_dev": 0.008170584087648423, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06071549915824916, "avg_duration_per_task_ms": 125902.70959595959, "avg_output_tokens_per_task": 24199.07239057239, "last_run_timestamp": "2026-10-01T19:15:31.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.7374669124010552, "accuracy_std_dev": 0.0902272932043027, "run_count": 11, "total_questions": 1895, "avg_cost_per_task_usd": 0.004063712401055408, "avg_duration_per_task_ms": 556408.5973614776, "avg_output_tokens_per_task": 25344.54142480211, "last_run_timestamp": "2026-10-31T08:04:26.691Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.7373190000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 184, "avg_cost_per_task_usd": 0.008365875, "avg_duration_per_task_ms": 223563.88043478262, "avg_output_tokens_per_task": 10361.597826086956, "last_run_timestamp": "2026-08-08T16:46:11.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.7366819680365297, "accuracy_std_dev": 0.008315102174527942, "run_count": 9, "total_questions": 1752, "avg_cost_per_task_usd": 0.006853743150684932, "avg_duration_per_task_ms": 245825.2191780822, "avg_output_tokens_per_task": 8475.880707762557, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.736486429054054, "accuracy_std_dev": 0.05162874176700847, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.3068156143018018, "avg_duration_per_task_ms": 1317780.9831081082, "avg_output_tokens_per_task": 97153.50563063064, "last_run_timestamp": "2026-10-31T16:18:48.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.736027, "accuracy_std_dev": 0.010289969057290673, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004868150505050505, "avg_duration_per_task_ms": 107346.51515151515, "avg_output_tokens_per_task": 7938.19797979798, "last_run_timestamp": "2026-10-03T16:59:36.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.7352164576271186, "accuracy_std_dev": 0.012339799164982321, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.006587220903954802, "avg_duration_per_task_ms": 332459.9661016949, "avg_output_tokens_per_task": 26156.45536723164, "last_run_timestamp": "2026-10-31T16:16:13.798Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7348357476056337, "accuracy_std_dev": 0.05024499959642196, "run_count": 9, "total_questions": 1775, "avg_cost_per_task_usd": 0.3023959695774648, "avg_duration_per_task_ms": 1292289.6828169015, "avg_output_tokens_per_task": 95338.63492957747, "last_run_timestamp": "2026-10-31T16:18:28.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.732440575, "accuracy_std_dev": 0.057246596407316816, "run_count": 3, "total_questions": 560, "avg_cost_per_task_usd": 0.014524144642857144, "avg_duration_per_task_ms": 428029.5660714286, "avg_output_tokens_per_task": 58878.205357142855, "last_run_timestamp": "2026-10-31T08:03:10.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.732155, "accuracy_std_dev": 0.02424727451075689, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.016461675757575758, "avg_duration_per_task_ms": 673272.2666666667, "avg_output_tokens_per_task": 60883.92525252525, "last_run_timestamp": "2026-10-31T08:31:11.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.732043837671728, "accuracy_std_dev": 0.018715369477343178, "run_count": 14, "total_questions": 2766, "avg_cost_per_task_usd": 0.006619636659436008, "avg_duration_per_task_ms": 127243.25451916124, "avg_output_tokens_per_task": 6931.6146059291395, "last_run_timestamp": "2026-10-17T16:03:44.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.731949111111111, "accuracy_std_dev": 0.02440615873296666, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.006272946689113356, "avg_duration_per_task_ms": 384042.4797979798, "avg_output_tokens_per_task": 21207.913580246914, "last_run_timestamp": "2026-10-31T16:08:31.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.7319023750000001, "accuracy_std_dev": 0.006382992557129826, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0060275726010100995, "avg_duration_per_task_ms": 414095.6856060606, "avg_output_tokens_per_task": 7948.38952020202, "last_run_timestamp": "2026-10-31T16:08:56.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.7306920882647944, "accuracy_std_dev": 0.018533662569629808, "run_count": 7, "total_questions": 997, "avg_cost_per_task_usd": 0.016062558676028082, "avg_duration_per_task_ms": 67044.65897693079, "avg_output_tokens_per_task": 9235.0962888666, "last_run_timestamp": "2026-10-31T16:30:53.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.730226631893972, "accuracy_std_dev": 0.11726978279845851, "run_count": 13, "total_questions": 2339, "avg_cost_per_task_usd": 0.03321617400598546, "avg_duration_per_task_ms": 1275412.009833262, "avg_output_tokens_per_task": 34879.14194100043, "last_run_timestamp": "2026-10-31T16:04:48.182Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7300354102564102, "accuracy_std_dev": 0.01294792675952992, "run_count": 39, "total_questions": 7722, "avg_cost_per_task_usd": 0.20437522118622112, "avg_duration_per_task_ms": 238565.11292411294, "avg_output_tokens_per_task": 38135.92411292411, "last_run_timestamp": "2026-10-31T08:33:11.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.7293447692307693, "accuracy_std_dev": 0.008838355156352458, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.19052701903651906, "avg_duration_per_task_ms": 238729.40753690753, "avg_output_tokens_per_task": 37914.3142968143, "last_run_timestamp": "2026-10-31T08:20:09.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7281145, "accuracy_std_dev": 0.004208499999999948, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.060785368686868686, "avg_duration_per_task_ms": 121296.46212121213, "avg_output_tokens_per_task": 24226.65404040404, "last_run_timestamp": "2026-10-01T19:12:41.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.7271808509532062, "accuracy_std_dev": 0.03540222199908109, "run_count": 6, "total_questions": 1154, "avg_cost_per_task_usd": 0.011152869150779895, "avg_duration_per_task_ms": 504501.5831889082, "avg_output_tokens_per_task": 55506.62998266898, "last_run_timestamp": "2026-10-10T08:24:27.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Cohere", "accuracy": 0.7267116666666668, "accuracy_std_dev": 0.01049864519301841, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.021460314814814814, "avg_duration_per_task_ms": 244130.04882154881, "avg_output_tokens_per_task": 76264.67003367003, "last_run_timestamp": "2026-10-31T08:19:36.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.7264309999999999, "accuracy_std_dev": 0.021044000000000007, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1303771111111111, "avg_duration_per_task_ms": 318087.7601010101, "avg_output_tokens_per_task": 43267.416666666664, "last_run_timestamp": "2026-10-04T05:57:29.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7263041800146951, "accuracy_std_dev": 0.01201198916070684, "run_count": 14, "total_questions": 2722, "avg_cost_per_task_usd": 0.1868326627479794, "avg_duration_per_task_ms": 344051.7843497428, "avg_output_tokens_per_task": 37176.1304188097, "last_run_timestamp": "2026-10-31T08:16:11.695Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.7257187692307693, "accuracy_std_dev": 0.010132498721014512, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.18636140015540018, "avg_duration_per_task_ms": 303150.1456876457, "avg_output_tokens_per_task": 37101.67715617715, "last_run_timestamp": "2026-10-31T08:09:30.774Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.7252524, "accuracy_std_dev": 0.016713400846027716, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004543460606060606, "avg_duration_per_task_ms": 167068.73636363636, "avg_output_tokens_per_task": 7707.889898989899, "last_run_timestamp": "2026-10-03T16:57:10.991Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.7250282222222222, "accuracy_std_dev": 0.015036749624967174, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.005513952300785634, "avg_duration_per_task_ms": 307362.290684624, "avg_output_tokens_per_task": 8054.221099887766, "last_run_timestamp": "2026-10-31T16:22:55.771Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.7236893527788727, "accuracy_std_dev": 0.03777209883376474, "run_count": 13, "total_questions": 2537, "avg_cost_per_task_usd": 0.010590881750098544, "avg_duration_per_task_ms": 263440.8253843122, "avg_output_tokens_per_task": 48345.98265668112, "last_run_timestamp": "2026-10-31T08:46:43.680Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.7234044255319149, "accuracy_std_dev": 0.1917994784299478, "run_count": 3, "total_questions": 517, "avg_cost_per_task_usd": 0.033565591876208896, "avg_duration_per_task_ms": 242772.43907156674, "avg_output_tokens_per_task": 37102.30560928433, "last_run_timestamp": "2026-09-29T20:57:20.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.7223064, "accuracy_std_dev": 0.010291729001484623, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.00442305606060606, "avg_duration_per_task_ms": 663677.5994949494, "avg_output_tokens_per_task": 7916.334343434343, "last_run_timestamp": "2026-10-31T16:02:18.363Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.7219408075949366, "accuracy_std_dev": 0.025598999999999983, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.27939689620253166, "avg_duration_per_task_ms": 450249.90379746835, "avg_output_tokens_per_task": 45941.00253164557, "last_run_timestamp": "2026-08-15T08:16:15.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.7212367804878049, "accuracy_std_dev": 0.01324537121083399, "run_count": 41, "total_questions": 8118, "avg_cost_per_task_usd": 0.19729785008622816, "avg_duration_per_task_ms": 270228.84429662477, "avg_output_tokens_per_task": 36826.08869179601, "last_run_timestamp": "2026-10-31T08:29:35.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.7183810211428571, "accuracy_std_dev": 0.051458865728389994, "run_count": 18, "total_questions": 3500, "avg_cost_per_task_usd": 0.008799481142857142, "avg_duration_per_task_ms": 536929.9885714286, "avg_output_tokens_per_task": 26082.768, "last_run_timestamp": "2026-10-31T16:52:55.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7183742857142856, "accuracy_std_dev": 0.012394494661678109, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01898589466089466, "avg_duration_per_task_ms": 501749.6746031746, "avg_output_tokens_per_task": 67465.82683982684, "last_run_timestamp": "2026-10-31T08:06:04.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.7164982000000001, "accuracy_std_dev": 0.007939319804618048, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007827072727272727, "avg_duration_per_task_ms": 80947.86969696969, "avg_output_tokens_per_task": 9940.124242424243, "last_run_timestamp": "2026-10-03T16:40:12.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7140853333333333, "accuracy_std_dev": 0.0635853157908508, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1300334169472503, "avg_duration_per_task_ms": 644759.9270482603, "avg_output_tokens_per_task": 39066.33557800225, "last_run_timestamp": "2026-10-31T16:56:31.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.7138721298568508, "accuracy_std_dev": 0.014796710480373683, "run_count": 5, "total_questions": 978, "avg_cost_per_task_usd": 0.011163598159509203, "avg_duration_per_task_ms": 105524.10429447853, "avg_output_tokens_per_task": 10243.394683026585, "last_run_timestamp": "2026-10-03T16:43:17.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.713468, "accuracy_std_dev": 0.009367361122536085, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010825097979797981, "avg_duration_per_task_ms": 93154.30303030302, "avg_output_tokens_per_task": 10165.817171717172, "last_run_timestamp": "2026-10-03T16:59:33.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Darkbloom", "accuracy": 0.713213, "accuracy_std_dev": null, "run_count": 1, "total_questions": 111, "avg_cost_per_task_usd": 0.007665369369369369, "avg_duration_per_task_ms": 1401051.936936937, "avg_output_tokens_per_task": 27183.8018018018, "last_run_timestamp": "2026-10-03T09:34:00.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7124580000000001, "accuracy_std_dev": 0.005150598062361318, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015565995959595959, "avg_duration_per_task_ms": 101217.36060606061, "avg_output_tokens_per_task": 10296.08787878788, "last_run_timestamp": "2026-10-03T16:46:22.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.7117846000000001, "accuracy_std_dev": 0.007633552268767163, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00877359797979798, "avg_duration_per_task_ms": 133676.15555555557, "avg_output_tokens_per_task": 5686.586868686869, "last_run_timestamp": "2026-10-03T16:45:33.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7112403023255814, "accuracy_std_dev": 0.09623599999999999, "run_count": 2, "total_questions": 344, "avg_cost_per_task_usd": 0.06576875, "avg_duration_per_task_ms": 744084.4970930233, "avg_output_tokens_per_task": 27095.636627906977, "last_run_timestamp": "2026-08-15T09:29:41.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "AkashML", "accuracy": 0.71001675, "accuracy_std_dev": 0.018948683824675006, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015486457070707072, "avg_duration_per_task_ms": 1105788.4797979798, "avg_output_tokens_per_task": 64321.568181818184, "last_run_timestamp": "2026-10-31T16:02:14.361Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.7098763333333333, "accuracy_std_dev": 0.0209969340354464, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0663709898989899, "avg_duration_per_task_ms": 1050299.8484848484, "avg_output_tokens_per_task": 22129.309764309764, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.7077440000000002, "accuracy_std_dev": 0.006867349678005355, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0059367939393939395, "avg_duration_per_task_ms": 109836.06363636363, "avg_output_tokens_per_task": 5507.80101010101, "last_run_timestamp": "2026-10-03T17:02:24.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.707071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008867828282828283, "avg_duration_per_task_ms": 116045.65151515152, "avg_output_tokens_per_task": 5749.39898989899, "last_run_timestamp": "2026-10-04T03:05:59.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.70665, "accuracy_std_dev": 0.05616899659064598, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006621275252525252, "avg_duration_per_task_ms": 125753.64267676767, "avg_output_tokens_per_task": 7346.292929292929, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.7060420000000001, "accuracy_std_dev": 0.04619957164303584, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.015228148709315378, "avg_duration_per_task_ms": 964326.9915824916, "avg_output_tokens_per_task": 49378.965768799106, "last_run_timestamp": "2026-10-31T16:11:46.431Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.7045455, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01336774494949495, "avg_duration_per_task_ms": 231791.09848484848, "avg_output_tokens_per_task": 33323.15909090909, "last_run_timestamp": "2026-10-01T19:27:22.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.7038606610169492, "accuracy_std_dev": 0.025828434379814764, "run_count": 11, "total_questions": 2124, "avg_cost_per_task_usd": 0.009122728342749528, "avg_duration_per_task_ms": 365582.73399246705, "avg_output_tokens_per_task": 45350.33474576271, "last_run_timestamp": "2026-10-31T08:03:09.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.703844, "accuracy_std_dev": 0.13064484273849722, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.611304741161616, "avg_duration_per_task_ms": 637806.0361952862, "avg_output_tokens_per_task": 40663.66961279461, "last_run_timestamp": "2026-10-31T08:38:33.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7037038000000001, "accuracy_std_dev": 0.015684797906253056, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.01112428686868687, "avg_duration_per_task_ms": 196538.2898989899, "avg_output_tokens_per_task": 5791.187878787879, "last_run_timestamp": "2026-10-03T16:59:57.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7032012317327765, "accuracy_std_dev": 0.20342432055346774, "run_count": 5, "total_questions": 958, "avg_cost_per_task_usd": 0.08802485803757831, "avg_duration_per_task_ms": 1349048.8903966597, "avg_output_tokens_per_task": 33413.259916492694, "last_run_timestamp": "2026-10-24T16:13:05.627Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7028622499999999, "accuracy_std_dev": 0.08200476941122087, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0070032777777777775, "avg_duration_per_task_ms": 172461.12373737374, "avg_output_tokens_per_task": 7212.60101010101, "last_run_timestamp": "2026-10-03T16:56:36.098Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.7006736, "accuracy_std_dev": 0.029399119487494893, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010841593939393938, "avg_duration_per_task_ms": 65671.14141414141, "avg_output_tokens_per_task": 8990.147474747475, "last_run_timestamp": "2026-10-03T16:36:16.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.7006460556023589, "accuracy_std_dev": 0.008635764504856131, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.010373346251053074, "avg_duration_per_task_ms": 142035.05391743893, "avg_output_tokens_per_task": 9363.947767481044, "last_run_timestamp": "2026-10-08T22:40:37.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6970787019475021, "accuracy_std_dev": 0.046119854611155975, "run_count": 12, "total_questions": 2362, "avg_cost_per_task_usd": 0.007317635478408128, "avg_duration_per_task_ms": 292695.22861981374, "avg_output_tokens_per_task": 40004.958086367486, "last_run_timestamp": "2026-10-31T08:15:24.314Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.6950760566414041, "accuracy_std_dev": 0.14052251643685132, "run_count": 38, "total_questions": 7521, "avg_cost_per_task_usd": 0.6626253082036964, "avg_duration_per_task_ms": 603719.9066613482, "avg_output_tokens_per_task": 41294.83499534636, "last_run_timestamp": "2026-10-31T08:51:32.081Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6948820649606299, "accuracy_std_dev": 0.027729042227588305, "run_count": 8, "total_questions": 1524, "avg_cost_per_task_usd": 0.015564356299212597, "avg_duration_per_task_ms": 765563.3963254594, "avg_output_tokens_per_task": 77634.76837270342, "last_run_timestamp": "2026-09-26T16:34:21.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6937554455696204, "accuracy_std_dev": 0.09367172425871108, "run_count": 10, "total_questions": 1975, "avg_cost_per_task_usd": 0.011443888607594935, "avg_duration_per_task_ms": 875181.9529113924, "avg_output_tokens_per_task": 33496.65417721519, "last_run_timestamp": "2026-10-31T16:08:08.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6936026666666666, "accuracy_std_dev": 0.010286382189844758, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.005539732323232322, "avg_duration_per_task_ms": 380174.12373737374, "avg_output_tokens_per_task": 6110.575757575758, "last_run_timestamp": "2026-10-08T23:09:15.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.693300564102564, "accuracy_std_dev": 0.1359986435897281, "run_count": 39, "total_questions": 7722, "avg_cost_per_task_usd": 0.6583114172494173, "avg_duration_per_task_ms": 444898.01476301474, "avg_output_tokens_per_task": 40970.9775964776, "last_run_timestamp": "2026-10-31T08:31:32.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.6929553846153845, "accuracy_std_dev": 0.13884156644759268, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.616928526029526, "avg_duration_per_task_ms": 639057.0835275835, "avg_output_tokens_per_task": 40970.16472416472, "last_run_timestamp": "2026-10-31T08:09:30.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.6917897692307693, "accuracy_std_dev": 0.14551018900679394, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.6185969393939393, "avg_duration_per_task_ms": 638891.4533799534, "avg_output_tokens_per_task": 41138.08430458431, "last_run_timestamp": "2026-10-31T08:28:17.671Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.6912458, "accuracy_std_dev": 0.010837313198390075, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0064883984848484845, "avg_duration_per_task_ms": 106966.29242424242, "avg_output_tokens_per_task": 15944.740404040404, "last_run_timestamp": "2026-10-31T16:08:56.904Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.6905724000000001, "accuracy_std_dev": 0.021171864222122725, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.009511277777777778, "avg_duration_per_task_ms": 270838.6898989899, "avg_output_tokens_per_task": 6542.475757575758, "last_run_timestamp": "2026-10-03T17:02:48.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.6898380904522613, "accuracy_std_dev": 0.12509647859063733, "run_count": 9, "total_questions": 1194, "avg_cost_per_task_usd": 0.0919431105527638, "avg_duration_per_task_ms": 1244619.9221105527, "avg_output_tokens_per_task": 26175.759631490786, "last_run_timestamp": "2026-10-31T16:05:14.699Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.689815, "accuracy_std_dev": 0.0034448118235979193, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013852137626262627, "avg_duration_per_task_ms": 234308.42045454544, "avg_output_tokens_per_task": 32893.11994949495, "last_run_timestamp": "2026-10-01T19:27:10.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.687366775928297, "accuracy_std_dev": 0.029137221034831948, "run_count": 7, "total_questions": 781, "avg_cost_per_task_usd": 0.008982548015364916, "avg_duration_per_task_ms": 54182.23815620998, "avg_output_tokens_per_task": 5818.932138284251, "last_run_timestamp": "2026-10-31T16:13:09.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.6866834096080909, "accuracy_std_dev": 0.0076575271261354015, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.007103366624525917, "avg_duration_per_task_ms": 1621893.5347661187, "avg_output_tokens_per_task": 50543.86852085967, "last_run_timestamp": "2026-10-31T16:36:31.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6854461436619719, "accuracy_std_dev": 0.012155576012164115, "run_count": 4, "total_questions": 710, "avg_cost_per_task_usd": 0.040622619718309855, "avg_duration_per_task_ms": 770310.514084507, "avg_output_tokens_per_task": 43146.859154929574, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403:free", "model_slug": null, "model_name": "Gemma 4 26B A4B IT (free)", "provider_name": "Darkbloom", "accuracy": 0.684722, "accuracy_std_dev": null, "run_count": 1, "total_questions": 120, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 382535.98333333334, "avg_output_tokens_per_task": 6994.291666666667, "last_run_timestamp": "2026-08-04T05:36:23.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.6843435000000001, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01356670707070707, "avg_duration_per_task_ms": 229761.702020202, "avg_output_tokens_per_task": 33820.56313131313, "last_run_timestamp": "2026-10-01T19:28:34.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6842701805792164, "accuracy_std_dev": 0.044485547973956784, "run_count": 9, "total_questions": 1761, "avg_cost_per_task_usd": 0.028337780806360024, "avg_duration_per_task_ms": 842767.0045428734, "avg_output_tokens_per_task": 18259.638273708122, "last_run_timestamp": "2026-10-31T16:23:19.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.681502625, "accuracy_std_dev": 0.06815065878246794, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007994105429292929, "avg_duration_per_task_ms": 1440482.744949495, "avg_output_tokens_per_task": 62597.993055555555, "last_run_timestamp": "2026-10-31T16:08:08.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.67979451369863, "accuracy_std_dev": 0.019550333432791698, "run_count": 9, "total_questions": 1752, "avg_cost_per_task_usd": 0.0022822819634703197, "avg_duration_per_task_ms": 207690.84988584474, "avg_output_tokens_per_task": 4088.747716894977, "last_run_timestamp": "2026-10-31T16:18:48.428Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.6785352626262626, "accuracy_std_dev": 0.014156861524116758, "run_count": 13, "total_questions": 1980, "avg_cost_per_task_usd": 0.005069473232323233, "avg_duration_per_task_ms": 756595.5121212121, "avg_output_tokens_per_task": 50528.773232323234, "last_run_timestamp": "2026-10-31T16:04:29.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.677399, "accuracy_std_dev": 0.12205074947844442, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0072654311868686865, "avg_duration_per_task_ms": 728833.754419192, "avg_output_tokens_per_task": 25199.926136363636, "last_run_timestamp": "2026-10-31T16:45:39.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.6726822675159235, "accuracy_std_dev": 0.0862043071017272, "run_count": 3, "total_questions": 471, "avg_cost_per_task_usd": 0.07077279405520169, "avg_duration_per_task_ms": 1745687.9214437366, "avg_output_tokens_per_task": 24749.823779193204, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.6692905358600583, "accuracy_std_dev": 0.017641172479174967, "run_count": 10, "total_questions": 1715, "avg_cost_per_task_usd": 0.00673951195335277, "avg_duration_per_task_ms": 477949.7562682216, "avg_output_tokens_per_task": 47964.45364431487, "last_run_timestamp": "2026-10-31T16:03:21.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.6689113333333334, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0026409175084175086, "avg_duration_per_task_ms": 47935.90572390572, "avg_output_tokens_per_task": 2383.6902356902356, "last_run_timestamp": "2026-10-02T15:53:31.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.6689113333333333, "accuracy_std_dev": 0.016128109050832822, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.005779179573512907, "avg_duration_per_task_ms": 407135.462962963, "avg_output_tokens_per_task": 15838.005050505051, "last_run_timestamp": "2026-10-31T16:17:15.718Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.6687991999999999, "accuracy_std_dev": 0.14462356096164505, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.6179110515151515, "avg_duration_per_task_ms": 635772.7828282828, "avg_output_tokens_per_task": 41062.00437710438, "last_run_timestamp": "2026-10-31T08:22:47.508Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6687465155984118, "accuracy_std_dev": 0.0639810001554281, "run_count": 9, "total_questions": 1763, "avg_cost_per_task_usd": 0.05512392682926829, "avg_duration_per_task_ms": 1146484.0079410097, "avg_output_tokens_per_task": 24612.394781622235, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.6683503999999999, "accuracy_std_dev": 0.046945141186708554, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015881437373737373, "avg_duration_per_task_ms": 296657.8616161616, "avg_output_tokens_per_task": 8706.40404040404, "last_run_timestamp": "2026-10-03T17:36:48.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.66835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013946772727272728, "avg_duration_per_task_ms": 324889.0353535354, "avg_output_tokens_per_task": 15315.242424242424, "last_run_timestamp": "2026-10-01T08:41:30.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6662925555555553, "accuracy_std_dev": 0.007644274285896932, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.005087682379349046, "avg_duration_per_task_ms": 104399.63636363637, "avg_output_tokens_per_task": 9580.892817059485, "last_run_timestamp": "2026-10-03T09:23:08.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6662457499999999, "accuracy_std_dev": 0.017551141520923935, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005718719696969698, "avg_duration_per_task_ms": 566951.7815656565, "avg_output_tokens_per_task": 16218.064393939394, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.6658247500000001, "accuracy_std_dev": 0.0064656044332684175, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0027968030303030305, "avg_duration_per_task_ms": 81352.02777777778, "avg_output_tokens_per_task": 2993.090909090909, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.6649832, "accuracy_std_dev": 0.006981815133616774, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.002071141414141414, "avg_duration_per_task_ms": 228896.86666666667, "avg_output_tokens_per_task": 3033.7636363636366, "last_run_timestamp": "2026-10-05T20:44:48.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.664235, "accuracy_std_dev": 0.019262151293260175, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.007772474747474748, "avg_duration_per_task_ms": 551813.6610549944, "avg_output_tokens_per_task": 59957.01627384961, "last_run_timestamp": "2026-10-31T16:08:31.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6637718184210527, "accuracy_std_dev": 0.20207857374716895, "run_count": 10, "total_questions": 1900, "avg_cost_per_task_usd": 0.012164477368421052, "avg_duration_per_task_ms": 311613.41736842104, "avg_output_tokens_per_task": 14454.243684210527, "last_run_timestamp": "2026-10-17T08:17:57.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.66372075, "accuracy_std_dev": 0.00844903779654822, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00248682702020202, "avg_duration_per_task_ms": 87766.55681818182, "avg_output_tokens_per_task": 2304.1906565656564, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.6633000000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013461267676767677, "avg_duration_per_task_ms": 327894.71717171714, "avg_output_tokens_per_task": 14738.742424242424, "last_run_timestamp": "2026-10-01T08:27:45.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.6627089841688654, "accuracy_std_dev": 0.007909198264520861, "run_count": 4, "total_questions": 379, "avg_cost_per_task_usd": 0.004262828496042217, "avg_duration_per_task_ms": 535749.2849604222, "avg_output_tokens_per_task": 28502.488126649077, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.6623645555555556, "accuracy_std_dev": 0.013998065156863738, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.00905068686868687, "avg_duration_per_task_ms": 323979.22923681256, "avg_output_tokens_per_task": 59978.55218855219, "last_run_timestamp": "2026-10-31T16:08:08.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.6603534999999999, "accuracy_std_dev": 0.021669912920914115, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008152684974747474, "avg_duration_per_task_ms": 795790.2424242424, "avg_output_tokens_per_task": 55529.520833333336, "last_run_timestamp": "2026-10-31T16:08:56.902Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.659933, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014984520202020202, "avg_duration_per_task_ms": 348987.1767676768, "avg_output_tokens_per_task": 16461.863636363636, "last_run_timestamp": "2026-10-01T08:28:22.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6599326666666666, "accuracy_std_dev": 0.013255807590972711, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015927558922558923, "avg_duration_per_task_ms": 14095.8367003367, "avg_output_tokens_per_task": 1794.7289562289561, "last_run_timestamp": "2026-10-02T23:17:48.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.6595585555555555, "accuracy_std_dev": 0.13091400954155713, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0148099898989899, "avg_duration_per_task_ms": 436249.1038159371, "avg_output_tokens_per_task": 12881.502805836139, "last_run_timestamp": "2026-10-31T16:01:53.629Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.658249, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0066776414141414145, "avg_duration_per_task_ms": 101359.9898989899, "avg_output_tokens_per_task": 9307.358585858587, "last_run_timestamp": "2026-10-04T03:07:35.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.65736, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.014864365482233502, "avg_duration_per_task_ms": 1717906.416243655, "avg_output_tokens_per_task": 64415.350253807104, "last_run_timestamp": "2026-10-01T08:48:38.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.65277775, "accuracy_std_dev": 0.007840200328276042, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.003156868686868687, "avg_duration_per_task_ms": 41742.583333333336, "avg_output_tokens_per_task": 2990.8636363636365, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.652357, "accuracy_std_dev": 0.01262599999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004161952020202021, "avg_duration_per_task_ms": 27160.11111111111, "avg_output_tokens_per_task": 2408.8232323232323, "last_run_timestamp": "2026-10-01T19:12:31.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.6523569999999999, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01652235606060606, "avg_duration_per_task_ms": 1917916.494949495, "avg_output_tokens_per_task": 65966.15909090909, "last_run_timestamp": "2026-10-09T00:15:37.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.651515, "accuracy_std_dev": 0.1905456270048375, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.058297676767676776, "avg_duration_per_task_ms": 1098090.5791245792, "avg_output_tokens_per_task": 23174.20202020202, "last_run_timestamp": "2026-10-17T16:03:44.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6492706666666667, "accuracy_std_dev": 0.010498645193018418, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0041127255892255896, "avg_duration_per_task_ms": 24949.132996632998, "avg_output_tokens_per_task": 2375.9343434343436, "last_run_timestamp": "2026-10-02T23:15:53.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6491281532846714, "accuracy_std_dev": 0.20940255131760213, "run_count": 9, "total_questions": 1644, "avg_cost_per_task_usd": 0.00955052798053528, "avg_duration_per_task_ms": 180102.42518248176, "avg_output_tokens_per_task": 15744.736618004867, "last_run_timestamp": "2026-10-31T16:54:17.849Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.6490834444444443, "accuracy_std_dev": 0.02275484053564142, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00663854152637486, "avg_duration_per_task_ms": 622965.2239057239, "avg_output_tokens_per_task": 43822.446127946125, "last_run_timestamp": "2026-10-31T16:01:53.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.6484773807106599, "accuracy_std_dev": 0.0054344006339245585, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.00290907614213198, "avg_duration_per_task_ms": 58072.35659898477, "avg_output_tokens_per_task": 2371.812182741117, "last_run_timestamp": "2026-10-27T00:30:02.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.6477272500000001, "accuracy_std_dev": 0.00710503227913143, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.003938823232323232, "avg_duration_per_task_ms": 63531.12626262626, "avg_output_tokens_per_task": 2389.935606060606, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "CoreWeave", "accuracy": 0.647493, "accuracy_std_dev": null, "run_count": 1, "total_questions": 113, "avg_cost_per_task_usd": 0.005189893805309735, "avg_duration_per_task_ms": 619234.2654867256, "avg_output_tokens_per_task": 34365.4778761062, "last_run_timestamp": "2026-10-01T08:28:32.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.6470586811145511, "accuracy_std_dev": 0.06899576470516142, "run_count": 4, "total_questions": 646, "avg_cost_per_task_usd": 0.006973526315789473, "avg_duration_per_task_ms": 477220.6222910217, "avg_output_tokens_per_task": 36135.41950464396, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.644781, "accuracy_std_dev": 0.0023808285322550727, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0024933156565656567, "avg_duration_per_task_ms": 50490.56060606061, "avg_output_tokens_per_task": 2691.5126262626263, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.64309775, "accuracy_std_dev": 0.010030734778045907, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0023864103535353536, "avg_duration_per_task_ms": 55257.29924242424, "avg_output_tokens_per_task": 2792.7045454545455, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.6419754444444444, "accuracy_std_dev": 0.014018014458475362, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.007092590347923681, "avg_duration_per_task_ms": 241593.06509539843, "avg_output_tokens_per_task": 11377.103254769921, "last_run_timestamp": "2026-10-03T10:20:18.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.641751, "accuracy_std_dev": 0.012926960106691747, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0027229565656565658, "avg_duration_per_task_ms": 151188.63838383838, "avg_output_tokens_per_task": 9770.50707070707, "last_run_timestamp": "2026-10-03T17:16:11.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.641751, "accuracy_std_dev": 0.015110385329302487, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0044971595959595965, "avg_duration_per_task_ms": 76672.9303030303, "avg_output_tokens_per_task": 8453.274747474747, "last_run_timestamp": "2026-10-03T16:51:25.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.641414, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00284089898989899, "avg_duration_per_task_ms": 209352.64646464647, "avg_output_tokens_per_task": 9266.040404040405, "last_run_timestamp": "2026-10-09T15:00:06.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.64025675, "accuracy_std_dev": 0.01324879906585875, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.001088048611111111, "avg_duration_per_task_ms": 331146.75505050505, "avg_output_tokens_per_task": 11572.473484848484, "last_run_timestamp": "2026-10-31T16:31:13.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.6401515, "accuracy_std_dev": 0.002759842885020819, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004124544191919192, "avg_duration_per_task_ms": 34476.35353535353, "avg_output_tokens_per_task": 2259.632575757576, "last_run_timestamp": "2026-10-01T19:13:29.560Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6372055000000001, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.015519383838383837, "avg_duration_per_task_ms": 14506.310606060606, "avg_output_tokens_per_task": 1746.2525252525252, "last_run_timestamp": "2026-10-01T19:12:45.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.6367847499999999, "accuracy_std_dev": 0.012277609526593496, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015237281565656567, "avg_duration_per_task_ms": 18840.242424242424, "avg_output_tokens_per_task": 1618.909090909091, "last_run_timestamp": "2026-10-01T19:14:40.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.6358028333333334, "accuracy_std_dev": 0.012808867337599457, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.001631114478114478, "avg_duration_per_task_ms": 75943.06734006734, "avg_output_tokens_per_task": 8246.542087542088, "last_run_timestamp": "2026-10-03T16:47:32.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6351663037907099, "accuracy_std_dev": 0.019472921033065377, "run_count": 10, "total_questions": 1873, "avg_cost_per_task_usd": 0.005367699412706888, "avg_duration_per_task_ms": 807188.147357181, "avg_output_tokens_per_task": 35119.06513614522, "last_run_timestamp": "2026-10-31T08:20:09.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.6324910327398047, "accuracy_std_dev": 0.011104531477434478, "run_count": 9, "total_questions": 1741, "avg_cost_per_task_usd": 0.01822674784606548, "avg_duration_per_task_ms": 490237.7823090178, "avg_output_tokens_per_task": 91135.2251579552, "last_run_timestamp": "2026-10-03T17:08:02.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.632155, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005966434343434343, "avg_duration_per_task_ms": 235473.01515151514, "avg_output_tokens_per_task": 48249.11363636364, "last_run_timestamp": "2026-10-01T08:24:52.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.6315160731707317, "accuracy_std_dev": 0.011095205901649571, "run_count": 4, "total_questions": 697, "avg_cost_per_task_usd": 0.008070608321377333, "avg_duration_per_task_ms": 35343.74175035868, "avg_output_tokens_per_task": 13371.598278335725, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.6313132, "accuracy_std_dev": 0.007896357575490096, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0027152242424242426, "avg_duration_per_task_ms": 260211.7595959596, "avg_output_tokens_per_task": 8830.749494949496, "last_run_timestamp": "2026-10-03T17:03:50.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.6306148156028369, "accuracy_std_dev": 0.05281625698168657, "run_count": 3, "total_questions": 282, "avg_cost_per_task_usd": 0.0037584255319148938, "avg_duration_per_task_ms": 208452.59574468085, "avg_output_tokens_per_task": 12308.468085106382, "last_run_timestamp": "2026-10-23T09:29:14.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.6303372741573033, "accuracy_std_dev": 0.1138422900195217, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.009750982584269664, "avg_duration_per_task_ms": 363946.58370786515, "avg_output_tokens_per_task": 27697.321348314606, "last_run_timestamp": "2026-10-31T16:12:07.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.6302757276308385, "accuracy_std_dev": 0.007550033035656602, "run_count": 9, "total_questions": 1777, "avg_cost_per_task_usd": 0.001856440067529544, "avg_duration_per_task_ms": 236824.7703995498, "avg_output_tokens_per_task": 10141.058525604953, "last_run_timestamp": "2026-10-31T16:10:09.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.6295420719476744, "accuracy_std_dev": 0.1792338661514529, "run_count": 14, "total_questions": 2752, "avg_cost_per_task_usd": 0.02676043095930232, "avg_duration_per_task_ms": 725458.0534156977, "avg_output_tokens_per_task": 37903.851744186046, "last_run_timestamp": "2026-10-31T16:08:33.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Venice", "accuracy": 0.6292411856287424, "accuracy_std_dev": 0.0117815, "run_count": 2, "total_questions": 334, "avg_cost_per_task_usd": 0.0071094161676646716, "avg_duration_per_task_ms": 112258.81736526947, "avg_output_tokens_per_task": 28232.077844311378, "last_run_timestamp": "2026-08-15T08:02:44.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.6290686666666666, "accuracy_std_dev": 0.03535789523021852, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00678852356902357, "avg_duration_per_task_ms": 106935.86756453423, "avg_output_tokens_per_task": 9465.799102132436, "last_run_timestamp": "2026-10-03T10:32:36.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6285795064570466, "accuracy_std_dev": 0.020919173358428876, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.01718929421673217, "avg_duration_per_task_ms": 631165.4817518248, "avg_output_tokens_per_task": 83912.1010668164, "last_run_timestamp": "2026-10-03T17:40:51.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6275255000000001, "accuracy_std_dev": 0.010334903785231893, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0036913661616161616, "avg_duration_per_task_ms": 150999.66287878787, "avg_output_tokens_per_task": 12500.281565656565, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.6272831084474886, "accuracy_std_dev": 0.013804623233441915, "run_count": 9, "total_questions": 1752, "avg_cost_per_task_usd": 0.016875459474885845, "avg_duration_per_task_ms": 80669.67808219178, "avg_output_tokens_per_task": 56020.17294520548, "last_run_timestamp": "2026-10-31T16:52:47.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.627104375, "accuracy_std_dev": 0.008333089267155095, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.024497946969696968, "avg_duration_per_task_ms": 1060409.1414141415, "avg_output_tokens_per_task": 101874.34027777778, "last_run_timestamp": "2026-10-23T09:03:34.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6265433333333333, "accuracy_std_dev": 0.01630548430913829, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004165183501683501, "avg_duration_per_task_ms": 241184.85185185185, "avg_output_tokens_per_task": 13613.127104377105, "last_run_timestamp": "2026-10-23T08:08:11.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.6212125000000001, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023548611111111114, "avg_duration_per_task_ms": 64938.90151515151, "avg_output_tokens_per_task": 2165.318181818182, "last_run_timestamp": "2026-10-01T19:21:21.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.621212, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00552190909090909, "avg_duration_per_task_ms": 174055.64141414143, "avg_output_tokens_per_task": 10415.944444444445, "last_run_timestamp": "2026-10-04T03:09:40.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.6199326452702703, "accuracy_std_dev": 0.053313699046305146, "run_count": 3, "total_questions": 592, "avg_cost_per_task_usd": 0.037519454391891895, "avg_duration_per_task_ms": 2801762.006756757, "avg_output_tokens_per_task": 26159.864864864863, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.6182949425287356, "accuracy_std_dev": 0.0013475000000000015, "run_count": 2, "total_questions": 348, "avg_cost_per_task_usd": 0.011105244252873564, "avg_duration_per_task_ms": 345775.4454022989, "avg_output_tokens_per_task": 38606.68103448276, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.6170940030769231, "accuracy_std_dev": 0.014320214565431623, "run_count": 5, "total_questions": 975, "avg_cost_per_task_usd": 0.015717948717948717, "avg_duration_per_task_ms": 780521.6543589743, "avg_output_tokens_per_task": 16712.84717948718, "last_run_timestamp": "2026-10-03T11:06:38.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6123375670628184, "accuracy_std_dev": 0.008273645159312946, "run_count": 3, "total_questions": 589, "avg_cost_per_task_usd": 0.003321100169779287, "avg_duration_per_task_ms": 426798.44821731746, "avg_output_tokens_per_task": 13430.818336162989, "last_run_timestamp": "2026-10-08T22:16:42.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.611953, "accuracy_std_dev": 0.007760307500350757, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005744295454545455, "avg_duration_per_task_ms": 648984.4204545454, "avg_output_tokens_per_task": 19126.670454545456, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.6104378, "accuracy_std_dev": 0.012074440366327535, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.003635177777777778, "avg_duration_per_task_ms": 32727.437373737375, "avg_output_tokens_per_task": 2146.0969696969696, "last_run_timestamp": "2026-10-03T17:03:29.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.6102123225480283, "accuracy_std_dev": 0.005364099163885748, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.004066017189079878, "avg_duration_per_task_ms": 27933.76643073812, "avg_output_tokens_per_task": 2160.4469160768454, "last_run_timestamp": "2026-10-03T17:03:50.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.609428, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00778269696969697, "avg_duration_per_task_ms": 302947.1666666667, "avg_output_tokens_per_task": 15369.68686868687, "last_run_timestamp": "2026-10-08T22:57:53.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6073235, "accuracy_std_dev": 0.010738651928896826, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008191137626262626, "avg_duration_per_task_ms": 318721.5845959596, "avg_output_tokens_per_task": 17321.883838383837, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6059645129698252, "accuracy_std_dev": 0.012733602453351516, "run_count": 10, "total_questions": 1889, "avg_cost_per_task_usd": 0.008509499205929062, "avg_duration_per_task_ms": 554697.4531498147, "avg_output_tokens_per_task": 13980.221281101112, "last_run_timestamp": "2026-10-08T22:50:36.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Mara", "accuracy": 0.6059481003717473, "accuracy_std_dev": 0.024067533945591992, "run_count": 3, "total_questions": 269, "avg_cost_per_task_usd": 0.03276318587360595, "avg_duration_per_task_ms": 68520.27137546468, "avg_output_tokens_per_task": 7032.501858736059, "last_run_timestamp": "2026-10-31T16:08:33.699Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.6042256518448439, "accuracy_std_dev": 0.02010368015853481, "run_count": 6, "total_questions": 1057, "avg_cost_per_task_usd": 0.0021907710501419112, "avg_duration_per_task_ms": 51585.36707663198, "avg_output_tokens_per_task": 2058.464522232734, "last_run_timestamp": "2026-10-08T23:15:07.689Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.6040405999999999, "accuracy_std_dev": 0.007924755859961838, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.003472381818181818, "avg_duration_per_task_ms": 58103.21111111111, "avg_output_tokens_per_task": 2119.49797979798, "last_run_timestamp": "2026-10-03T16:57:23.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.6033671999999999, "accuracy_std_dev": 0.005694314722598297, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010998522222222222, "avg_duration_per_task_ms": 31713.89797979798, "avg_output_tokens_per_task": 2095.668686868687, "last_run_timestamp": "2026-10-03T17:43:44.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.599327, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014991068181818182, "avg_duration_per_task_ms": 284433.01515151514, "avg_output_tokens_per_task": 12302.075757575758, "last_run_timestamp": "2026-08-15T15:26:03.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.599327, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012440297979797978, "avg_duration_per_task_ms": 230792.08080808082, "avg_output_tokens_per_task": 41242.71717171717, "last_run_timestamp": "2026-10-04T03:37:16.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.5990461666666667, "accuracy_std_dev": 0.015102210725769769, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003087529461279461, "avg_duration_per_task_ms": 677793.6026936027, "avg_output_tokens_per_task": 13663.569865319865, "last_run_timestamp": "2026-10-30T14:52:53.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.5983168000000001, "accuracy_std_dev": 0.003626295707743628, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.003269590909090909, "avg_duration_per_task_ms": 65326.75858585859, "avg_output_tokens_per_task": 2077.4313131313133, "last_run_timestamp": "2026-10-03T16:59:44.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.5972300452232899, "accuracy_std_dev": 0.06739364692023929, "run_count": 9, "total_questions": 1769, "avg_cost_per_task_usd": 0.012534433578292821, "avg_duration_per_task_ms": 1310890.6302996043, "avg_output_tokens_per_task": 57642.5850763143, "last_run_timestamp": "2026-10-31T16:05:14.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5963805, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01735105808080808, "avg_duration_per_task_ms": 112232.29292929293, "avg_output_tokens_per_task": 43198.833333333336, "last_run_timestamp": "2026-10-01T19:14:48.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5948373333333333, "accuracy_std_dev": 0.010316724200162651, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04306378282828283, "avg_duration_per_task_ms": 401108.6296296296, "avg_output_tokens_per_task": 21434.68181818182, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.5942281016850668, "accuracy_std_dev": 0.00965016140081236, "run_count": 9, "total_questions": 1721, "avg_cost_per_task_usd": 0.009055742010459035, "avg_duration_per_task_ms": 523063.0418361418, "avg_output_tokens_per_task": 45085.088901801275, "last_run_timestamp": "2026-10-03T17:04:03.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.591751, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0029208080808080807, "avg_duration_per_task_ms": 521963.51515151514, "avg_output_tokens_per_task": 11688.588383838383, "last_run_timestamp": "2026-10-01T16:32:22.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.5905352222222223, "accuracy_std_dev": 0.008575970638654383, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.01031239337822671, "avg_duration_per_task_ms": 11512.907968574635, "avg_output_tokens_per_task": 13295.387205387206, "last_run_timestamp": "2026-10-31T16:21:16.340Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.5896185734720416, "accuracy_std_dev": 0.2364552374805523, "run_count": 8, "total_questions": 1538, "avg_cost_per_task_usd": 0.08412247919375812, "avg_duration_per_task_ms": 1774600.33289987, "avg_output_tokens_per_task": 39146.597529258775, "last_run_timestamp": "2026-10-31T16:02:18.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.5875422, "accuracy_std_dev": 0.007299620439447542, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006416512121212121, "avg_duration_per_task_ms": 659187.0757575758, "avg_output_tokens_per_task": 42517.9595959596, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5835020000000001, "accuracy_std_dev": 0.037153058162148644, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02795441818181818, "avg_duration_per_task_ms": 1395358.9363636363, "avg_output_tokens_per_task": 69775.76262626263, "last_run_timestamp": "2026-10-13T21:28:41.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.5833334540682414, "accuracy_std_dev": 0.047090518334207944, "run_count": 8, "total_questions": 762, "avg_cost_per_task_usd": 0.030894883202099738, "avg_duration_per_task_ms": 61623.28215223097, "avg_output_tokens_per_task": 6621.586614173229, "last_run_timestamp": "2026-10-31T16:41:30.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.5830470075601375, "accuracy_std_dev": 0.0830182052186651, "run_count": 9, "total_questions": 1455, "avg_cost_per_task_usd": 0.05815560893470791, "avg_duration_per_task_ms": 352334.5237113402, "avg_output_tokens_per_task": 22047.301030927836, "last_run_timestamp": "2026-10-31T16:21:16.340Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Fireworks", "accuracy": 0.5810130708860759, "accuracy_std_dev": 0.012019500000000016, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.01962629113924051, "avg_duration_per_task_ms": 376708.9949367089, "avg_output_tokens_per_task": 97953.7417721519, "last_run_timestamp": "2026-10-31T08:09:25.783Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.5800594410646388, "accuracy_std_dev": 0.007005609872630659, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.025549254752851712, "avg_duration_per_task_ms": 1152259.712294043, "avg_output_tokens_per_task": 67901.9062103929, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Google", "accuracy": 0.5774414999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.010289136363636365, "avg_duration_per_task_ms": 331432.74242424243, "avg_output_tokens_per_task": 40883.856060606064, "last_run_timestamp": "2026-10-17T16:06:05.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.56060625, "accuracy_std_dev": 0.009297342103391676, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.017428513888888886, "avg_duration_per_task_ms": 127446.60858585859, "avg_output_tokens_per_task": 43389.579545454544, "last_run_timestamp": "2026-10-01T19:27:01.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.5558924000000001, "accuracy_std_dev": 0.010399706661247687, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004659020202020202, "avg_duration_per_task_ms": 490788.4282828283, "avg_output_tokens_per_task": 16414.2101010101, "last_run_timestamp": "2026-10-08T23:30:03.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.5526449293849659, "accuracy_std_dev": 0.02398069638824749, "run_count": 7, "total_questions": 1317, "avg_cost_per_task_usd": 0.005556834472285498, "avg_duration_per_task_ms": 809029.5223993926, "avg_output_tokens_per_task": 14330.80941533789, "last_run_timestamp": "2026-10-30T20:09:39.337Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5521885000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01778226515151515, "avg_duration_per_task_ms": 116667.00757575757, "avg_output_tokens_per_task": 44273.378787878784, "last_run_timestamp": "2026-10-01T19:15:36.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.54798, "accuracy_std_dev": 0.06951982075998184, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005136612373737373, "avg_duration_per_task_ms": 120175.0505050505, "avg_output_tokens_per_task": 4605.713383838384, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5420490229357798, "accuracy_std_dev": 0.06975384672944968, "run_count": 3, "total_questions": 218, "avg_cost_per_task_usd": 0.03485861009174312, "avg_duration_per_task_ms": 397279.0229357798, "avg_output_tokens_per_task": 16660.94495412844, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.5415076988564168, "accuracy_std_dev": 0.08088323645385365, "run_count": 4, "total_questions": 787, "avg_cost_per_task_usd": 0.0033490050825921223, "avg_duration_per_task_ms": 120991.54510800508, "avg_output_tokens_per_task": 3568.362134688691, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.539057, "accuracy_std_dev": 0.13614692394027858, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.003152918181818182, "avg_duration_per_task_ms": 212250.40606060607, "avg_output_tokens_per_task": 7608.965656565657, "last_run_timestamp": "2026-10-31T16:10:36.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.538721, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005473, "avg_duration_per_task_ms": 117831.5808080808, "avg_output_tokens_per_task": 11819.661616161617, "last_run_timestamp": "2026-10-04T03:06:12.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.5373176666666667, "accuracy_std_dev": 0.013403444242266851, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0019087592592592591, "avg_duration_per_task_ms": 97924.95622895623, "avg_output_tokens_per_task": 1886.641414141414, "last_run_timestamp": "2026-10-08T22:10:22.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.537037, "accuracy_std_dev": null, "run_count": 1, "total_questions": 126, "avg_cost_per_task_usd": 0.007595468253968254, "avg_duration_per_task_ms": 1060044.9047619049, "avg_output_tokens_per_task": 22408.52380952381, "last_run_timestamp": "2026-08-08T10:11:59.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.5280586666666666, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01799902861952862, "avg_duration_per_task_ms": 14939.786195286195, "avg_output_tokens_per_task": 1607.080808080808, "last_run_timestamp": "2026-10-02T23:16:52.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.5266555, "accuracy_std_dev": 0.023895888382383546, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01547567760942761, "avg_duration_per_task_ms": 13716.89898989899, "avg_output_tokens_per_task": 1354.1801346801346, "last_run_timestamp": "2026-10-02T23:17:37.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.5238700798479087, "accuracy_std_dev": 0.008445731536551466, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.016655803548795946, "avg_duration_per_task_ms": 761511.7452471483, "avg_output_tokens_per_task": 42502.96070975919, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.5202019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017511833333333334, "avg_duration_per_task_ms": 16177.118686868687, "avg_output_tokens_per_task": 1559.209595959596, "last_run_timestamp": "2026-10-01T16:04:12.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.5193605, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.003164820707070707, "avg_duration_per_task_ms": 106661.49242424243, "avg_output_tokens_per_task": 2729.3585858585857, "last_run_timestamp": "2026-08-15T08:46:42.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.5168355, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017194939393939394, "avg_duration_per_task_ms": 13199.247474747475, "avg_output_tokens_per_task": 1523.2777777777778, "last_run_timestamp": "2026-10-01T16:28:34.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.5151519999999999, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018041959595959597, "avg_duration_per_task_ms": 13498.479797979799, "avg_output_tokens_per_task": 1611.5353535353536, "last_run_timestamp": "2026-10-01T16:26:08.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.5151515, "accuracy_std_dev": 0.008417499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004014873737373737, "avg_duration_per_task_ms": 26643.18181818182, "avg_output_tokens_per_task": 2582.7247474747473, "last_run_timestamp": "2026-08-15T09:25:42.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.510943, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013169762626262625, "avg_duration_per_task_ms": 12925.411616161617, "avg_output_tokens_per_task": 1121.608585858586, "last_run_timestamp": "2026-10-01T16:03:45.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.5084175, "accuracy_std_dev": 0.025252499999999983, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004674189393939395, "avg_duration_per_task_ms": 291399.3181818182, "avg_output_tokens_per_task": 2695.719696969697, "last_run_timestamp": "2026-08-15T09:11:29.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.5075759999999999, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008372954545454546, "avg_duration_per_task_ms": 96066.76515151515, "avg_output_tokens_per_task": 2787.2348484848485, "last_run_timestamp": "2026-08-15T08:49:03.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.5050510000000001, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008273518518518518, "avg_duration_per_task_ms": 18715.93265993266, "avg_output_tokens_per_task": 1873.8602693602693, "last_run_timestamp": "2026-10-02T23:16:50.042Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009755954545454545, "avg_duration_per_task_ms": 700759.0555555555, "avg_output_tokens_per_task": 64762.606060606064, "last_run_timestamp": "2026-10-01T08:24:17.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.4995792500000001, "accuracy_std_dev": 0.02025051336849268, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0008581388888888889, "avg_duration_per_task_ms": 12992.723484848484, "avg_output_tokens_per_task": 1853.0088383838383, "last_run_timestamp": "2026-10-01T19:12:29.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.49700711111111107, "accuracy_std_dev": 0.01216009606545061, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.002385578002244669, "avg_duration_per_task_ms": 9967.221099887767, "avg_output_tokens_per_task": 2372.858024691358, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.49242400000000003, "accuracy_std_dev": 0.0008419999999999817, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0017634166666666666, "avg_duration_per_task_ms": 30891.38888888889, "avg_output_tokens_per_task": 1731.7373737373737, "last_run_timestamp": "2026-10-01T16:05:56.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.49168807800511505, "accuracy_std_dev": 0.022345304455019185, "run_count": 4, "total_questions": 782, "avg_cost_per_task_usd": 0.010985424552429669, "avg_duration_per_task_ms": 526999.0754475703, "avg_output_tokens_per_task": 27562.6368286445, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.49074049999999997, "accuracy_std_dev": 0.009259500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008383863636363636, "avg_duration_per_task_ms": 22708.954545454544, "avg_output_tokens_per_task": 1902.2954545454545, "last_run_timestamp": "2026-10-01T19:13:37.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.48782048076923085, "accuracy_std_dev": 0.015008531315555174, "run_count": 4, "total_questions": 780, "avg_cost_per_task_usd": 0.011197282051282051, "avg_duration_per_task_ms": 480526.36282051285, "avg_output_tokens_per_task": 27926.11923076923, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.4835387777777777, "accuracy_std_dev": 0.011018803965925972, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0025987850729517394, "avg_duration_per_task_ms": 48627.889450056115, "avg_output_tokens_per_task": 2852.319865319865, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.482768822853688, "accuracy_std_dev": 0.014790995023742912, "run_count": 17, "total_questions": 3308, "avg_cost_per_task_usd": 0.001309269044740024, "avg_duration_per_task_ms": 60513.06045949214, "avg_output_tokens_per_task": 2352.4238210399035, "last_run_timestamp": "2026-10-31T16:01:53.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.48148122222222217, "accuracy_std_dev": 0.009848511007341592, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.002354208754208754, "avg_duration_per_task_ms": 15824.776655443322, "avg_output_tokens_per_task": 2212.813131313131, "last_run_timestamp": "2026-10-31T16:05:14.701Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.478956, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.048212727272727275, "avg_duration_per_task_ms": 51496.83838383838, "avg_output_tokens_per_task": 16895.032828282827, "last_run_timestamp": "2026-08-15T14:46:10.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4765793529411765, "accuracy_std_dev": 0.006944000000000006, "run_count": 2, "total_questions": 306, "avg_cost_per_task_usd": 0.0011232875816993464, "avg_duration_per_task_ms": 126677.51307189543, "avg_output_tokens_per_task": 2069.3627450980393, "last_run_timestamp": "2026-10-01T16:04:45.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.4762437777777777, "accuracy_std_dev": 0.013290307736406826, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0013282469135802468, "avg_duration_per_task_ms": 46260.15375982042, "avg_output_tokens_per_task": 2383.4377104377104, "last_run_timestamp": "2026-10-31T16:36:11.519Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.47568244444444446, "accuracy_std_dev": 0.009052379136154844, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0013990594837261505, "avg_duration_per_task_ms": 79190.52525252526, "avg_output_tokens_per_task": 2352.858024691358, "last_run_timestamp": "2026-10-31T16:23:41.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.47558900000000004, "accuracy_std_dev": 0.011198623844026553, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010472727272727272, "avg_duration_per_task_ms": 222278.38005050505, "avg_output_tokens_per_task": 2355.248737373737, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.4728232857142857, "accuracy_std_dev": 0.008882080414508524, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0031378203463203465, "avg_duration_per_task_ms": 63855.042568542565, "avg_output_tokens_per_task": 2211.1046176046175, "last_run_timestamp": "2026-10-17T16:05:55.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.469697, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006110727272727273, "avg_duration_per_task_ms": 42879.368686868685, "avg_output_tokens_per_task": 2318.919191919192, "last_run_timestamp": "2026-10-04T03:02:23.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.46944120359955005, "accuracy_std_dev": 0.013089737311683874, "run_count": 9, "total_questions": 1778, "avg_cost_per_task_usd": 0.0012956473565804273, "avg_duration_per_task_ms": 85305.89820022497, "avg_output_tokens_per_task": 2966.7919010123733, "last_run_timestamp": "2026-10-31T16:31:54.106Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4675925, "accuracy_std_dev": 0.012321014262226957, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0007978666666666667, "avg_duration_per_task_ms": 159410.78737373737, "avg_output_tokens_per_task": 2242.5338383838384, "last_run_timestamp": "2026-10-31T16:02:14.361Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.4637588786346397, "accuracy_std_dev": 0.005380526437998433, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.0022411302149178258, "avg_duration_per_task_ms": 37374.46017699115, "avg_output_tokens_per_task": 2724.637168141593, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.461279, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0011232171717171719, "avg_duration_per_task_ms": 136788.7626262626, "avg_output_tokens_per_task": 2036.2272727272727, "last_run_timestamp": "2026-10-01T16:27:00.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.45959600000000006, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010278964646464647, "avg_duration_per_task_ms": 96379.95959595959, "avg_output_tokens_per_task": 1848.388888888889, "last_run_timestamp": "2026-10-01T16:47:11.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.457071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0017076717171717172, "avg_duration_per_task_ms": 27081.62626262626, "avg_output_tokens_per_task": 1337.1363636363637, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.442761, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0013494898989898992, "avg_duration_per_task_ms": 69228.69696969698, "avg_output_tokens_per_task": 1528.5050505050506, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.43602700000000005, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010042146464646464, "avg_duration_per_task_ms": 21558.090909090908, "avg_output_tokens_per_task": 1481.030303030303, "last_run_timestamp": "2026-10-01T19:16:07.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.4320892331288344, "accuracy_std_dev": 0.09307635724371684, "run_count": 10, "total_questions": 1956, "avg_cost_per_task_usd": 0.004473661042944785, "avg_duration_per_task_ms": 43451.16871165644, "avg_output_tokens_per_task": 1885.2627811860941, "last_run_timestamp": "2026-10-31T16:05:36.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.43139700000000003, "accuracy_std_dev": 0.012951914704011906, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010746982323232325, "avg_duration_per_task_ms": 24227.715909090908, "avg_output_tokens_per_task": 1509.2487373737374, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.429293, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0015700353535353536, "avg_duration_per_task_ms": 59122.85858585859, "avg_output_tokens_per_task": 1664.8737373737374, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4276093333333334, "accuracy_std_dev": 0.00629925838668507, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0010292508417508418, "avg_duration_per_task_ms": 16725.542087542086, "avg_output_tokens_per_task": 1523.1902356902358, "last_run_timestamp": "2026-10-02T23:19:57.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.422559, "accuracy_std_dev": 0.09221941654915557, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.019149646464646466, "avg_duration_per_task_ms": 3630846.4646464647, "avg_output_tokens_per_task": 17674.203703703704, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.405724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0003259848484848485, "avg_duration_per_task_ms": 29056.348484848484, "avg_output_tokens_per_task": 1917.1515151515152, "last_run_timestamp": "2026-10-08T22:01:16.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy-mt2-30b-a3b-20260521", "model_slug": "tencent/hy-mt2-30b-a3b", "model_name": "Tencent: Hy-MT2-30B-A3B", "provider_name": "Reka", "accuracy": 0.388889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00023147979797979797, "avg_duration_per_task_ms": 8604.353535353535, "avg_output_tokens_per_task": 710.540404040404, "last_run_timestamp": "2026-09-24T06:00:03.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.3855215, "accuracy_std_dev": 0.011293261320362695, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0001567070707070707, "avg_duration_per_task_ms": 7111.70202020202, "avg_output_tokens_per_task": 932.0808080808081, "last_run_timestamp": "2026-10-01T19:39:32.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "NextBit", "accuracy": 0.3804713333333334, "accuracy_std_dev": 0.013950341891469493, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00034128451178451176, "avg_duration_per_task_ms": 43460.74410774411, "avg_output_tokens_per_task": 1987.3602693602693, "last_run_timestamp": "2026-10-30T14:15:27.188Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.346801, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008616373737373737, "avg_duration_per_task_ms": 1179181.5303030303, "avg_output_tokens_per_task": 32995.232323232325, "last_run_timestamp": "2026-10-03T00:19:24.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.3325851111111111, "accuracy_std_dev": 0.09545144872126171, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.008632653759820426, "avg_duration_per_task_ms": 536841.7272727273, "avg_output_tokens_per_task": 21346.39674523008, "last_run_timestamp": "2026-10-31T16:45:48.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.2882121979695431, "accuracy_std_dev": 0.01622631095200357, "run_count": 9, "total_questions": 1773, "avg_cost_per_task_usd": 0.0028333936830231246, "avg_duration_per_task_ms": 49259.41624365482, "avg_output_tokens_per_task": 35365.48110547095, "last_run_timestamp": "2026-10-31T16:04:29.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.2873176666666667, "accuracy_std_dev": 0.0007938452130121062, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00006128619528619528, "avg_duration_per_task_ms": 10231.205387205388, "avg_output_tokens_per_task": 580.9023569023569, "last_run_timestamp": "2026-10-08T22:00:27.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.286195, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005941969696969697, "avg_duration_per_task_ms": 14998.156565656565, "avg_output_tokens_per_task": 1642.1363636363637, "last_run_timestamp": "2026-10-13T18:56:05.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.2800224444444444, "accuracy_std_dev": 0.02023328750193112, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0012389450056116721, "avg_duration_per_task_ms": 144968.38888888888, "avg_output_tokens_per_task": 17614.97474747475, "last_run_timestamp": "2026-10-31T16:08:31.064Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.27693599999999996, "accuracy_std_dev": 0.010943000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0007441464646464646, "avg_duration_per_task_ms": 48473.70202020202, "avg_output_tokens_per_task": 2545.467171717172, "last_run_timestamp": "2026-08-15T09:28:07.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.27665533333333336, "accuracy_std_dev": 0.0007938452130121063, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00006201683501683503, "avg_duration_per_task_ms": 9496.395622895623, "avg_output_tokens_per_task": 580.2053872053872, "last_run_timestamp": "2026-10-01T20:43:57.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.2745902896174863, "accuracy_std_dev": 0.009941000000000005, "run_count": 2, "total_questions": 366, "avg_cost_per_task_usd": 0.0000641912568306011, "avg_duration_per_task_ms": 8433.43169398907, "avg_output_tokens_per_task": 532.0136612021857, "last_run_timestamp": "2026-10-01T16:01:04.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.27161569453924916, "accuracy_std_dev": 0.020450238993441835, "run_count": 3, "total_questions": 586, "avg_cost_per_task_usd": 0.000055156996587030725, "avg_duration_per_task_ms": 10158.571672354949, "avg_output_tokens_per_task": 550.3515358361775, "last_run_timestamp": "2026-10-01T20:47:49.128Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.2713588193548387, "accuracy_std_dev": 0.013427628284002107, "run_count": 9, "total_questions": 1705, "avg_cost_per_task_usd": 0.009860867448680352, "avg_duration_per_task_ms": 405997.0574780059, "avg_output_tokens_per_task": 44105.84926686217, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-4b-it", "model_slug": "google/gemma-3-4b-it", "model_name": "Google: Gemma 3 4B", "provider_name": "Parasail", "accuracy": 0.271044, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00011306565656565657, "avg_duration_per_task_ms": 10846.11616161616, "avg_output_tokens_per_task": 1804.4646464646464, "last_run_timestamp": "2026-10-15T03:00:03.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.2656565000000001, "accuracy_std_dev": 0.010207038417190359, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.00039605808080808073, "avg_duration_per_task_ms": 384073.6873737374, "avg_output_tokens_per_task": 9498.437373737373, "last_run_timestamp": "2026-10-31T16:08:31.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.2568276666666667, "accuracy_std_dev": 0.01356372386592676, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00042733389450056114, "avg_duration_per_task_ms": 74390.75308641975, "avg_output_tokens_per_task": 8223.042087542088, "last_run_timestamp": "2026-10-31T16:55:07.715Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.253367, "accuracy_std_dev": 0.23989899999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.016424065656565658, "avg_duration_per_task_ms": 206615.3207070707, "avg_output_tokens_per_task": 20468.073232323233, "last_run_timestamp": "2026-08-15T15:24:35.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.21801350000000003, "accuracy_std_dev": 0.11886565594085619, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0018304709595959596, "avg_duration_per_task_ms": 56840.131313131315, "avg_output_tokens_per_task": 1584.419191919192, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.17239079999999998, "accuracy_std_dev": 0.1395319132204529, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.001048532323232323, "avg_duration_per_task_ms": 421240.27575757576, "avg_output_tokens_per_task": 3399.7606060606063, "last_run_timestamp": "2026-10-24T16:03:19.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.17174282233502539, "accuracy_std_dev": 0.01788397279378631, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.019897788494077834, "avg_duration_per_task_ms": 1784246.4196277496, "avg_output_tokens_per_task": 110294.06598984772, "last_run_timestamp": "2026-10-08T23:13:58.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.1161615, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017850737373737373, "avg_duration_per_task_ms": 46853.982323232325, "avg_output_tokens_per_task": 11776.444444444445, "last_run_timestamp": "2026-10-01T16:00:58.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.112795, "accuracy_std_dev": 0.006733999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018038613636363634, "avg_duration_per_task_ms": 21125.79292929293, "avg_output_tokens_per_task": 11795.222222222223, "last_run_timestamp": "2026-10-01T16:00:58.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017984406565656568, "avg_duration_per_task_ms": 21554.590909090908, "avg_output_tokens_per_task": 11812.823232323231, "last_run_timestamp": "2026-10-01T16:02:13.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.0025255000000000004, "accuracy_std_dev": 0.0008415000000000001, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0034195530303030306, "avg_duration_per_task_ms": 159765.32575757575, "avg_output_tokens_per_task": 3219.0782828282827, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.0016840000000000002, "accuracy_std_dev": 0, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.003431577441077441, "avg_duration_per_task_ms": 155214.41582491584, "avg_output_tokens_per_task": 3231.1026936026938, "last_run_timestamp": "2026-10-31T16:04:43.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8233334999999999, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.7708268799999999, "avg_duration_per_task_ms": 139666.89, "avg_output_tokens_per_task": 5287.85, "last_run_timestamp": "2026-08-15T08:09:14.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.813333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00614692, "avg_duration_per_task_ms": 141813.64, "avg_output_tokens_per_task": 15680.88, "last_run_timestamp": "2026-08-08T08:30:38.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20567228, "avg_duration_per_task_ms": 69455.26, "avg_output_tokens_per_task": 15737.28, "last_run_timestamp": "2026-08-15T00:15:11.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8005555833333333, "accuracy_std_dev": 0.022353925693482164, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.8806157300000002, "avg_duration_per_task_ms": 125606.79666666666, "avg_output_tokens_per_task": 5738.843333333333, "last_run_timestamp": "2026-10-31T11:20:38.480Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8000000909090909, "accuracy_std_dev": 0.020695063581156234, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.8742483381818181, "avg_duration_per_task_ms": 136193.65272727274, "avg_output_tokens_per_task": 5706.489090909091, "last_run_timestamp": "2026-10-31T12:24:55.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.8, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.39313749000000003, "avg_duration_per_task_ms": 477750.16, "avg_output_tokens_per_task": 34082.08, "last_run_timestamp": "2026-10-01T16:00:45.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.7981221830985916, "accuracy_std_dev": 0.05639978873679898, "run_count": 3, "total_questions": 142, "avg_cost_per_task_usd": 0.007624964788732395, "avg_duration_per_task_ms": 30939.802816901407, "avg_output_tokens_per_task": 8761.732394366198, "last_run_timestamp": "2026-10-31T11:56:20.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.7977776666666667, "accuracy_std_dev": 0.01911642045177097, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008363246666666668, "avg_duration_per_task_ms": 272879.14, "avg_output_tokens_per_task": 12625.026666666667, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.795, "accuracy_std_dev": 0.02696008022985094, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.8826118166666667, "avg_duration_per_task_ms": 119564.475, "avg_output_tokens_per_task": 5739.836666666667, "last_run_timestamp": "2026-10-31T11:38:36.784Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.7946430446428572, "accuracy_std_dev": 0.027357193240535482, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.20150866741071427, "avg_duration_per_task_ms": 823434.4174107143, "avg_output_tokens_per_task": 34488.85267857143, "last_run_timestamp": "2026-10-31T19:01:09.612Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.7938889166666666, "accuracy_std_dev": 0.016433554015379277, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.8273622516666667, "avg_duration_per_task_ms": 126448.29, "avg_output_tokens_per_task": 5727.116666666667, "last_run_timestamp": "2026-10-31T11:18:52.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7936329325842696, "accuracy_std_dev": 0.028642456621828596, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.12726180674157303, "avg_duration_per_task_ms": 2869756.1258426965, "avg_output_tokens_per_task": 27340.368539325842, "last_run_timestamp": "2026-10-31T20:31:41.691Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00828808, "avg_duration_per_task_ms": 196026.56, "avg_output_tokens_per_task": 16303, "last_run_timestamp": "2026-10-14T19:21:35.384Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03146312, "avg_duration_per_task_ms": 636210.72, "avg_output_tokens_per_task": 16745.44, "last_run_timestamp": "2026-10-14T19:17:59.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7916667916666666, "accuracy_std_dev": 0.021709424691016815, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.9256366683333332, "avg_duration_per_task_ms": 93473.74, "avg_output_tokens_per_task": 5715.59, "last_run_timestamp": "2026-10-31T12:30:34.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.79166675, "accuracy_std_dev": 0.012801703448662584, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.71616581, "avg_duration_per_task_ms": 111830.815, "avg_output_tokens_per_task": 6900.955, "last_run_timestamp": "2026-10-01T08:03:01.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.7906666, "accuracy_std_dev": 0.009977824303925176, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.814870324, "avg_duration_per_task_ms": 182126.652, "avg_output_tokens_per_task": 7046.908, "last_run_timestamp": "2026-10-01T08:05:22.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.7899033875278396, "accuracy_std_dev": 0.023747050173666146, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.18324271046770602, "avg_duration_per_task_ms": 436487.29844097997, "avg_output_tokens_per_task": 30283.42316258352, "last_run_timestamp": "2026-10-31T19:12:41.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.788889, "accuracy_std_dev": 0.015234869040023505, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.057788776666666666, "avg_duration_per_task_ms": 893038.1233333333, "avg_output_tokens_per_task": 40447.416666666664, "last_run_timestamp": "2026-10-31T13:13:33.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.7883333333333333, "accuracy_std_dev": 0.020570601102598386, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.4772300233333334, "avg_duration_per_task_ms": 119792.23833333333, "avg_output_tokens_per_task": 7037.026666666667, "last_run_timestamp": "2026-10-31T12:17:12.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.7882883714036617, "accuracy_std_dev": 0.015477690995371508, "run_count": 23, "total_questions": 1147, "avg_cost_per_task_usd": 0.0821072258064516, "avg_duration_per_task_ms": 149668.3635571055, "avg_output_tokens_per_task": 12209.048823016565, "last_run_timestamp": "2026-10-31T12:32:50.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.788, "accuracy_std_dev": 0.02246951515275753, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.15925035199999998, "avg_duration_per_task_ms": 71120.884, "avg_output_tokens_per_task": 6245.168, "last_run_timestamp": "2026-10-31T11:10:36.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.7875705677966102, "accuracy_std_dev": 0.021829507772196534, "run_count": 12, "total_questions": 590, "avg_cost_per_task_usd": 0.07927353559322034, "avg_duration_per_task_ms": 1138885.6372881357, "avg_output_tokens_per_task": 26180.43559322034, "last_run_timestamp": "2026-10-31T20:22:41.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01468154, "avg_duration_per_task_ms": 178223.4, "avg_output_tokens_per_task": 11352.52, "last_run_timestamp": "2026-08-08T08:29:14.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.49828548, "avg_duration_per_task_ms": 348417.68, "avg_output_tokens_per_task": 21971.04, "last_run_timestamp": "2026-10-14T15:16:05.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.786127239884393, "accuracy_std_dev": 0.023660230134130145, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.09665253179190753, "avg_duration_per_task_ms": 380894.4450867052, "avg_output_tokens_per_task": 16737.754335260117, "last_run_timestamp": "2026-10-31T12:56:48.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.7858794166666667, "accuracy_std_dev": 0.021093449409710124, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.07943493055555556, "avg_duration_per_task_ms": 334937.25694444444, "avg_output_tokens_per_task": 18674.54861111111, "last_run_timestamp": "2026-10-31T19:43:30.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.7844446666666666, "accuracy_std_dev": 0.03478329384760578, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04264756333333333, "avg_duration_per_task_ms": 115094.22333333333, "avg_output_tokens_per_task": 4914.21, "last_run_timestamp": "2026-10-31T12:26:47.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.7844444444444445, "accuracy_std_dev": 0.02266234163693451, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.19777408000000002, "avg_duration_per_task_ms": 762878.1888888889, "avg_output_tokens_per_task": 31234.38888888889, "last_run_timestamp": "2026-10-31T19:47:27.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.7844443333333334, "accuracy_std_dev": 0.01632984089872824, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0785912688888889, "avg_duration_per_task_ms": 47362.52222222222, "avg_output_tokens_per_task": 4720.888888888889, "last_run_timestamp": "2026-10-31T11:58:26.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.7833333333333333, "accuracy_std_dev": 0.011706202852429232, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.1729425, "avg_duration_per_task_ms": 66541.35, "avg_output_tokens_per_task": 6203.445, "last_run_timestamp": "2026-10-31T12:38:55.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7833330000000001, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07443635, "avg_duration_per_task_ms": 98397.15, "avg_output_tokens_per_task": 21518.67, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7833330000000001, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06795213, "avg_duration_per_task_ms": 191712.46, "avg_output_tokens_per_task": 8639.07, "last_run_timestamp": "2026-08-15T08:17:17.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.7831052054794521, "accuracy_std_dev": 0.017155537965592597, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.14992966438356164, "avg_duration_per_task_ms": 183129.3904109589, "avg_output_tokens_per_task": 16313.164383561643, "last_run_timestamp": "2026-10-15T21:26:28.195Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.7825, "accuracy_std_dev": 0.023790007240436065, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08953825, "avg_duration_per_task_ms": 274829.5175, "avg_output_tokens_per_task": 19992.6725, "last_run_timestamp": "2026-10-31T12:57:09.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7822223333333334, "accuracy_std_dev": 0.01571327975171178, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.011317603333333334, "avg_duration_per_task_ms": 599501.3033333333, "avg_output_tokens_per_task": 21464.33, "last_run_timestamp": "2026-10-31T14:08:11.994Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.781333, "accuracy_std_dev": 0.027129319932501044, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.0587014, "avg_duration_per_task_ms": 871009.496, "avg_output_tokens_per_task": 40629.384, "last_run_timestamp": "2026-10-31T13:41:45.309Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.7811976407035177, "accuracy_std_dev": 0.02110393286871771, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.09273568467336683, "avg_duration_per_task_ms": 66515.5364321608, "avg_output_tokens_per_task": 4679.810301507538, "last_run_timestamp": "2026-10-31T13:12:01.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.7808333749999999, "accuracy_std_dev": 0.010769114145758466, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0823930725, "avg_duration_per_task_ms": 41435.8875, "avg_output_tokens_per_task": 4744.8375, "last_run_timestamp": "2026-10-31T12:40:46.044Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7801419645390072, "accuracy_std_dev": 0.026804102476217243, "run_count": 3, "total_questions": 141, "avg_cost_per_task_usd": 0.09157175886524822, "avg_duration_per_task_ms": 67531.68794326241, "avg_output_tokens_per_task": 7567.7943262411345, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7800001, "accuracy_std_dev": 0.02108170349592273, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.174296886, "avg_duration_per_task_ms": 72081.318, "avg_output_tokens_per_task": 6231.684, "last_run_timestamp": "2026-10-31T12:10:32.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.78, "accuracy_std_dev": 0.021081938910409114, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.18010515333333332, "avg_duration_per_task_ms": 64530.58666666667, "avg_output_tokens_per_task": 6325.823333333334, "last_run_timestamp": "2026-10-31T12:07:29.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00686524, "avg_duration_per_task_ms": 150471.32, "avg_output_tokens_per_task": 16577.68, "last_run_timestamp": "2026-08-08T08:27:33.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "PrimeIntellect", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0042799399999999994, "avg_duration_per_task_ms": 627315.22, "avg_output_tokens_per_task": 12675.1, "last_run_timestamp": "2026-10-31T21:10:18.835Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7799999000000001, "accuracy_std_dev": 0.034896621932359, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.080800072, "avg_duration_per_task_ms": 206899.728, "avg_output_tokens_per_task": 25416.206, "last_run_timestamp": "2026-10-31T19:22:16.313Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.7786668, "accuracy_std_dev": 0.01484714413481597, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.014840148000000001, "avg_duration_per_task_ms": 379677.016, "avg_output_tokens_per_task": 21736.348, "last_run_timestamp": "2026-10-31T12:28:58.923Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.7786666, "accuracy_std_dev": 0.010666575000439441, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.031286299999999996, "avg_duration_per_task_ms": 221994.964, "avg_output_tokens_per_task": 12740.492, "last_run_timestamp": "2026-10-31T12:48:16.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.7784256831875607, "accuracy_std_dev": 0.029691712716069543, "run_count": 21, "total_questions": 1029, "avg_cost_per_task_usd": 0.07680297376093294, "avg_duration_per_task_ms": 74568.79203109816, "avg_output_tokens_per_task": 4283.289601554908, "last_run_timestamp": "2026-10-31T13:24:01.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.77833325, "accuracy_std_dev": 0.005527984369324845, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12165100000000001, "avg_duration_per_task_ms": 123453.655, "avg_output_tokens_per_task": 4597.39, "last_run_timestamp": "2026-10-28T05:13:02.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.7779999000000001, "accuracy_std_dev": 0.020011142453393298, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.081736628, "avg_duration_per_task_ms": 135707.96, "avg_output_tokens_per_task": 4671.7, "last_run_timestamp": "2026-10-31T12:38:49.692Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.777778, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.012714583333333333, "avg_duration_per_task_ms": 516568.7291666667, "avg_output_tokens_per_task": 14230.208333333334, "last_run_timestamp": "2026-08-03T17:20:47.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "AtlasCloud", "accuracy": 0.777778, "accuracy_std_dev": 0.01257094435593444, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.016507473333333335, "avg_duration_per_task_ms": 92510.76666666666, "avg_output_tokens_per_task": 10861.266666666666, "last_run_timestamp": "2026-10-31T11:43:38.949Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.777459306590258, "accuracy_std_dev": 0.02712463143584833, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.17213766475644698, "avg_duration_per_task_ms": 55665.0888252149, "avg_output_tokens_per_task": 4643.535816618911, "last_run_timestamp": "2026-10-31T12:26:01.294Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.777333342857143, "accuracy_std_dev": 0.021153949387347253, "run_count": 35, "total_questions": 1750, "avg_cost_per_task_usd": 0.4968248080000001, "avg_duration_per_task_ms": 118772.41142857143, "avg_output_tokens_per_task": 7078.027428571429, "last_run_timestamp": "2026-10-31T11:36:20.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.7771427857142857, "accuracy_std_dev": 0.017405512982051608, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.4396931257142857, "avg_duration_per_task_ms": 90015.33428571429, "avg_output_tokens_per_task": 5705.564285714286, "last_run_timestamp": "2026-10-31T12:26:17.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.7766666666666666, "accuracy_std_dev": 0.021430304782143333, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.4784604191666666, "avg_duration_per_task_ms": 102044.3925, "avg_output_tokens_per_task": 7128.966666666666, "last_run_timestamp": "2026-10-31T12:54:18.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.7766666666666666, "accuracy_std_dev": 0.014782246611692315, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17363205, "avg_duration_per_task_ms": 73619.57333333333, "avg_output_tokens_per_task": 6340.086666666667, "last_run_timestamp": "2026-10-31T12:24:45.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.7766666388888891, "accuracy_std_dev": 0.023280401603963375, "run_count": 36, "total_questions": 1800, "avg_cost_per_task_usd": 0.5182516222222223, "avg_duration_per_task_ms": 108585.71333333333, "avg_output_tokens_per_task": 7098.225555555556, "last_run_timestamp": "2026-10-31T12:26:08.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7766665833333334, "accuracy_std_dev": 0.0379571517689318, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.10135336166666668, "avg_duration_per_task_ms": 172908.82, "avg_output_tokens_per_task": 6639.301666666666, "last_run_timestamp": "2026-10-31T11:53:06.149Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7766665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.017346550000000002, "avg_duration_per_task_ms": 335231.13, "avg_output_tokens_per_task": 20113.88, "last_run_timestamp": "2026-10-03T08:17:48.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.7766665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.41419341000000004, "avg_duration_per_task_ms": 38158.51, "avg_output_tokens_per_task": 9152, "last_run_timestamp": "2026-08-15T14:16:11.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.7764815, "accuracy_std_dev": 0.02583109973193382, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.13240519666666667, "avg_duration_per_task_ms": 319498.1977777778, "avg_output_tokens_per_task": 19319.296666666665, "last_run_timestamp": "2026-10-31T13:25:38.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.7760940000000001, "accuracy_std_dev": 0.020936325423770052, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.06970348989898989, "avg_duration_per_task_ms": 95095.22222222222, "avg_output_tokens_per_task": 4292.19696969697, "last_run_timestamp": "2026-10-31T12:44:06.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.7760001999999999, "accuracy_std_dev": 0.028472283599318148, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.06820797999999999, "avg_duration_per_task_ms": 658315.768, "avg_output_tokens_per_task": 13133.756, "last_run_timestamp": "2026-10-31T13:30:16.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.7758335, "accuracy_std_dev": 0.016308646311082982, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.15398643500000003, "avg_duration_per_task_ms": 330861.565, "avg_output_tokens_per_task": 19948.0325, "last_run_timestamp": "2026-10-31T13:41:05.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7757695461847389, "accuracy_std_dev": 0.016169521055368324, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.08735203012048193, "avg_duration_per_task_ms": 139136.04417670684, "avg_output_tokens_per_task": 4443.712851405622, "last_run_timestamp": "2026-10-31T12:29:23.419Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.7757574545454545, "accuracy_std_dev": 0.02074823590031709, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08103076363636365, "avg_duration_per_task_ms": 336159.44, "avg_output_tokens_per_task": 20208.643636363635, "last_run_timestamp": "2026-10-31T12:24:53.587Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.7753625384615385, "accuracy_std_dev": 0.017323958686038136, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.06710363879598663, "avg_duration_per_task_ms": 1125092.9698996656, "avg_output_tokens_per_task": 36573.67558528428, "last_run_timestamp": "2026-10-24T13:26:35.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.7753500890756303, "accuracy_std_dev": 0.023224689913460387, "run_count": 12, "total_questions": 595, "avg_cost_per_task_usd": 0.20088185714285717, "avg_duration_per_task_ms": 515898.5899159664, "avg_output_tokens_per_task": 28073.515966386556, "last_run_timestamp": "2026-10-31T18:53:29.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.7753334, "accuracy_std_dev": 0.026339740994170767, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.065886822, "avg_duration_per_task_ms": 257998.344, "avg_output_tokens_per_task": 14164.592, "last_run_timestamp": "2026-10-31T12:39:11.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.7751514545454545, "accuracy_std_dev": 0.026375843918816165, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.10104016181818182, "avg_duration_per_task_ms": 294074.4890909091, "avg_output_tokens_per_task": 21520.51818181818, "last_run_timestamp": "2026-10-31T11:29:11.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.775, "accuracy_std_dev": 0.01787294665828779, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09322250500000001, "avg_duration_per_task_ms": 167383.47, "avg_output_tokens_per_task": 5123.47, "last_run_timestamp": "2026-10-31T12:32:51.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.7746668, "accuracy_std_dev": 0.012927338819726197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.758212416, "avg_duration_per_task_ms": 112131.876, "avg_output_tokens_per_task": 6866.944, "last_run_timestamp": "2026-10-01T08:05:27.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.7745454545454545, "accuracy_std_dev": 0.02425759601131023, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.48932630545454553, "avg_duration_per_task_ms": 124123.69272727273, "avg_output_tokens_per_task": 7977.827272727272, "last_run_timestamp": "2026-10-31T11:58:26.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7744445, "accuracy_std_dev": 0.018425810536219746, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.45357258333333333, "avg_duration_per_task_ms": 120535.45, "avg_output_tokens_per_task": 7078.328333333333, "last_run_timestamp": "2026-10-31T11:40:34.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.7744444166666666, "accuracy_std_dev": 0.018823368390816466, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.4382412116666667, "avg_duration_per_task_ms": 94200.57833333334, "avg_output_tokens_per_task": 5678.8533333333335, "last_run_timestamp": "2026-10-31T12:52:03.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7742857142857142, "accuracy_std_dev": 0.026945723560596418, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21623159142857146, "avg_duration_per_task_ms": 562306.5571428571, "avg_output_tokens_per_task": 12602.945714285714, "last_run_timestamp": "2026-10-10T11:06:43.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.7739394545454545, "accuracy_std_dev": 0.019165412105066386, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06798592909090909, "avg_duration_per_task_ms": 45561.432727272724, "avg_output_tokens_per_task": 4731.818181818182, "last_run_timestamp": "2026-10-31T12:41:41.331Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.7733335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11407636, "avg_duration_per_task_ms": 360681.75, "avg_output_tokens_per_task": 19997.86, "last_run_timestamp": "2026-10-01T22:40:52.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.7733334, "accuracy_std_dev": 0.02108173511929225, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.05645720600000001, "avg_duration_per_task_ms": 156313.234, "avg_output_tokens_per_task": 16426.692, "last_run_timestamp": "2026-10-31T11:34:45.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.773333375, "accuracy_std_dev": 0.02236071704428047, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04385152750000001, "avg_duration_per_task_ms": 107910.09, "avg_output_tokens_per_task": 4940.6725, "last_run_timestamp": "2026-10-31T12:00:41.768Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.7733333571428571, "accuracy_std_dev": 0.023771504885372875, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.5080915685714286, "avg_duration_per_task_ms": 123754.58285714286, "avg_output_tokens_per_task": 8039.411428571429, "last_run_timestamp": "2026-10-31T11:50:39.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7733333333333333, "accuracy_std_dev": 0.02309381831909905, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.16708652333333332, "avg_duration_per_task_ms": 79748.64333333333, "avg_output_tokens_per_task": 6314.02, "last_run_timestamp": "2026-10-31T11:48:36.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.7733332857142857, "accuracy_std_dev": 0.022537305934283946, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08668051142857143, "avg_duration_per_task_ms": 364495.4428571428, "avg_output_tokens_per_task": 20483.94, "last_run_timestamp": "2026-10-24T12:45:56.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.77333325, "accuracy_std_dev": 0.029814193116492343, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06547900499999999, "avg_duration_per_task_ms": 79239.345, "avg_output_tokens_per_task": 12411.3875, "last_run_timestamp": "2026-10-31T11:09:23.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7733331666666667, "accuracy_std_dev": 0.021081851068448003, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05726335, "avg_duration_per_task_ms": 158508.54333333333, "avg_output_tokens_per_task": 4588.7266666666665, "last_run_timestamp": "2026-10-31T13:11:06.716Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012345539999999999, "avg_duration_per_task_ms": 657478.84, "avg_output_tokens_per_task": 14196.68, "last_run_timestamp": "2026-10-03T00:00:06.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.16341722, "avg_duration_per_task_ms": 174236.4, "avg_output_tokens_per_task": 10664.8, "last_run_timestamp": "2026-08-08T08:27:49.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26629643999999997, "avg_duration_per_task_ms": 438741.58, "avg_output_tokens_per_task": 27876.64, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11485508, "avg_duration_per_task_ms": 381375.48, "avg_output_tokens_per_task": 14199.1, "last_run_timestamp": "2026-10-13T03:50:19.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.7730554583333333, "accuracy_std_dev": 0.031238603972727888, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.4483734791666666, "avg_duration_per_task_ms": 79019.83166666667, "avg_output_tokens_per_task": 5657.6883333333335, "last_run_timestamp": "2026-10-31T12:41:58.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.7725926666666666, "accuracy_std_dev": 0.033176625054396364, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05704026, "avg_duration_per_task_ms": 115075.10666666667, "avg_output_tokens_per_task": 6558.98, "last_run_timestamp": "2026-10-31T12:50:03.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.7724602745591939, "accuracy_std_dev": 0.036002860879163164, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.18971839546599495, "avg_duration_per_task_ms": 615216.5390428212, "avg_output_tokens_per_task": 29565.539042821158, "last_run_timestamp": "2026-10-31T19:27:20.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.77222225, "accuracy_std_dev": 0.02746491926417226, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.1697550733333333, "avg_duration_per_task_ms": 565928.75, "avg_output_tokens_per_task": 35366.556666666664, "last_run_timestamp": "2026-10-31T19:08:11.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7722222222222223, "accuracy_std_dev": 0.03095202181433487, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03593436666666667, "avg_duration_per_task_ms": 446735.1666666667, "avg_output_tokens_per_task": 13815.786666666667, "last_run_timestamp": "2026-10-08T22:42:47.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.7722222083333333, "accuracy_std_dev": 0.029606428127997207, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.5155781208333333, "avg_duration_per_task_ms": 125395.55583333333, "avg_output_tokens_per_task": 7961.098333333333, "last_run_timestamp": "2026-10-31T13:09:20.558Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.7720001999999999, "accuracy_std_dev": 0.024367767418456705, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.14564374000000002, "avg_duration_per_task_ms": 314489.068, "avg_output_tokens_per_task": 20798.24, "last_run_timestamp": "2026-10-31T13:16:57.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.772, "accuracy_std_dev": 0.02679938639596063, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.808827636, "avg_duration_per_task_ms": 183544.5, "avg_output_tokens_per_task": 6940.7, "last_run_timestamp": "2026-10-01T08:03:00.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7718182727272727, "accuracy_std_dev": 0.031151820244762325, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06332707454545455, "avg_duration_per_task_ms": 243359.18363636362, "avg_output_tokens_per_task": 11661.423636363636, "last_run_timestamp": "2026-10-31T12:57:09.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "StreamLake", "accuracy": 0.7716665, "accuracy_std_dev": 0.0016665000000000152, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.050993250000000004, "avg_duration_per_task_ms": 677393.46, "avg_output_tokens_per_task": 30421.97, "last_run_timestamp": "2026-10-31T13:28:05.897Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7715774062499999, "accuracy_std_dev": 0.022644299042712767, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.14963372544642858, "avg_duration_per_task_ms": 341867.6473214286, "avg_output_tokens_per_task": 20209.140625, "last_run_timestamp": "2026-10-31T14:04:49.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.7714602976588629, "accuracy_std_dev": 0.028416062527791094, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.08344332775919731, "avg_duration_per_task_ms": 249520.2474916388, "avg_output_tokens_per_task": 12314.311036789297, "last_run_timestamp": "2026-10-31T19:22:52.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.7713334, "accuracy_std_dev": 0.0305578329670152, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.08027632600000001, "avg_duration_per_task_ms": 358319.328, "avg_output_tokens_per_task": 18206.802, "last_run_timestamp": "2026-10-31T20:27:36.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.7713334, "accuracy_std_dev": 0.020011225740568715, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.051776854, "avg_duration_per_task_ms": 108112.036, "avg_output_tokens_per_task": 4665.94, "last_run_timestamp": "2026-10-31T13:28:16.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.77120534375, "accuracy_std_dev": 0.030014310742266483, "run_count": 18, "total_questions": 896, "avg_cost_per_task_usd": 0.03428859486607143, "avg_duration_per_task_ms": 485944.55691964284, "avg_output_tokens_per_task": 12331.224330357143, "last_run_timestamp": "2026-10-31T19:33:25.394Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.770833375, "accuracy_std_dev": 0.02283704805758343, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2110248925, "avg_duration_per_task_ms": 381381.2675, "avg_output_tokens_per_task": 17629.1275, "last_run_timestamp": "2026-10-31T12:14:30.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.7707407777777777, "accuracy_std_dev": 0.023188894510460925, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11877632888888891, "avg_duration_per_task_ms": 486710.77777777775, "avg_output_tokens_per_task": 21345.64, "last_run_timestamp": "2026-10-17T11:37:24.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.7706668, "accuracy_std_dev": 0.027194791985966728, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.13800210599999999, "avg_duration_per_task_ms": 324537.226, "avg_output_tokens_per_task": 20086.83, "last_run_timestamp": "2026-10-31T13:20:51.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.7706666, "accuracy_std_dev": 0.0204828947622156, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10916058, "avg_duration_per_task_ms": 258128.42, "avg_output_tokens_per_task": 11359.708, "last_run_timestamp": "2026-10-03T16:07:05.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7704762857142857, "accuracy_std_dev": 0.028139468010578456, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.5353783828571429, "avg_duration_per_task_ms": 182998.61857142858, "avg_output_tokens_per_task": 10560.904285714285, "last_run_timestamp": "2026-10-31T12:38:33.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7700000714285714, "accuracy_std_dev": 0.014638549891415803, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.14041841571428573, "avg_duration_per_task_ms": 235377.64428571428, "avg_output_tokens_per_task": 25220.13142857143, "last_run_timestamp": "2026-10-31T13:20:49.855Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.77, "accuracy_std_dev": 0.02472072904467018, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.47639071666666666, "avg_duration_per_task_ms": 109574.44833333333, "avg_output_tokens_per_task": 7160.583333333333, "last_run_timestamp": "2026-10-31T11:27:21.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.77, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.35129818, "avg_duration_per_task_ms": 177012.35, "avg_output_tokens_per_task": 12679.64, "last_run_timestamp": "2026-10-01T19:06:01.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.769999875, "accuracy_std_dev": 0.03382961802340332, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0406579, "avg_duration_per_task_ms": 682767.975, "avg_output_tokens_per_task": 12586.08, "last_run_timestamp": "2026-10-31T12:34:48.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7699997500000001, "accuracy_std_dev": 0.029999861111136825, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08088121, "avg_duration_per_task_ms": 203815.155, "avg_output_tokens_per_task": 9386.985, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7699214781144781, "accuracy_std_dev": 0.02849514533750542, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.010507353535353536, "avg_duration_per_task_ms": 977066.3771043771, "avg_output_tokens_per_task": 18208.95286195286, "last_run_timestamp": "2026-10-31T12:24:58.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.7696296666666667, "accuracy_std_dev": 0.024966483982420205, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.040881691111111104, "avg_duration_per_task_ms": 198238.98888888888, "avg_output_tokens_per_task": 13013.271111111111, "last_run_timestamp": "2026-10-31T20:06:41.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.7695237142857143, "accuracy_std_dev": 0.02386662296246901, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11461242285714283, "avg_duration_per_task_ms": 472929.0742857143, "avg_output_tokens_per_task": 11699.722857142857, "last_run_timestamp": "2026-10-23T07:26:58.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.7693333, "accuracy_std_dev": 0.027680754646685506, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.080204858, "avg_duration_per_task_ms": 55518.588, "avg_output_tokens_per_task": 4833.54, "last_run_timestamp": "2026-10-31T12:18:44.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.7690024251781471, "accuracy_std_dev": 0.02344176596675554, "run_count": 17, "total_questions": 842, "avg_cost_per_task_usd": 0.05268543230403801, "avg_duration_per_task_ms": 269011.41448931117, "avg_output_tokens_per_task": 11083.264845605701, "last_run_timestamp": "2026-10-31T12:29:57.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7685184999999999, "accuracy_std_dev": 0.027604418545769074, "run_count": 36, "total_questions": 1800, "avg_cost_per_task_usd": 0.5604697755555554, "avg_duration_per_task_ms": 113686.57388888889, "avg_output_tokens_per_task": 8067.973333333333, "last_run_timestamp": "2026-10-31T12:31:31.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.7685184444444445, "accuracy_std_dev": 0.02067440250976561, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07816885555555556, "avg_duration_per_task_ms": 93704.89777777778, "avg_output_tokens_per_task": 4401.093333333333, "last_run_timestamp": "2026-10-31T12:34:51.325Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.7684848181818182, "accuracy_std_dev": 0.034709847551636586, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.05697716181818182, "avg_duration_per_task_ms": 301468.96545454545, "avg_output_tokens_per_task": 12590.405454545455, "last_run_timestamp": "2026-10-31T11:18:52.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.768333375, "accuracy_std_dev": 0.027638527357375157, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.12531909, "avg_duration_per_task_ms": 166385.7025, "avg_output_tokens_per_task": 7956.0025, "last_run_timestamp": "2026-10-31T13:24:47.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.76833325, "accuracy_std_dev": 0.0414661475053748, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11548195250000001, "avg_duration_per_task_ms": 265843.155, "avg_output_tokens_per_task": 11738.8375, "last_run_timestamp": "2026-10-31T21:36:17.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.7681482222222222, "accuracy_std_dev": 0.017435059205697385, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.052077991111111104, "avg_duration_per_task_ms": 111760.7, "avg_output_tokens_per_task": 4431.0888888888885, "last_run_timestamp": "2026-10-31T13:22:10.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.7681481111111111, "accuracy_std_dev": 0.022394386178705306, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05412603333333333, "avg_duration_per_task_ms": 221687.28666666665, "avg_output_tokens_per_task": 12553.397777777778, "last_run_timestamp": "2026-10-31T20:15:14.788Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.768, "accuracy_std_dev": 0.020612843894038495, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.06597447600000002, "avg_duration_per_task_ms": 69126.944, "avg_output_tokens_per_task": 4654.312, "last_run_timestamp": "2026-10-31T12:41:59.417Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.7677776666666667, "accuracy_std_dev": 0.043104839177934014, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06842919666666666, "avg_duration_per_task_ms": 974829.46, "avg_output_tokens_per_task": 44158.48333333333, "last_run_timestamp": "2026-10-31T13:18:29.756Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.7675, "accuracy_std_dev": 0.019561171859068173, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0578507225, "avg_duration_per_task_ms": 50974.42, "avg_output_tokens_per_task": 4677.1725, "last_run_timestamp": "2026-10-24T12:36:03.010Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.7674509411764706, "accuracy_std_dev": 0.021678839985437112, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.12603030588235295, "avg_duration_per_task_ms": 92367.94588235294, "avg_output_tokens_per_task": 4470.871764705882, "last_run_timestamp": "2026-10-31T12:50:48.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.7672726363636364, "accuracy_std_dev": 0.030278820027065213, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.43524433636363635, "avg_duration_per_task_ms": 90898.93818181819, "avg_output_tokens_per_task": 5582.549090909091, "last_run_timestamp": "2026-10-31T12:33:27.768Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.7672223333333333, "accuracy_std_dev": 0.022682791106671363, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.418396455, "avg_duration_per_task_ms": 91613.17333333334, "avg_output_tokens_per_task": 5586.18, "last_run_timestamp": "2026-10-31T11:48:47.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.7668518333333335, "accuracy_std_dev": 0.02021403460126433, "run_count": 36, "total_questions": 1800, "avg_cost_per_task_usd": 0.5330153783333336, "avg_duration_per_task_ms": 118459.865, "avg_output_tokens_per_task": 8062.295, "last_run_timestamp": "2026-10-31T12:26:06.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7666669999999999, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2750355, "avg_duration_per_task_ms": 98712.99, "avg_output_tokens_per_task": 5708.24, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "ModelRun", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01300872, "avg_duration_per_task_ms": 90405.02, "avg_output_tokens_per_task": 13808.24, "last_run_timestamp": "2026-10-29T03:35:28.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0912162, "avg_duration_per_task_ms": 191018.42, "avg_output_tokens_per_task": 9260.76, "last_run_timestamp": "2026-08-04T03:12:10.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7666668, "accuracy_std_dev": 0.007302967435227963, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.220517392, "avg_duration_per_task_ms": 158612.068, "avg_output_tokens_per_task": 23228.836, "last_run_timestamp": "2026-10-01T19:04:58.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7666667777777777, "accuracy_std_dev": 0.026851093732235272, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.14037539777777777, "avg_duration_per_task_ms": 284966.99333333335, "avg_output_tokens_per_task": 11990.22, "last_run_timestamp": "2026-10-31T19:58:31.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.76666675, "accuracy_std_dev": 0.024267078752653787, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.036940722499999995, "avg_duration_per_task_ms": 209470.17, "avg_output_tokens_per_task": 17358.2575, "last_run_timestamp": "2026-10-03T08:39:48.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08312547333333332, "avg_duration_per_task_ms": 156615.94, "avg_output_tokens_per_task": 8699.633333333333, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.010886757164351354, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.031067933333333332, "avg_duration_per_task_ms": 60278.37, "avg_output_tokens_per_task": 4752.3966666666665, "last_run_timestamp": "2026-10-31T12:25:28.648Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.023517736937610882, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04758358666666667, "avg_duration_per_task_ms": 110427.16444444444, "avg_output_tokens_per_task": 4684.946666666667, "last_run_timestamp": "2026-10-31T12:08:49.485Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.7666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.27949224, "avg_duration_per_task_ms": 363191.33, "avg_output_tokens_per_task": 35460.86, "last_run_timestamp": "2026-10-15T21:31:19.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.7666665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01352679, "avg_duration_per_task_ms": 90848.6, "avg_output_tokens_per_task": 4507.38, "last_run_timestamp": "2026-10-10T11:18:50.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.7661113333333333, "accuracy_std_dev": 0.02045003245039534, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.4937038083333333, "avg_duration_per_task_ms": 124542.97333333333, "avg_output_tokens_per_task": 8042.625, "last_run_timestamp": "2026-10-31T12:30:31.663Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.766068730941704, "accuracy_std_dev": 0.02506483163840778, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.06749501121076233, "avg_duration_per_task_ms": 62014.139013452914, "avg_output_tokens_per_task": 4517.719730941704, "last_run_timestamp": "2026-10-31T13:21:43.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.7660001999999999, "accuracy_std_dev": 0.0260256591340162, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.12242998000000002, "avg_duration_per_task_ms": 384915.524, "avg_output_tokens_per_task": 20927.716, "last_run_timestamp": "2026-10-31T13:14:15.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.766, "accuracy_std_dev": 0.036203222693566926, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.017495754, "avg_duration_per_task_ms": 289899.184, "avg_output_tokens_per_task": 11370.218, "last_run_timestamp": "2026-10-31T20:10:47.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.7659258888888889, "accuracy_std_dev": 0.02853523512137403, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02177499333333333, "avg_duration_per_task_ms": 226666.30666666667, "avg_output_tokens_per_task": 12033.74888888889, "last_run_timestamp": "2026-10-31T19:23:14.281Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.7655824146341464, "accuracy_std_dev": 0.030599644599243303, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.07519595528455283, "avg_duration_per_task_ms": 226711.57317073172, "avg_output_tokens_per_task": 10093.731707317073, "last_run_timestamp": "2026-10-10T19:17:08.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.765560294605809, "accuracy_std_dev": 0.01753797388069671, "run_count": 5, "total_questions": 241, "avg_cost_per_task_usd": 0.26006869709543573, "avg_duration_per_task_ms": 145161.65560165976, "avg_output_tokens_per_task": 7387.091286307054, "last_run_timestamp": "2026-10-31T11:39:17.172Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.7654054594594596, "accuracy_std_dev": 0.021965741528503096, "run_count": 37, "total_questions": 1850, "avg_cost_per_task_usd": 0.5855930291891892, "avg_duration_per_task_ms": 71348.61243243243, "avg_output_tokens_per_task": 5712.135675675676, "last_run_timestamp": "2026-10-31T12:33:51.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.7653333999999999, "accuracy_std_dev": 0.023999911113168737, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.088749776, "avg_duration_per_task_ms": 386636.328, "avg_output_tokens_per_task": 14600.352, "last_run_timestamp": "2026-10-31T12:27:43.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.7651853333333333, "accuracy_std_dev": 0.06605486475482165, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.2539050577777778, "avg_duration_per_task_ms": 487309.1822222222, "avg_output_tokens_per_task": 29336.053333333333, "last_run_timestamp": "2026-10-31T19:26:18.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.7650000833333334, "accuracy_std_dev": 0.026822549594381503, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.53746671, "avg_duration_per_task_ms": 174605.76, "avg_output_tokens_per_task": 10683.748333333333, "last_run_timestamp": "2026-10-31T11:55:38.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.765, "accuracy_std_dev": 0.03840572873934304, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08522504, "avg_duration_per_task_ms": 558480.23, "avg_output_tokens_per_task": 11022.365, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7647618571428572, "accuracy_std_dev": 0.016218418466392492, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1530029685714286, "avg_duration_per_task_ms": 398501.6857142857, "avg_output_tokens_per_task": 18823.014285714286, "last_run_timestamp": "2026-10-31T20:34:04.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.7647368421052632, "accuracy_std_dev": 0.0276849081598111, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.04469010842105263, "avg_duration_per_task_ms": 88120.05473684211, "avg_output_tokens_per_task": 4333.72, "last_run_timestamp": "2026-10-31T13:24:41.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.7644446666666667, "accuracy_std_dev": 0.02474571917187565, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3338584511111111, "avg_duration_per_task_ms": 643731.1911111111, "avg_output_tokens_per_task": 28077.5, "last_run_timestamp": "2026-10-31T19:49:31.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.7644444444444445, "accuracy_std_dev": 0.019116355871412038, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11003770888888888, "avg_duration_per_task_ms": 252792.1288888889, "avg_output_tokens_per_task": 19117.18222222222, "last_run_timestamp": "2026-10-31T19:46:00.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7644443333333334, "accuracy_std_dev": 0.024241503540186792, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.021445343333333335, "avg_duration_per_task_ms": 267641.68333333335, "avg_output_tokens_per_task": 15646.876666666667, "last_run_timestamp": "2026-10-31T13:19:25.796Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.7644443333333334, "accuracy_std_dev": 0.04565023765059233, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01549936, "avg_duration_per_task_ms": 297786.8, "avg_output_tokens_per_task": 16565.086666666666, "last_run_timestamp": "2026-10-28T04:10:41.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.76439777486911, "accuracy_std_dev": 0.015263579494993957, "run_count": 4, "total_questions": 191, "avg_cost_per_task_usd": 0.06284435078534031, "avg_duration_per_task_ms": 156780.0209424084, "avg_output_tokens_per_task": 9570.141361256545, "last_run_timestamp": "2026-10-31T19:43:06.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7641665000000001, "accuracy_std_dev": 0.021586383312171588, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0185151775, "avg_duration_per_task_ms": 270096.6575, "avg_output_tokens_per_task": 14323.5575, "last_run_timestamp": "2026-10-31T12:30:14.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.764, "accuracy_std_dev": 0.020483142122242875, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.080228768, "avg_duration_per_task_ms": 494377.212, "avg_output_tokens_per_task": 14669.972, "last_run_timestamp": "2026-10-31T12:27:34.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.7638889166666667, "accuracy_std_dev": 0.035219828608939255, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.140352225, "avg_duration_per_task_ms": 834808.895, "avg_output_tokens_per_task": 31424.338333333333, "last_run_timestamp": "2026-10-31T20:06:11.663Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7638888333333334, "accuracy_std_dev": 0.03795297203740732, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09057705333333334, "avg_duration_per_task_ms": 303333.575, "avg_output_tokens_per_task": 15558.463333333333, "last_run_timestamp": "2026-10-31T19:58:49.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.7637037777777778, "accuracy_std_dev": 0.021798347556015337, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06269429111111112, "avg_duration_per_task_ms": 113855.65555555555, "avg_output_tokens_per_task": 4583.311111111111, "last_run_timestamp": "2026-10-31T12:40:08.732Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7637036666666667, "accuracy_std_dev": 0.01835498758739252, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.029433213333333333, "avg_duration_per_task_ms": 122522.84, "avg_output_tokens_per_task": 12030.735555555555, "last_run_timestamp": "2026-10-31T19:08:17.847Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.7635896153846154, "accuracy_std_dev": 0.024476071585053976, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.03524474461538461, "avg_duration_per_task_ms": 67488.58923076923, "avg_output_tokens_per_task": 4789.504615384615, "last_run_timestamp": "2026-10-31T13:07:57.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.7633335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.021769780000000002, "avg_duration_per_task_ms": 1044368.78, "avg_output_tokens_per_task": 12563.69, "last_run_timestamp": "2026-08-15T08:15:52.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.763333375, "accuracy_std_dev": 0.024267095921934618, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0630558525, "avg_duration_per_task_ms": 1180279.39, "avg_output_tokens_per_task": 60139.9825, "last_run_timestamp": "2026-10-31T19:16:07.249Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.03605539719220273, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.157135485, "avg_duration_per_task_ms": 411704.8, "avg_output_tokens_per_task": 26150.13, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.013743584364622665, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01189864, "avg_duration_per_task_ms": 642869.1, "avg_output_tokens_per_task": 23140.98, "last_run_timestamp": "2026-10-24T13:28:08.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "BaseTen", "accuracy": 0.763333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020903190000000002, "avg_duration_per_task_ms": 62428.04, "avg_output_tokens_per_task": 8093.56, "last_run_timestamp": "2026-10-31T12:43:49.743Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.7630302727272728, "accuracy_std_dev": 0.03170123269496259, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.15556449636363637, "avg_duration_per_task_ms": 593095.6363636364, "avg_output_tokens_per_task": 12420.629090909091, "last_run_timestamp": "2026-10-31T13:02:37.599Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.7629022732240437, "accuracy_std_dev": 0.028478290118661355, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.01388206375227687, "avg_duration_per_task_ms": 226746.868852459, "avg_output_tokens_per_task": 15915.701275045538, "last_run_timestamp": "2026-10-17T11:00:48.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7628571428571429, "accuracy_std_dev": 0.023328310442945685, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11986269428571429, "avg_duration_per_task_ms": 415507.14, "avg_output_tokens_per_task": 18962.734285714287, "last_run_timestamp": "2026-10-03T08:33:31.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.7627778333333333, "accuracy_std_dev": 0.03313374040368652, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07174823666666666, "avg_duration_per_task_ms": 893696.3066666666, "avg_output_tokens_per_task": 51834.78666666667, "last_run_timestamp": "2026-10-27T03:58:39.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7626668, "accuracy_std_dev": 0.014966487007978884, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.009859876, "avg_duration_per_task_ms": 578544.56, "avg_output_tokens_per_task": 20173.776, "last_run_timestamp": "2026-10-31T14:48:08.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.7626666, "accuracy_std_dev": 0.02954472165785287, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.009234846, "avg_duration_per_task_ms": 195074.734, "avg_output_tokens_per_task": 4475.264, "last_run_timestamp": "2026-10-31T12:25:34.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.7625418160535117, "accuracy_std_dev": 0.02147753020972668, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.09006360702341139, "avg_duration_per_task_ms": 48936.09866220736, "avg_output_tokens_per_task": 4348.224080267559, "last_run_timestamp": "2026-10-31T12:39:14.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.7625417625418061, "accuracy_std_dev": 0.01910679234955175, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.03341546153846154, "avg_duration_per_task_ms": 144903.0602006689, "avg_output_tokens_per_task": 21777.678929765887, "last_run_timestamp": "2026-10-31T12:18:15.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.7624352026726057, "accuracy_std_dev": 0.020878013624710118, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.010567316258351892, "avg_duration_per_task_ms": 295155.5011135857, "avg_output_tokens_per_task": 12618.436525612473, "last_run_timestamp": "2026-10-31T19:48:48.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7623223984962407, "accuracy_std_dev": 0.0190669296226313, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.011861127819548874, "avg_duration_per_task_ms": 357840.4411027569, "avg_output_tokens_per_task": 17112.51127819549, "last_run_timestamp": "2026-10-31T11:47:32.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.030469494967188978, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029655940000000006, "avg_duration_per_task_ms": 256385.88, "avg_output_tokens_per_task": 18856.086666666666, "last_run_timestamp": "2026-10-08T22:27:05.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7622222916666667, "accuracy_std_dev": 0.03325923894759566, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.5556523033333334, "avg_duration_per_task_ms": 182337.15583333332, "avg_output_tokens_per_task": 10601.08, "last_run_timestamp": "2026-10-31T12:37:18.963Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.7622221111111112, "accuracy_std_dev": 0.02931314119959945, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10448193111111109, "avg_duration_per_task_ms": 237398.75333333333, "avg_output_tokens_per_task": 11214.604444444445, "last_run_timestamp": "2026-10-31T20:03:31.357Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7622099340659342, "accuracy_std_dev": 0.03802706823103689, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.09558790476190476, "avg_duration_per_task_ms": 736797.2087912088, "avg_output_tokens_per_task": 10344.02564102564, "last_run_timestamp": "2026-10-31T20:25:10.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.762177653295129, "accuracy_std_dev": 0.016008036097981727, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.027902553008595985, "avg_duration_per_task_ms": 104077.65616045846, "avg_output_tokens_per_task": 17491.627507163324, "last_run_timestamp": "2026-10-31T11:55:25.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.7621775042979944, "accuracy_std_dev": 0.026194934880387766, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.06229761318051576, "avg_duration_per_task_ms": 372098.9226361032, "avg_output_tokens_per_task": 49911.94555873926, "last_run_timestamp": "2026-10-31T19:59:18.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.7620513846153846, "accuracy_std_dev": 0.024443852124782315, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.02194166153846154, "avg_duration_per_task_ms": 173997.9723076923, "avg_output_tokens_per_task": 16407.72, "last_run_timestamp": "2026-10-31T12:11:52.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.7619782998454405, "accuracy_std_dev": 0.039147489254698645, "run_count": 13, "total_questions": 647, "avg_cost_per_task_usd": 0.06417079598145287, "avg_duration_per_task_ms": 203709.1777434312, "avg_output_tokens_per_task": 11216.542503863988, "last_run_timestamp": "2026-10-31T14:18:08.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.7617847878787878, "accuracy_std_dev": 0.03876959831491683, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.02413183333333333, "avg_duration_per_task_ms": 113963.81313131313, "avg_output_tokens_per_task": 15997.81313131313, "last_run_timestamp": "2026-10-24T12:54:33.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.761784404040404, "accuracy_std_dev": 0.046884570628299496, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.03727159090909091, "avg_duration_per_task_ms": 138498.65151515152, "avg_output_tokens_per_task": 4381.035353535353, "last_run_timestamp": "2026-10-31T12:40:46.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7616666666666667, "accuracy_std_dev": 0.018282859816110712, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.031528701666666666, "avg_duration_per_task_ms": 227802.605, "avg_output_tokens_per_task": 15291.826666666666, "last_run_timestamp": "2026-10-31T12:46:55.135Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.761666625, "accuracy_std_dev": 0.015545671961493828, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08149474, "avg_duration_per_task_ms": 103926, "avg_output_tokens_per_task": 22421.02, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.7614285714285715, "accuracy_std_dev": 0.027276150573386282, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.5232190085714287, "avg_duration_per_task_ms": 215585.69428571427, "avg_output_tokens_per_task": 9953.878571428571, "last_run_timestamp": "2026-10-31T12:37:18.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.7614249072580644, "accuracy_std_dev": 0.03191360961423825, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.04908138104838709, "avg_duration_per_task_ms": 385516.27419354836, "avg_output_tokens_per_task": 12910.963709677419, "last_run_timestamp": "2026-10-31T13:30:45.600Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.761389, "accuracy_std_dev": 0.03531505729383243, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.19697044166666666, "avg_duration_per_task_ms": 478792.93666666665, "avg_output_tokens_per_task": 27912.631666666668, "last_run_timestamp": "2026-10-31T19:55:50.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7612120909090909, "accuracy_std_dev": 0.02288640210595304, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.54388782, "avg_duration_per_task_ms": 180374.2618181818, "avg_output_tokens_per_task": 10717.156363636364, "last_run_timestamp": "2026-10-31T12:58:32.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7611111666666667, "accuracy_std_dev": 0.02274381750364017, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03547926999999999, "avg_duration_per_task_ms": 167748.38666666666, "avg_output_tokens_per_task": 16394.798333333332, "last_run_timestamp": "2026-10-31T12:45:56.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.76099995, "accuracy_std_dev": 0.018651859903170524, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.010585624, "avg_duration_per_task_ms": 322498.887, "avg_output_tokens_per_task": 14838.748, "last_run_timestamp": "2026-10-31T13:21:16.354Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7609786762860727, "accuracy_std_dev": 0.02511202768812936, "run_count": 16, "total_questions": 797, "avg_cost_per_task_usd": 0.054708953575909665, "avg_duration_per_task_ms": 274645.58720200753, "avg_output_tokens_per_task": 12041.819322459221, "last_run_timestamp": "2026-10-31T12:07:50.786Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.76083325, "accuracy_std_dev": 0.022897649686976593, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.018742625, "avg_duration_per_task_ms": 681240.3, "avg_output_tokens_per_task": 12523.52, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.7607886785714285, "accuracy_std_dev": 0.017367969160654122, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.018826165178571424, "avg_duration_per_task_ms": 120325.66071428571, "avg_output_tokens_per_task": 14433.707589285714, "last_run_timestamp": "2026-10-31T12:07:52.801Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.7607406666666667, "accuracy_std_dev": 0.034633054297881376, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.006436806666666667, "avg_duration_per_task_ms": 125547.04444444444, "avg_output_tokens_per_task": 4262.8533333333335, "last_run_timestamp": "2026-10-24T14:25:23.734Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7606666, "accuracy_std_dev": 0.023371542050964467, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.010802634, "avg_duration_per_task_ms": 1014797.92, "avg_output_tokens_per_task": 13061.106, "last_run_timestamp": "2026-10-31T20:29:45.923Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.7606666, "accuracy_std_dev": 0.03705257046198009, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.00899061, "avg_duration_per_task_ms": 185087.438, "avg_output_tokens_per_task": 15485.002, "last_run_timestamp": "2026-10-31T19:07:06.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.7606666, "accuracy_std_dev": 0.031048465341140474, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.027007422000000003, "avg_duration_per_task_ms": 150781.166, "avg_output_tokens_per_task": 8984.718, "last_run_timestamp": "2026-10-24T19:16:17.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7605362068965518, "accuracy_std_dev": 0.008806243921218638, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.007250201149425288, "avg_duration_per_task_ms": 286627.4166666667, "avg_output_tokens_per_task": 12069.367816091954, "last_run_timestamp": "2026-10-31T20:10:56.893Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.760476, "accuracy_std_dev": 0.023999188385086207, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21848949428571432, "avg_duration_per_task_ms": 711094.3942857143, "avg_output_tokens_per_task": 57307.56, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.7604166354166666, "accuracy_std_dev": 0.010476531320050537, "run_count": 4, "total_questions": 192, "avg_cost_per_task_usd": 0.29691326041666666, "avg_duration_per_task_ms": 310718.984375, "avg_output_tokens_per_task": 7030.182291666667, "last_run_timestamp": "2026-10-31T12:13:14.359Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.7603703333333334, "accuracy_std_dev": 0.033126907327897526, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08524436222222224, "avg_duration_per_task_ms": 566210.18, "avg_output_tokens_per_task": 12520.904444444444, "last_run_timestamp": "2026-10-31T12:26:16.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.7601351621621621, "accuracy_std_dev": 0.012280818789432917, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.032259290540540546, "avg_duration_per_task_ms": 475460.4324324324, "avg_output_tokens_per_task": 12381.702702702703, "last_run_timestamp": "2026-10-31T12:56:10.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "GMICloud", "accuracy": 0.7600002499999999, "accuracy_std_dev": 0.008164965810553043, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015787874999999996, "avg_duration_per_task_ms": 119015.24, "avg_output_tokens_per_task": 12260.915, "last_run_timestamp": "2026-10-31T13:06:19.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Alibaba", "accuracy": 0.7600002499999999, "accuracy_std_dev": 0.027080189561513434, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02989183, "avg_duration_per_task_ms": 125270.795, "avg_output_tokens_per_task": 12269.195, "last_run_timestamp": "2026-10-31T12:01:23.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.7600001999999999, "accuracy_std_dev": 0.03126929626582598, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.114411384, "avg_duration_per_task_ms": 874872.748, "avg_output_tokens_per_task": 24203.156, "last_run_timestamp": "2026-10-31T12:44:40.958Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.7600001, "accuracy_std_dev": 0.02905935688018578, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.045049942, "avg_duration_per_task_ms": 188166.976, "avg_output_tokens_per_task": 9673.868, "last_run_timestamp": "2026-10-31T20:17:26.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.054461335, "avg_duration_per_task_ms": 91417, "avg_output_tokens_per_task": 15101.33, "last_run_timestamp": "2026-10-03T08:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.76, "accuracy_std_dev": 0.014401594564491825, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.019761673333333334, "avg_duration_per_task_ms": 291193.25333333336, "avg_output_tokens_per_task": 17811.393333333333, "last_run_timestamp": "2026-10-10T11:23:23.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01597332, "avg_duration_per_task_ms": 42256.58, "avg_output_tokens_per_task": 1764.66, "last_run_timestamp": "2026-10-01T23:50:45.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.3443486, "avg_duration_per_task_ms": 184596.7, "avg_output_tokens_per_task": 12502.76, "last_run_timestamp": "2026-10-01T19:04:40.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Venice", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19103682, "avg_duration_per_task_ms": 233230.24, "avg_output_tokens_per_task": 15236.66, "last_run_timestamp": "2026-08-04T03:12:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.76, "accuracy_std_dev": 0.005443582705045132, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.40005456666666667, "avg_duration_per_task_ms": 394539.76666666666, "avg_output_tokens_per_task": 34690.113333333335, "last_run_timestamp": "2026-10-02T23:12:57.355Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.76, "accuracy_std_dev": 0.025991443038046205, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.08891490399999999, "avg_duration_per_task_ms": 301406.882, "avg_output_tokens_per_task": 12044.934, "last_run_timestamp": "2026-10-31T12:46:14.833Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Cohere", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10151575999999998, "avg_duration_per_task_ms": 434936.03, "avg_output_tokens_per_task": 13962.46, "last_run_timestamp": "2026-10-24T12:32:07.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.7599999000000001, "accuracy_std_dev": 0.030840204640858015, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.008153706000000002, "avg_duration_per_task_ms": 200971.108, "avg_output_tokens_per_task": 4219.412, "last_run_timestamp": "2026-10-31T12:49:39.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.759846288184438, "accuracy_std_dev": 0.015452349289717374, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.15869831700288184, "avg_duration_per_task_ms": 791662.6916426513, "avg_output_tokens_per_task": 30669.08069164265, "last_run_timestamp": "2026-10-31T19:07:30.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.7594444166666666, "accuracy_std_dev": 0.033715523315080306, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09267692833333334, "avg_duration_per_task_ms": 244983.77333333335, "avg_output_tokens_per_task": 12483.865, "last_run_timestamp": "2026-10-31T12:53:55.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.7592155882352941, "accuracy_std_dev": 0.027618637596970693, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.011689743529411765, "avg_duration_per_task_ms": 166650.9011764706, "avg_output_tokens_per_task": 5474.670588235294, "last_run_timestamp": "2026-10-31T12:46:12.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7591665000000001, "accuracy_std_dev": 0.003632454025861855, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.09515249499999999, "avg_duration_per_task_ms": 195938.94, "avg_output_tokens_per_task": 9656.23, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.7587878181818182, "accuracy_std_dev": 0.01653114932933463, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.17482494545454544, "avg_duration_per_task_ms": 315756.4018181818, "avg_output_tokens_per_task": 10740.534545454546, "last_run_timestamp": "2026-10-31T11:13:27.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.7585185555555556, "accuracy_std_dev": 0.020793455498792097, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.027907260000000003, "avg_duration_per_task_ms": 202475.69777777776, "avg_output_tokens_per_task": 13617.924444444445, "last_run_timestamp": "2026-10-31T19:55:08.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.758333375, "accuracy_std_dev": 0.012360224066916245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.034803532500000005, "avg_duration_per_task_ms": 275650.13, "avg_output_tokens_per_task": 13717.48, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.7583333333333333, "accuracy_std_dev": 0.03125451671181126, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06620967333333333, "avg_duration_per_task_ms": 161431.49, "avg_output_tokens_per_task": 10316.483333333334, "last_run_timestamp": "2026-10-31T19:43:05.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.014433785597946933, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.42515429, "avg_duration_per_task_ms": 460450.54, "avg_output_tokens_per_task": 34342.89, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.7580951428571429, "accuracy_std_dev": 0.016605241912709113, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.048255082857142854, "avg_duration_per_task_ms": 69808.27142857143, "avg_output_tokens_per_task": 4328.277142857143, "last_run_timestamp": "2026-10-31T12:44:24.094Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.7579487692307693, "accuracy_std_dev": 0.03142044042230091, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.019486126153846157, "avg_duration_per_task_ms": 195283.25692307693, "avg_output_tokens_per_task": 14827.041538461539, "last_run_timestamp": "2026-10-31T12:48:59.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7578806257545272, "accuracy_std_dev": 0.03401018014418623, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.10939716700201207, "avg_duration_per_task_ms": 158414.4527162978, "avg_output_tokens_per_task": 9282.295774647888, "last_run_timestamp": "2026-10-31T19:46:27.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.7577777777777778, "accuracy_std_dev": 0.026480824650543643, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.025315553333333334, "avg_duration_per_task_ms": 238703.38, "avg_output_tokens_per_task": 12990.084444444445, "last_run_timestamp": "2026-10-31T19:02:15.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.7577777777777778, "accuracy_std_dev": 0.01859267080077988, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1710706822222222, "avg_duration_per_task_ms": 189011.57111111112, "avg_output_tokens_per_task": 11157.495555555555, "last_run_timestamp": "2026-10-31T19:18:33.522Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.75777775, "accuracy_std_dev": 0.03177850620866512, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.5383182933333334, "avg_duration_per_task_ms": 190201.31833333333, "avg_output_tokens_per_task": 10680.48, "last_run_timestamp": "2026-10-31T12:48:37.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "ParetoAlpha", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.013698571612479267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.28612407999999995, "avg_duration_per_task_ms": 2065607.3133333332, "avg_output_tokens_per_task": 10347.066666666668, "last_run_timestamp": "2026-10-31T11:54:15.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.7577286370967742, "accuracy_std_dev": 0.0230900961972877, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.01778327016129032, "avg_duration_per_task_ms": 195849.37096774194, "avg_output_tokens_per_task": 5723.635080645161, "last_run_timestamp": "2026-10-31T12:36:56.960Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.7576190714285714, "accuracy_std_dev": 0.02056452651139227, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.021698402857142855, "avg_duration_per_task_ms": 123305.51285714285, "avg_output_tokens_per_task": 16890.44, "last_run_timestamp": "2026-10-31T12:41:20.328Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.757619, "accuracy_std_dev": 0.020871170540108334, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.0069606071428571425, "avg_duration_per_task_ms": 264179.4914285714, "avg_output_tokens_per_task": 15717.655714285715, "last_run_timestamp": "2026-10-31T13:22:04.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.7575, "accuracy_std_dev": 0.05171382256708548, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.26214604, "avg_duration_per_task_ms": 26048.875, "avg_output_tokens_per_task": 12109.95, "last_run_timestamp": "2026-10-31T11:18:07.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7575, "accuracy_std_dev": 0.023078766095699303, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.012481125, "avg_duration_per_task_ms": 215467.87, "avg_output_tokens_per_task": 4890.4975, "last_run_timestamp": "2026-10-31T11:56:22.568Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.03991088294989224, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023026752, "avg_duration_per_task_ms": 161149.652, "avg_output_tokens_per_task": 18473.992, "last_run_timestamp": "2026-10-03T08:55:31.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.7572916875, "accuracy_std_dev": 0.027315378536455315, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.090318465, "avg_duration_per_task_ms": 189685.91875, "avg_output_tokens_per_task": 11015.33625, "last_run_timestamp": "2026-10-31T19:54:53.711Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.757037, "accuracy_std_dev": 0.024966444422686845, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.022134202222222222, "avg_duration_per_task_ms": 462417.70666666667, "avg_output_tokens_per_task": 13881.615555555556, "last_run_timestamp": "2026-10-31T19:04:16.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.756708080495356, "accuracy_std_dev": 0.027120750325406145, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.19143365634674925, "avg_duration_per_task_ms": 498299.07585139317, "avg_output_tokens_per_task": 20833.128482972137, "last_run_timestamp": "2026-10-31T14:12:32.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "auto-routing", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08839812, "avg_duration_per_task_ms": 73380.11, "avg_output_tokens_per_task": 8719.83, "last_run_timestamp": "2026-10-31T12:44:24.100Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "AtlasCloud", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06446149, "avg_duration_per_task_ms": 318929.97, "avg_output_tokens_per_task": 13284.66, "last_run_timestamp": "2026-10-31T12:25:36.318Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.7566668, "accuracy_std_dev": 0.01819650760338369, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.151380274, "avg_duration_per_task_ms": 165330.14, "avg_output_tokens_per_task": 19976.098, "last_run_timestamp": "2026-10-31T12:20:13.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.7566666666666667, "accuracy_std_dev": 0.022026835486641215, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06927284666666667, "avg_duration_per_task_ms": 313911.26, "avg_output_tokens_per_task": 14319.526666666667, "last_run_timestamp": "2026-10-31T12:43:48.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.7564444666666666, "accuracy_std_dev": 0.023958871344776582, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.04415017200000001, "avg_duration_per_task_ms": 138405.18933333334, "avg_output_tokens_per_task": 4510.322666666667, "last_run_timestamp": "2026-10-31T12:43:47.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7563351081871345, "accuracy_std_dev": 0.013338594901982123, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.033182684210526316, "avg_duration_per_task_ms": 2146298.769005848, "avg_output_tokens_per_task": 50213.23391812865, "last_run_timestamp": "2026-10-09T04:05:56.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7562962222222223, "accuracy_std_dev": 0.028650564911156574, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.18542253777777779, "avg_duration_per_task_ms": 171878.6222222222, "avg_output_tokens_per_task": 9625.26888888889, "last_run_timestamp": "2026-10-31T20:09:19.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.7562961111111112, "accuracy_std_dev": 0.02966657881426859, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03134207111111111, "avg_duration_per_task_ms": 175492.92222222223, "avg_output_tokens_per_task": 14182.893333333333, "last_run_timestamp": "2026-10-31T11:54:37.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.7558333333333334, "accuracy_std_dev": 0.021131098726495254, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.01632131833333333, "avg_duration_per_task_ms": 115260.22166666666, "avg_output_tokens_per_task": 4430.61, "last_run_timestamp": "2026-10-31T12:09:39.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.7557575454545454, "accuracy_std_dev": 0.021134116315326572, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.0626866909090909, "avg_duration_per_task_ms": 166564.66363636364, "avg_output_tokens_per_task": 4635.490909090909, "last_run_timestamp": "2026-10-31T13:36:23.199Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.7555554444444444, "accuracy_std_dev": 0.018856167739030906, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0086613, "avg_duration_per_task_ms": 117282.64, "avg_output_tokens_per_task": 5380.108888888889, "last_run_timestamp": "2026-10-31T13:20:27.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.7554545454545455, "accuracy_std_dev": 0.03830433230509108, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.16801238363636364, "avg_duration_per_task_ms": 309650.71636363637, "avg_output_tokens_per_task": 24145.685454545455, "last_run_timestamp": "2026-10-31T13:11:05.620Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7552378571428572, "accuracy_std_dev": 0.014126057041900291, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.009880265714285713, "avg_duration_per_task_ms": 245506.26571428572, "avg_output_tokens_per_task": 16074.99142857143, "last_run_timestamp": "2026-10-31T12:27:44.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7552085625, "accuracy_std_dev": 0.029782500000000045, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.02997340625, "avg_duration_per_task_ms": 1679708.6145833333, "avg_output_tokens_per_task": 72130.22916666667, "last_run_timestamp": "2026-10-31T20:01:41.151Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Modal", "accuracy": 0.7551024285714285, "accuracy_std_dev": 0.011805499999999969, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.27501580612244897, "avg_duration_per_task_ms": 65997.72448979592, "avg_output_tokens_per_task": 8842.591836734693, "last_run_timestamp": "2026-10-31T12:36:21.331Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7550476285714286, "accuracy_std_dev": 0.02909609450973075, "run_count": 35, "total_questions": 1750, "avg_cost_per_task_usd": 0.20630642971428564, "avg_duration_per_task_ms": 116033.43085714286, "avg_output_tokens_per_task": 7739.464571428572, "last_run_timestamp": "2026-10-31T12:48:39.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.7550336026845637, "accuracy_std_dev": 0.024862145639416477, "run_count": 15, "total_questions": 745, "avg_cost_per_task_usd": 0.08558708322147651, "avg_duration_per_task_ms": 635566.877852349, "avg_output_tokens_per_task": 21211.187919463086, "last_run_timestamp": "2026-10-31T13:12:07.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.7550335167785235, "accuracy_std_dev": 0.023364430687122125, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.012232758389261746, "avg_duration_per_task_ms": 267603.8926174497, "avg_output_tokens_per_task": 4166.208053691275, "last_run_timestamp": "2026-10-31T12:14:04.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.755, "accuracy_std_dev": 0.008333000000000035, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07072132, "avg_duration_per_task_ms": 164484.88, "avg_output_tokens_per_task": 7418.93, "last_run_timestamp": "2026-08-15T08:13:33.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7549998, "accuracy_std_dev": 0.029636688974310207, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.014858368, "avg_duration_per_task_ms": 71120.474, "avg_output_tokens_per_task": 4319.272, "last_run_timestamp": "2026-10-31T11:35:00.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7547041935483871, "accuracy_std_dev": 0.023888446777469652, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.01950188306451613, "avg_duration_per_task_ms": 216554.71370967742, "avg_output_tokens_per_task": 4241.487903225807, "last_run_timestamp": "2026-10-31T11:51:07.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.7542953608247424, "accuracy_std_dev": 0.02652500000000002, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.021299783505154636, "avg_duration_per_task_ms": 633495.5257731959, "avg_output_tokens_per_task": 26267.113402061856, "last_run_timestamp": "2026-08-08T08:30:53.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.7541725722460657, "accuracy_std_dev": 0.027627708682145743, "run_count": 14, "total_questions": 699, "avg_cost_per_task_usd": 0.0887023748211731, "avg_duration_per_task_ms": 129758.05722460659, "avg_output_tokens_per_task": 12053.809728183118, "last_run_timestamp": "2026-10-31T18:56:42.115Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.75416675, "accuracy_std_dev": 0.019096569464219906, "run_count": 13, "total_questions": 640, "avg_cost_per_task_usd": 0.0324113640625, "avg_duration_per_task_ms": 163294.796875, "avg_output_tokens_per_task": 15840.8265625, "last_run_timestamp": "2026-10-31T11:41:35.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.7538461538461538, "accuracy_std_dev": 0.029405579849477954, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.008398384615384618, "avg_duration_per_task_ms": 128001.65076923076, "avg_output_tokens_per_task": 4472.704615384615, "last_run_timestamp": "2026-10-31T13:18:32.761Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.75375, "accuracy_std_dev": 0.02086800226622888, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.2756704208333333, "avg_duration_per_task_ms": 243435.00416666668, "avg_output_tokens_per_task": 11821.2175, "last_run_timestamp": "2026-10-31T12:36:58.963Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.75375, "accuracy_std_dev": 0.02114362919711279, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.010135285, "avg_duration_per_task_ms": 220308.23, "avg_output_tokens_per_task": 4260.24875, "last_run_timestamp": "2026-10-31T13:20:23.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.753611125, "accuracy_std_dev": 0.03106020227224396, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.20096273666666667, "avg_duration_per_task_ms": 97219.05666666667, "avg_output_tokens_per_task": 7598.600833333333, "last_run_timestamp": "2026-10-31T12:38:13.068Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.16966541999999996, "avg_duration_per_task_ms": 147439, "avg_output_tokens_per_task": 22903.86, "last_run_timestamp": "2026-10-01T19:04:26.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7533335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.13189366, "avg_duration_per_task_ms": 651730.99, "avg_output_tokens_per_task": 24955.07, "last_run_timestamp": "2026-08-15T08:15:35.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33087047999999997, "avg_duration_per_task_ms": 241177.17, "avg_output_tokens_per_task": 11982.77, "last_run_timestamp": "2026-10-01T16:01:02.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "GMICloud", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07113415, "avg_duration_per_task_ms": 677434.95, "avg_output_tokens_per_task": 13100.77, "last_run_timestamp": "2026-10-31T13:06:57.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.75333348, "accuracy_std_dev": 0.022784037298284095, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.24681676960000004, "avg_duration_per_task_ms": 219719.8136, "avg_output_tokens_per_task": 10167.9728, "last_run_timestamp": "2026-10-31T12:43:33.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7533334166666666, "accuracy_std_dev": 0.10114155829781209, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.15308477166666667, "avg_duration_per_task_ms": 578170.1783333333, "avg_output_tokens_per_task": 28402.441666666666, "last_run_timestamp": "2026-10-31T19:37:30.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.01721340993398913, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.33945626, "avg_duration_per_task_ms": 116389.55333333333, "avg_output_tokens_per_task": 7044.336666666667, "last_run_timestamp": "2026-10-31T12:36:01.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.03925236487426234, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03720142666666666, "avg_duration_per_task_ms": 244561.18666666668, "avg_output_tokens_per_task": 13729.346666666666, "last_run_timestamp": "2026-10-31T19:28:27.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.02277092037860767, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.020851383333333334, "avg_duration_per_task_ms": 133350.03333333333, "avg_output_tokens_per_task": 13555.86, "last_run_timestamp": "2026-10-31T19:21:19.082Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018967786666666667, "avg_duration_per_task_ms": 85971.64666666667, "avg_output_tokens_per_task": 11730.173333333334, "last_run_timestamp": "2026-10-31T12:40:46.055Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.029186609056590034, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09943221, "avg_duration_per_task_ms": 284246.225, "avg_output_tokens_per_task": 12177.723333333333, "last_run_timestamp": "2026-10-31T12:41:58.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7533332857142857, "accuracy_std_dev": 0.015118704911601446, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3460215285714286, "avg_duration_per_task_ms": 119668.60857142857, "avg_output_tokens_per_task": 7072.9971428571425, "last_run_timestamp": "2026-10-31T12:24:55.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.7533332222222222, "accuracy_std_dev": 0.028109090837812517, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01274101777777778, "avg_duration_per_task_ms": 229472.85111111111, "avg_output_tokens_per_task": 13907.048888888889, "last_run_timestamp": "2026-10-31T20:16:35.156Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02472732, "avg_duration_per_task_ms": 1279934.76, "avg_output_tokens_per_task": 22994.42, "last_run_timestamp": "2026-08-08T08:29:47.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Decart", "accuracy": 0.753333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01197713, "avg_duration_per_task_ms": 212560.45, "avg_output_tokens_per_task": 13957.42, "last_run_timestamp": "2026-10-24T11:49:49.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-codex-20260224", "model_slug": "openai/gpt-5.3-codex", "model_name": "OpenAI: GPT-5.3-Codex", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29971074000000003, "avg_duration_per_task_ms": 180697.5, "avg_output_tokens_per_task": 13569.24, "last_run_timestamp": "2026-10-02T23:12:05.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.7531531351351352, "accuracy_std_dev": 0.03358419418792134, "run_count": 37, "total_questions": 1850, "avg_cost_per_task_usd": 0.21483056864864872, "avg_duration_per_task_ms": 93416.17405405405, "avg_output_tokens_per_task": 7484.544324324324, "last_run_timestamp": "2026-10-31T13:20:20.506Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.7531401420289855, "accuracy_std_dev": 0.02334674149443907, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.0109654, "avg_duration_per_task_ms": 132639.57101449274, "avg_output_tokens_per_task": 12361.4, "last_run_timestamp": "2026-10-24T19:21:18.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7528572857142857, "accuracy_std_dev": 0.02855957373238457, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.3433513242857144, "avg_duration_per_task_ms": 226520.09285714285, "avg_output_tokens_per_task": 10840.655714285715, "last_run_timestamp": "2026-10-31T12:24:26.627Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.7528204615384616, "accuracy_std_dev": 0.022412150364050934, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.00771266, "avg_duration_per_task_ms": 231992.75692307693, "avg_output_tokens_per_task": 14548.033846153847, "last_run_timestamp": "2026-10-31T13:30:46.598Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.7527778333333334, "accuracy_std_dev": 0.030118007616798902, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.20071909666666668, "avg_duration_per_task_ms": 119608.17166666666, "avg_output_tokens_per_task": 7765.845, "last_run_timestamp": "2026-10-31T11:54:55.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.7527778333333334, "accuracy_std_dev": 0.03696434635301368, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.19816059333333333, "avg_duration_per_task_ms": 99704.65333333334, "avg_output_tokens_per_task": 7614.635, "last_run_timestamp": "2026-10-31T11:28:40.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.7527776666666667, "accuracy_std_dev": 0.016206541129707153, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03677672666666666, "avg_duration_per_task_ms": 268844.9166666667, "avg_output_tokens_per_task": 21482.343333333334, "last_run_timestamp": "2026-10-31T20:23:01.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7527272727272727, "accuracy_std_dev": 0.029875735292863623, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.013207607272727274, "avg_duration_per_task_ms": 136477.7381818182, "avg_output_tokens_per_task": 16262.556363636364, "last_run_timestamp": "2026-10-24T11:51:53.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Fireworks", "accuracy": 0.752715030075188, "accuracy_std_dev": 0.009940333684036977, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.02512772681704261, "avg_duration_per_task_ms": 75665.01253132832, "avg_output_tokens_per_task": 12251.558897243107, "last_run_timestamp": "2026-10-31T12:26:39.122Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.7526667, "accuracy_std_dev": 0.02374861961062159, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.007248376000000002, "avg_duration_per_task_ms": 194889.812, "avg_output_tokens_per_task": 4384.802, "last_run_timestamp": "2026-10-31T13:46:40.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7525925555555556, "accuracy_std_dev": 0.029049942333509857, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02899676666666667, "avg_duration_per_task_ms": 458008.2733333333, "avg_output_tokens_per_task": 11690.06, "last_run_timestamp": "2026-10-31T19:35:25.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7523809047619048, "accuracy_std_dev": 0.017612644913472624, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.36889707904761904, "avg_duration_per_task_ms": 141180.9876190476, "avg_output_tokens_per_task": 7127.063809523809, "last_run_timestamp": "2026-10-31T12:26:17.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7523076923076923, "accuracy_std_dev": 0.0332661532107905, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.10102499846153845, "avg_duration_per_task_ms": 302464.04, "avg_output_tokens_per_task": 18714.755384615386, "last_run_timestamp": "2026-10-31T13:11:23.394Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.7522223333333333, "accuracy_std_dev": 0.021914618269294344, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07768509666666666, "avg_duration_per_task_ms": 545757.0816666667, "avg_output_tokens_per_task": 35147.68, "last_run_timestamp": "2026-10-31T20:08:33.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.75222225, "accuracy_std_dev": 0.028393307985406832, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.5304961100000001, "avg_duration_per_task_ms": 187932.66333333333, "avg_output_tokens_per_task": 10407.706666666667, "last_run_timestamp": "2026-10-31T11:53:29.846Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.7519260539291217, "accuracy_std_dev": 0.019651146282719818, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.027055047765793528, "avg_duration_per_task_ms": 539313.875192604, "avg_output_tokens_per_task": 22031.878274268103, "last_run_timestamp": "2026-10-31T19:23:15.539Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.751852, "accuracy_std_dev": 0.019316204987292706, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.014403708888888889, "avg_duration_per_task_ms": 237123.64, "avg_output_tokens_per_task": 14731.191111111111, "last_run_timestamp": "2026-10-31T19:19:01.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Nebius", "accuracy": 0.7516778724832214, "accuracy_std_dev": 0.030780262062995275, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.08526638255033557, "avg_duration_per_task_ms": 172059.79865771811, "avg_output_tokens_per_task": 8282.208053691274, "last_run_timestamp": "2026-10-31T11:50:00.858Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.7516667499999999, "accuracy_std_dev": 0.01723988431479461, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.066217935, "avg_duration_per_task_ms": 295605.795, "avg_output_tokens_per_task": 8579.39, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.751666625, "accuracy_std_dev": 0.046874494071769725, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11106282249999999, "avg_duration_per_task_ms": 257380.8975, "avg_output_tokens_per_task": 11828.0925, "last_run_timestamp": "2026-10-24T19:31:21.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.7514287142857142, "accuracy_std_dev": 0.02844421520758676, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.032982819999999996, "avg_duration_per_task_ms": 221535.9714285714, "avg_output_tokens_per_task": 35206.688571428575, "last_run_timestamp": "2026-10-31T11:46:18.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7512689898477157, "accuracy_std_dev": 0.021764084474197413, "run_count": 4, "total_questions": 197, "avg_cost_per_task_usd": 0.12904325888324872, "avg_duration_per_task_ms": 273181.16751269036, "avg_output_tokens_per_task": 46030.85279187817, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7511111111111111, "accuracy_std_dev": 0.03744961388747538, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04471753777777778, "avg_duration_per_task_ms": 182484.97555555555, "avg_output_tokens_per_task": 9416.271111111111, "last_run_timestamp": "2026-10-31T20:15:42.614Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.7506668, "accuracy_std_dev": 0.025854254755455656, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.264866352, "avg_duration_per_task_ms": 264537.464, "avg_output_tokens_per_task": 10440.212, "last_run_timestamp": "2026-10-24T11:49:28.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.7506666, "accuracy_std_dev": 0.03785357285435553, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.007070539999999999, "avg_duration_per_task_ms": 292989.264, "avg_output_tokens_per_task": 3954.392, "last_run_timestamp": "2026-10-31T13:04:37.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7505555833333333, "accuracy_std_dev": 0.02754919561577778, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.01935584833333333, "avg_duration_per_task_ms": 217986.275, "avg_output_tokens_per_task": 15601.298333333334, "last_run_timestamp": "2026-10-31T12:32:12.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7504761428571428, "accuracy_std_dev": 0.04779025223509368, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.020752462857142853, "avg_duration_per_task_ms": 333009.8742857143, "avg_output_tokens_per_task": 11171.528571428571, "last_run_timestamp": "2026-10-31T13:00:18.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.7503704444444445, "accuracy_std_dev": 0.0275968517733902, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.025161266666666664, "avg_duration_per_task_ms": 319969.4244444444, "avg_output_tokens_per_task": 10853.25111111111, "last_run_timestamp": "2026-10-31T19:15:14.763Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29130333999999997, "avg_duration_per_task_ms": 176962.37, "avg_output_tokens_per_task": 10386.26, "last_run_timestamp": "2026-10-01T19:04:36.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.75, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01952235, "avg_duration_per_task_ms": 2267441.92, "avg_output_tokens_per_task": 19667.19, "last_run_timestamp": "2026-10-03T08:13:52.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.75, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08838628, "avg_duration_per_task_ms": 204069.19, "avg_output_tokens_per_task": 7577, "last_run_timestamp": "2026-08-15T08:10:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7499999999999999, "accuracy_std_dev": 0.03349968490896593, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.028200379999999997, "avg_duration_per_task_ms": 163955.21, "avg_output_tokens_per_task": 15323.823333333334, "last_run_timestamp": "2026-10-31T12:28:07.595Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.749524, "accuracy_std_dev": 0.021633867377662402, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.35241304, "avg_duration_per_task_ms": 87048.52428571429, "avg_output_tokens_per_task": 6993.714285714285, "last_run_timestamp": "2026-10-31T11:29:30.760Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.7495238571428572, "accuracy_std_dev": 0.021338220105372136, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.016605834285714285, "avg_duration_per_task_ms": 72972.84285714285, "avg_output_tokens_per_task": 4253.202857142857, "last_run_timestamp": "2026-10-31T12:03:51.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7494445555555556, "accuracy_std_dev": 0.024107483087258824, "run_count": 36, "total_questions": 1800, "avg_cost_per_task_usd": 0.37011108833333334, "avg_duration_per_task_ms": 153729.88, "avg_output_tokens_per_task": 11750.439444444444, "last_run_timestamp": "2026-10-31T12:39:28.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.7493334, "accuracy_std_dev": 0.035427579965896655, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.014355716000000001, "avg_duration_per_task_ms": 155681.656, "avg_output_tokens_per_task": 14723.468, "last_run_timestamp": "2026-10-03T16:15:22.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.7493333, "accuracy_std_dev": 0.015549210771289982, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.012228522, "avg_duration_per_task_ms": 315949.068, "avg_output_tokens_per_task": 16300.002, "last_run_timestamp": "2026-10-31T11:27:38.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7492774046242774, "accuracy_std_dev": 0.025230016427736986, "run_count": 14, "total_questions": 692, "avg_cost_per_task_usd": 0.013838482658959535, "avg_duration_per_task_ms": 56300.32225433526, "avg_output_tokens_per_task": 4085.725433526012, "last_run_timestamp": "2026-10-31T12:14:05.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.7492306923076923, "accuracy_std_dev": 0.034644866837859514, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.031206456923076928, "avg_duration_per_task_ms": 254745.31692307693, "avg_output_tokens_per_task": 16975.855384615385, "last_run_timestamp": "2026-10-31T12:25:12.423Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.749091, "accuracy_std_dev": 0.0224325111713902, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1957560618181818, "avg_duration_per_task_ms": 114755.80545454545, "avg_output_tokens_per_task": 7618.518181818182, "last_run_timestamp": "2026-10-31T11:08:14.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.7490679239373601, "accuracy_std_dev": 0.028614120182218948, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.09606371140939597, "avg_duration_per_task_ms": 393542.0715883669, "avg_output_tokens_per_task": 24831.977628635348, "last_run_timestamp": "2026-10-31T12:39:48.592Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.748889, "accuracy_std_dev": 0.015713326891527487, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03620722, "avg_duration_per_task_ms": 114735.25333333333, "avg_output_tokens_per_task": 14087.753333333334, "last_run_timestamp": "2026-10-31T11:35:01.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.7488888333333333, "accuracy_std_dev": 0.03852200108992549, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.04173693, "avg_duration_per_task_ms": 185285.73166666666, "avg_output_tokens_per_task": 18834.241666666665, "last_run_timestamp": "2026-10-31T12:26:48.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.7487437738693468, "accuracy_std_dev": 0.05136696424259468, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.08314239447236181, "avg_duration_per_task_ms": 784494.41959799, "avg_output_tokens_per_task": 50526.82914572865, "last_run_timestamp": "2026-10-31T18:44:20.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.7485714285714286, "accuracy_std_dev": 0.023560522731632144, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.34892749714285715, "avg_duration_per_task_ms": 98456.37428571428, "avg_output_tokens_per_task": 7086.754285714285, "last_run_timestamp": "2026-10-31T11:27:20.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.7483334166666666, "accuracy_std_dev": 0.027772250006251255, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.02606768166666667, "avg_duration_per_task_ms": 149836.03333333333, "avg_output_tokens_per_task": 9361.645, "last_run_timestamp": "2026-10-31T19:14:12.566Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Wafer", "accuracy": 0.748333375, "accuracy_std_dev": 0.024664456546503833, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0092696725, "avg_duration_per_task_ms": 91481.9675, "avg_output_tokens_per_task": 13410.6275, "last_run_timestamp": "2026-10-31T13:03:50.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.748333375, "accuracy_std_dev": 0.018782375754530515, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0418515775, "avg_duration_per_task_ms": 191146.0675, "avg_output_tokens_per_task": 12827.9725, "last_run_timestamp": "2026-10-03T08:33:09.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.7483333333333333, "accuracy_std_dev": 0.028037772142633272, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.50234278, "avg_duration_per_task_ms": 193180.515, "avg_output_tokens_per_task": 10059.131666666666, "last_run_timestamp": "2026-10-31T12:10:56.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7483333, "accuracy_std_dev": 0.02761851834566802, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.045017131999999994, "avg_duration_per_task_ms": 459054.072, "avg_output_tokens_per_task": 23025.238, "last_run_timestamp": "2026-10-24T19:38:23.863Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Modal", "accuracy": 0.748322067114094, "accuracy_std_dev": 0.02148614665313445, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.029814261744966444, "avg_duration_per_task_ms": 257681.3489932886, "avg_output_tokens_per_task": 10900.818791946309, "last_run_timestamp": "2026-10-31T12:26:05.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.7481482222222222, "accuracy_std_dev": 0.020068464436510978, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.19089935333333333, "avg_duration_per_task_ms": 286228.22222222225, "avg_output_tokens_per_task": 11572.273333333333, "last_run_timestamp": "2026-10-31T12:37:16.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7480002, "accuracy_std_dev": 0.020396130352593862, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030171100000000003, "avg_duration_per_task_ms": 197857.312, "avg_output_tokens_per_task": 18663.832, "last_run_timestamp": "2026-10-03T08:18:33.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.74791675, "accuracy_std_dev": 0.031222055284486006, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01487202, "avg_duration_per_task_ms": 51526.7675, "avg_output_tokens_per_task": 4120.2525, "last_run_timestamp": "2026-10-31T13:13:13.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.7478510515759312, "accuracy_std_dev": 0.031000808296901307, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.018540833810888252, "avg_duration_per_task_ms": 126193.2005730659, "avg_output_tokens_per_task": 4035.12893982808, "last_run_timestamp": "2026-10-31T11:27:58.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.7477776666666667, "accuracy_std_dev": 0.028393017748891882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.020487799999999997, "avg_duration_per_task_ms": 112394.16, "avg_output_tokens_per_task": 6879.653333333334, "last_run_timestamp": "2026-10-31T12:42:03.915Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.747619, "accuracy_std_dev": 0.025305279849978234, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.33171734571428574, "avg_duration_per_task_ms": 235739.5942857143, "avg_output_tokens_per_task": 12164.34, "last_run_timestamp": "2026-10-01T16:01:04.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7474074444444444, "accuracy_std_dev": 0.021417470403134346, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06300137777777777, "avg_duration_per_task_ms": 204272.4, "avg_output_tokens_per_task": 11398.04, "last_run_timestamp": "2026-10-31T19:58:43.811Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7473684210526316, "accuracy_std_dev": 0.02465141176979054, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.02123823368421052, "avg_duration_per_task_ms": 106810.42210526316, "avg_output_tokens_per_task": 14307.62947368421, "last_run_timestamp": "2026-10-31T13:23:12.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.7472221666666666, "accuracy_std_dev": 0.011613686586906374, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17439429333333334, "avg_duration_per_task_ms": 161420.59666666668, "avg_output_tokens_per_task": 12499.9, "last_run_timestamp": "2026-10-10T11:19:48.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7468778674351586, "accuracy_std_dev": 0.03317731677283287, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.06897791354466859, "avg_duration_per_task_ms": 486659.6887608069, "avg_output_tokens_per_task": 10547.132564841499, "last_run_timestamp": "2026-10-24T19:42:30.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7467730182232346, "accuracy_std_dev": 0.0416315871638084, "run_count": 9, "total_questions": 439, "avg_cost_per_task_usd": 0.11620976082004557, "avg_duration_per_task_ms": 452232.8268792711, "avg_output_tokens_per_task": 21963.851936218678, "last_run_timestamp": "2026-10-31T19:29:13.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.746667, "accuracy_std_dev": 0.014142135623730963, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01997175, "avg_duration_per_task_ms": 504783.88, "avg_output_tokens_per_task": 14879.43, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Relace", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00930136, "avg_duration_per_task_ms": 89811.58, "avg_output_tokens_per_task": 12020.36, "last_run_timestamp": "2026-10-24T15:18:33.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07074242, "avg_duration_per_task_ms": 800280.44, "avg_output_tokens_per_task": 15183.42, "last_run_timestamp": "2026-10-02T06:00:07.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.7466668888888889, "accuracy_std_dev": 0.0339933762614766, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08365215111111111, "avg_duration_per_task_ms": 72579.65555555555, "avg_output_tokens_per_task": 11821.215555555556, "last_run_timestamp": "2026-10-31T19:56:16.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.023726903000602494, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.007031926666666666, "avg_duration_per_task_ms": 152232.88666666666, "avg_output_tokens_per_task": 12687.024444444445, "last_run_timestamp": "2026-10-31T19:25:36.518Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.7466665833333334, "accuracy_std_dev": 0.0259628652195732, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.5233425716666666, "avg_duration_per_task_ms": 222591.87333333332, "avg_output_tokens_per_task": 9975.42, "last_run_timestamp": "2026-10-31T13:21:45.027Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7466665714285714, "accuracy_std_dev": 0.03380611384628165, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03743281428571429, "avg_duration_per_task_ms": 197093.65714285715, "avg_output_tokens_per_task": 13601.3, "last_run_timestamp": "2026-10-03T08:15:57.213Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.7466665625000001, "accuracy_std_dev": 0.022360772946078904, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.013514558749999999, "avg_duration_per_task_ms": 139486.7775, "avg_output_tokens_per_task": 15486.175, "last_run_timestamp": "2026-10-31T13:03:01.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DeepInfra", "accuracy": 0.7466665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00520861, "avg_duration_per_task_ms": 148091.91, "avg_output_tokens_per_task": 12193.49, "last_run_timestamp": "2026-10-31T11:47:33.345Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.011547101609639251, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04422623666666667, "avg_duration_per_task_ms": 266653.04, "avg_output_tokens_per_task": 13360.296666666667, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.58250779, "avg_duration_per_task_ms": 102747.13, "avg_output_tokens_per_task": 7546.85, "last_run_timestamp": "2026-10-01T19:04:42.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.7466663333333334, "accuracy_std_dev": 0.033993463423951875, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01425808, "avg_duration_per_task_ms": 139346.08, "avg_output_tokens_per_task": 12850.746666666666, "last_run_timestamp": "2026-10-31T12:18:46.817Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.7463888333333334, "accuracy_std_dev": 0.027854064690912814, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.5516252541666667, "avg_duration_per_task_ms": 200356.9575, "avg_output_tokens_per_task": 9752.154166666667, "last_run_timestamp": "2026-10-31T13:01:06.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7463769999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.05924082608695652, "avg_duration_per_task_ms": 289784.32608695654, "avg_output_tokens_per_task": 9531.695652173914, "last_run_timestamp": "2026-08-15T14:23:29.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.7461904285714286, "accuracy_std_dev": 0.021560263063973885, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12111663714285714, "avg_duration_per_task_ms": 120926.64285714286, "avg_output_tokens_per_task": 10598.828571428572, "last_run_timestamp": "2026-10-17T19:14:08.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7461111666666667, "accuracy_std_dev": 0.018946052393719277, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.3487555299999999, "avg_duration_per_task_ms": 215726.44333333333, "avg_output_tokens_per_task": 11930.17, "last_run_timestamp": "2026-10-31T13:31:12.841Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.7460141846496107, "accuracy_std_dev": 0.015367727629578602, "run_count": 18, "total_questions": 899, "avg_cost_per_task_usd": 0.04452976084538376, "avg_duration_per_task_ms": 330627.27141268074, "avg_output_tokens_per_task": 25778.021134593993, "last_run_timestamp": "2026-10-31T19:20:57.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.7460000999999999, "accuracy_std_dev": 0.04038166415701563, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.024082161999999997, "avg_duration_per_task_ms": 403869.142, "avg_output_tokens_per_task": 15439.562, "last_run_timestamp": "2026-10-17T11:15:55.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.745926, "accuracy_std_dev": 0.0403005321856231, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11525609333333334, "avg_duration_per_task_ms": 227448.44222222222, "avg_output_tokens_per_task": 9598.077777777778, "last_run_timestamp": "2026-10-31T20:27:27.677Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.74583325, "accuracy_std_dev": 0.025913946803555434, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0561430525, "avg_duration_per_task_ms": 815372.045, "avg_output_tokens_per_task": 15741.0025, "last_run_timestamp": "2026-10-31T20:02:53.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.74583325, "accuracy_std_dev": 0.03402396010442498, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02499106, "avg_duration_per_task_ms": 929627.78, "avg_output_tokens_per_task": 52531.49, "last_run_timestamp": "2026-10-24T12:31:49.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7457142857142857, "accuracy_std_dev": 0.029692331634924993, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.5093009542857143, "avg_duration_per_task_ms": 175938.91142857142, "avg_output_tokens_per_task": 7450.982857142857, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.7454545909090909, "accuracy_std_dev": 0.02161524816516654, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.4409956472727272, "avg_duration_per_task_ms": 587766.49, "avg_output_tokens_per_task": 10456.687272727273, "last_run_timestamp": "2026-10-31T11:24:57.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.7453333, "accuracy_std_dev": 0.02454934079379731, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.17119798400000003, "avg_duration_per_task_ms": 320387.726, "avg_output_tokens_per_task": 11023.856, "last_run_timestamp": "2026-10-31T12:25:36.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.7453333, "accuracy_std_dev": 0.027936434196403816, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.02067147, "avg_duration_per_task_ms": 399151.052, "avg_output_tokens_per_task": 11925.912, "last_run_timestamp": "2026-10-17T11:29:07.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.7452380714285715, "accuracy_std_dev": 0.025093932830934848, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.3444721642857143, "avg_duration_per_task_ms": 207641.8, "avg_output_tokens_per_task": 11831.092857142858, "last_run_timestamp": "2026-10-31T13:36:32.496Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.7451853333333333, "accuracy_std_dev": 0.031234315132203188, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01557802666666667, "avg_duration_per_task_ms": 147555.71333333335, "avg_output_tokens_per_task": 15042.41111111111, "last_run_timestamp": "2026-10-31T12:25:02.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.7451852222222222, "accuracy_std_dev": 0.017716056438394953, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.21677278888888887, "avg_duration_per_task_ms": 284791.18666666665, "avg_output_tokens_per_task": 11557.691111111111, "last_run_timestamp": "2026-10-31T11:21:59.486Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.745, "accuracy_std_dev": 0.026822730826670128, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.15284815, "avg_duration_per_task_ms": 494527.49, "avg_output_tokens_per_task": 28246.935, "last_run_timestamp": "2026-10-17T19:47:21.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.744999875, "accuracy_std_dev": 0.030867001888900313, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0114373625, "avg_duration_per_task_ms": 404697.66, "avg_output_tokens_per_task": 5754.185, "last_run_timestamp": "2026-10-31T12:24:48.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.7448276551724138, "accuracy_std_dev": 0.0035438860340341496, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.022715406896551726, "avg_duration_per_task_ms": 289516.0620689655, "avg_output_tokens_per_task": 10103.63448275862, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.744681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.05595836170212766, "avg_duration_per_task_ms": 34218.255319148935, "avg_output_tokens_per_task": 6745.085106382979, "last_run_timestamp": "2026-08-04T03:12:18.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.7446667, "accuracy_std_dev": 0.01956180291818729, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.055664519999999995, "avg_duration_per_task_ms": 332682.316, "avg_output_tokens_per_task": 14458.648, "last_run_timestamp": "2026-10-31T11:42:33.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7444445833333333, "accuracy_std_dev": 0.031661685769128825, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.5453893666666667, "avg_duration_per_task_ms": 180530.81666666668, "avg_output_tokens_per_task": 10281.893333333333, "last_run_timestamp": "2026-10-31T12:28:38.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7444444444444445, "accuracy_std_dev": 0.031426873772868374, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09879648, "avg_duration_per_task_ms": 287953.65555555554, "avg_output_tokens_per_task": 7148.386666666666, "last_run_timestamp": "2026-10-31T19:33:49.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Together", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02140772, "avg_duration_per_task_ms": 53184.22, "avg_output_tokens_per_task": 14364.18, "last_run_timestamp": "2026-10-31T11:41:11.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04041968666666667, "avg_duration_per_task_ms": 291588.68666666665, "avg_output_tokens_per_task": 24330.48, "last_run_timestamp": "2026-09-30T11:25:05.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.74416675, "accuracy_std_dev": 0.03012731088543251, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.064155875, "avg_duration_per_task_ms": 1722921.32, "avg_output_tokens_per_task": 8915.945, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.744131569416499, "accuracy_std_dev": 0.0294471423722235, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.06635609456740442, "avg_duration_per_task_ms": 1016068.6076458753, "avg_output_tokens_per_task": 74326.28571428571, "last_run_timestamp": "2026-10-24T19:38:23.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.744, "accuracy_std_dev": 0.04013309523821955, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.12688716400000002, "avg_duration_per_task_ms": 700323.218, "avg_output_tokens_per_task": 36239.088, "last_run_timestamp": "2026-10-31T12:47:12.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7439998000000001, "accuracy_std_dev": 0.025508095447524134, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.010447856000000002, "avg_duration_per_task_ms": 427694.796, "avg_output_tokens_per_task": 12591.384, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7438325352839931, "accuracy_std_dev": 0.030628998095826893, "run_count": 12, "total_questions": 581, "avg_cost_per_task_usd": 0.023620459552495695, "avg_duration_per_task_ms": 98292.74010327022, "avg_output_tokens_per_task": 8358.294320137693, "last_run_timestamp": "2026-10-31T19:46:13.143Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.743738, "accuracy_std_dev": 0.01688456971344439, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.00938970520231214, "avg_duration_per_task_ms": 347914.32658959535, "avg_output_tokens_per_task": 3877.916184971098, "last_run_timestamp": "2026-10-03T08:46:08.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.7437037777777777, "accuracy_std_dev": 0.025936447601258752, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03902438222222222, "avg_duration_per_task_ms": 589876.5155555556, "avg_output_tokens_per_task": 15857.186666666666, "last_run_timestamp": "2026-10-31T19:47:52.982Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.7436363636363637, "accuracy_std_dev": 0.0487718196806176, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.0472110309090909, "avg_duration_per_task_ms": 288762.1018181818, "avg_output_tokens_per_task": 15032.698181818181, "last_run_timestamp": "2026-10-31T12:45:59.472Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7433335, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64823893, "avg_duration_per_task_ms": 264133.83, "avg_output_tokens_per_task": 10511, "last_run_timestamp": "2026-10-01T16:00:37.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.026597020163085114, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.32731308, "avg_duration_per_task_ms": 196690.64, "avg_output_tokens_per_task": 11548.81, "last_run_timestamp": "2026-10-31T12:07:21.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.017532868606008445, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012350623333333333, "avg_duration_per_task_ms": 125234.99333333333, "avg_output_tokens_per_task": 4198.8, "last_run_timestamp": "2026-10-03T08:23:09.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.7433333, "accuracy_std_dev": 0.03441571669470216, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.028436006, "avg_duration_per_task_ms": 438756.644, "avg_output_tokens_per_task": 14726.93, "last_run_timestamp": "2026-10-17T13:55:17.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.7433332500000001, "accuracy_std_dev": 0.023804656407675817, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.055247185, "avg_duration_per_task_ms": 380831.14, "avg_output_tokens_per_task": 8536.295, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7433034866071428, "accuracy_std_dev": 0.01807640693429757, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.047182370535714294, "avg_duration_per_task_ms": 707194.1785714285, "avg_output_tokens_per_task": 14275.674107142857, "last_run_timestamp": "2026-10-31T19:48:13.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.7431746190476191, "accuracy_std_dev": 0.018008818983714862, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.36043625523809525, "avg_duration_per_task_ms": 86437.01714285715, "avg_output_tokens_per_task": 7106.278095238095, "last_run_timestamp": "2026-10-31T12:25:51.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7430302727272727, "accuracy_std_dev": 0.027760033635986, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5088923236363636, "avg_duration_per_task_ms": 192892.78363636363, "avg_output_tokens_per_task": 9984.365454545454, "last_run_timestamp": "2026-10-31T12:42:53.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7429720200803213, "accuracy_std_dev": 0.03338175982419142, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.004595510040160643, "avg_duration_per_task_ms": 259445.41365461846, "avg_output_tokens_per_task": 12133.893574297188, "last_run_timestamp": "2026-10-31T19:49:31.140Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.742963, "accuracy_std_dev": 0.03221999055796815, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.13625242222222222, "avg_duration_per_task_ms": 360519.17555555556, "avg_output_tokens_per_task": 11834.471111111112, "last_run_timestamp": "2026-10-31T21:07:15.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7429168125, "accuracy_std_dev": 0.02570869361037903, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.05779907625, "avg_duration_per_task_ms": 136331.88625, "avg_output_tokens_per_task": 9998.7625, "last_run_timestamp": "2026-10-31T19:13:33.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.74277775, "accuracy_std_dev": 0.0244128236149399, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.14337107166666668, "avg_duration_per_task_ms": 621697.9133333333, "avg_output_tokens_per_task": 26711.696666666667, "last_run_timestamp": "2026-10-31T19:26:38.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.7426668, "accuracy_std_dev": 0.03690245640821219, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011645404, "avg_duration_per_task_ms": 585840.916, "avg_output_tokens_per_task": 18794.172, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7426668, "accuracy_std_dev": 0.0246215659770048, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.079647184, "avg_duration_per_task_ms": 303286.308, "avg_output_tokens_per_task": 11264.834, "last_run_timestamp": "2026-10-31T21:26:40.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.7426667, "accuracy_std_dev": 0.028472143112347543, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.020416034, "avg_duration_per_task_ms": 153083.812, "avg_output_tokens_per_task": 16185.95, "last_run_timestamp": "2026-10-31T12:48:17.235Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.7426666000000001, "accuracy_std_dev": 0.04434197309367279, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.16675142799999998, "avg_duration_per_task_ms": 302047.356, "avg_output_tokens_per_task": 9682.606, "last_run_timestamp": "2026-10-31T11:21:43.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.7425926666666667, "accuracy_std_dev": 0.03165311687654157, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.013586744444444443, "avg_duration_per_task_ms": 236886.98, "avg_output_tokens_per_task": 22693.746666666666, "last_run_timestamp": "2026-10-24T12:29:34.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7425405959079282, "accuracy_std_dev": 0.026283120985422872, "run_count": 8, "total_questions": 391, "avg_cost_per_task_usd": 0.12785004603580563, "avg_duration_per_task_ms": 203562.63938618926, "avg_output_tokens_per_task": 10880.268542199488, "last_run_timestamp": "2026-10-31T19:27:07.786Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.7424241818181818, "accuracy_std_dev": 0.03530779679235989, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.36784247818181814, "avg_duration_per_task_ms": 381853.54363636364, "avg_output_tokens_per_task": 9773.532727272728, "last_run_timestamp": "2026-10-31T12:43:33.294Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.7422838796296296, "accuracy_std_dev": 0.02605883730643627, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.04679922222222222, "avg_duration_per_task_ms": 176761.8811728395, "avg_output_tokens_per_task": 8966.202160493827, "last_run_timestamp": "2026-10-31T19:46:31.278Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0028365266666666666, "avg_duration_per_task_ms": 68565.14666666667, "avg_output_tokens_per_task": 11675.026666666667, "last_run_timestamp": "2026-10-01T16:01:02.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.27693500666666665, "avg_duration_per_task_ms": 100083.4, "avg_output_tokens_per_task": 5197.013333333333, "last_run_timestamp": "2026-10-30T00:15:31.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Cohere", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.011330936570096763, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01793052666666667, "avg_duration_per_task_ms": 86484.18, "avg_output_tokens_per_task": 20778.286666666667, "last_run_timestamp": "2026-10-31T11:37:00.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7422221111111111, "accuracy_std_dev": 0.03442657601151391, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07352381777777778, "avg_duration_per_task_ms": 217584.44444444444, "avg_output_tokens_per_task": 9519.468888888889, "last_run_timestamp": "2026-10-31T19:53:34.188Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.742222, "accuracy_std_dev": 0.03861798340151903, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.010298153333333334, "avg_duration_per_task_ms": 95025.84666666666, "avg_output_tokens_per_task": 13300.326666666666, "last_run_timestamp": "2026-10-31T12:36:20.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.7420636446886446, "accuracy_std_dev": 0.016469967104193704, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.01720449084249084, "avg_duration_per_task_ms": 82040.02197802198, "avg_output_tokens_per_task": 4114.483516483517, "last_run_timestamp": "2026-10-31T12:00:42.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.7416667499999999, "accuracy_std_dev": 0.02074966532712037, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.019690735, "avg_duration_per_task_ms": 215383.61, "avg_output_tokens_per_task": 12170.26, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.741666625, "accuracy_std_dev": 0.025331164930464133, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021842495, "avg_duration_per_task_ms": 141327.99, "avg_output_tokens_per_task": 16250.775, "last_run_timestamp": "2026-10-31T12:24:54.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.741666625, "accuracy_std_dev": 0.030866893899198455, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021115824999999998, "avg_duration_per_task_ms": 150544.0025, "avg_output_tokens_per_task": 8219.325, "last_run_timestamp": "2026-10-31T18:57:59.813Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.7416481202672606, "accuracy_std_dev": 0.022563192218990436, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.21138012694877506, "avg_duration_per_task_ms": 402109.59020044544, "avg_output_tokens_per_task": 9801.538975501113, "last_run_timestamp": "2026-10-31T11:26:03.627Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Baidu", "accuracy": 0.7414965918367348, "accuracy_std_dev": 0.01208300000000001, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.014484438775510205, "avg_duration_per_task_ms": 90891.24489795919, "avg_output_tokens_per_task": 9684.755102040815, "last_run_timestamp": "2026-10-31T13:18:30.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.74136442746114, "accuracy_std_dev": 0.023598813561278888, "run_count": 8, "total_questions": 386, "avg_cost_per_task_usd": 0.1499042176165803, "avg_duration_per_task_ms": 675715.4041450777, "avg_output_tokens_per_task": 29694.012953367877, "last_run_timestamp": "2026-10-31T19:55:58.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.7413334, "accuracy_std_dev": 0.02436751571744648, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.097602524, "avg_duration_per_task_ms": 217804.16, "avg_output_tokens_per_task": 9920.02, "last_run_timestamp": "2026-10-31T14:40:25.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.7413333, "accuracy_std_dev": 0.03095523075685272, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.40647999399999996, "avg_duration_per_task_ms": 588538.78, "avg_output_tokens_per_task": 9896.416, "last_run_timestamp": "2026-10-31T12:27:47.241Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7412121818181818, "accuracy_std_dev": 0.0275340250329006, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3449351727272727, "avg_duration_per_task_ms": 200135.41454545455, "avg_output_tokens_per_task": 11800.205454545454, "last_run_timestamp": "2026-10-31T13:12:27.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Nebius", "accuracy": 0.7410254102564102, "accuracy_std_dev": 0.015687179899124656, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.010222671794871796, "avg_duration_per_task_ms": 114819.93846153846, "avg_output_tokens_per_task": 3734.005128205128, "last_run_timestamp": "2026-10-31T12:06:01.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7406779661016949, "accuracy_std_dev": 0.0192449897289312, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.02004586440677966, "avg_duration_per_task_ms": 1117850.5423728814, "avg_output_tokens_per_task": 22657.922033898307, "last_run_timestamp": "2026-09-12T08:11:08.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.7406668999999999, "accuracy_std_dev": 0.04294179161015525, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.009957568, "avg_duration_per_task_ms": 419095.964, "avg_output_tokens_per_task": 14787.174, "last_run_timestamp": "2026-10-17T11:26:35.790Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.7406667, "accuracy_std_dev": 0.03147484286553311, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.067724174, "avg_duration_per_task_ms": 257262.924, "avg_output_tokens_per_task": 9632.852, "last_run_timestamp": "2026-10-31T20:31:43.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.7406666000000001, "accuracy_std_dev": 0.018961987708043703, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.014243233999999999, "avg_duration_per_task_ms": 72756.944, "avg_output_tokens_per_task": 4429.298, "last_run_timestamp": "2026-10-31T12:42:18.186Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7403698151001541, "accuracy_std_dev": 0.028547774310492945, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.012115494607087829, "avg_duration_per_task_ms": 425113.8289676425, "avg_output_tokens_per_task": 14181.97688751926, "last_run_timestamp": "2026-10-31T11:58:45.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.7403334, "accuracy_std_dev": 0.020245596504919312, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.379164166, "avg_duration_per_task_ms": 189558.308, "avg_output_tokens_per_task": 11056.844, "last_run_timestamp": "2026-10-31T11:51:20.518Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7403274419642857, "accuracy_std_dev": 0.036586619279744714, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.07693905803571428, "avg_duration_per_task_ms": 873819.4196428572, "avg_output_tokens_per_task": 14604.90625, "last_run_timestamp": "2026-10-24T20:35:30.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.74014702004008, "accuracy_std_dev": 0.022155950173486136, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.06005350100200401, "avg_duration_per_task_ms": 140448.37274549098, "avg_output_tokens_per_task": 4119.851703406814, "last_run_timestamp": "2026-10-31T12:27:45.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7400001666666666, "accuracy_std_dev": 0.01154690915954955, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.034900296666666664, "avg_duration_per_task_ms": 129527.63333333333, "avg_output_tokens_per_task": 16132.923333333334, "last_run_timestamp": "2026-10-27T00:42:49.766Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.740000125, "accuracy_std_dev": 0.02924983380908983, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.072280915, "avg_duration_per_task_ms": 322638.715, "avg_output_tokens_per_task": 11472.485, "last_run_timestamp": "2026-10-24T22:27:21.158Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.74, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.23335095, "avg_duration_per_task_ms": 96091.26, "avg_output_tokens_per_task": 5555.9, "last_run_timestamp": "2026-10-01T19:04:26.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.74, "accuracy_std_dev": 0.04773656244431517, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.00978464181818182, "avg_duration_per_task_ms": 283078.21636363637, "avg_output_tokens_per_task": 15624.66909090909, "last_run_timestamp": "2026-10-31T11:40:30.784Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Crusoe", "accuracy": 0.74, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14729807, "avg_duration_per_task_ms": 386775.27, "avg_output_tokens_per_task": 24280.02, "last_run_timestamp": "2026-10-31T19:58:44.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.74, "accuracy_std_dev": 0.03511903567511501, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.023083885, "avg_duration_per_task_ms": 150345.7, "avg_output_tokens_per_task": 16221.6925, "last_run_timestamp": "2026-10-31T11:30:43.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Near AI", "accuracy": 0.74, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01862194, "avg_duration_per_task_ms": 339225.77, "avg_output_tokens_per_task": 12354.38, "last_run_timestamp": "2026-10-31T13:22:50.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Mistral", "accuracy": 0.74, "accuracy_std_dev": 1.1102230246251565e-16, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03857922666666667, "avg_duration_per_task_ms": 77482.69333333333, "avg_output_tokens_per_task": 13512.933333333332, "last_run_timestamp": "2026-10-31T11:52:45.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.74, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.37915447, "avg_duration_per_task_ms": 148084.91, "avg_output_tokens_per_task": 12525.4, "last_run_timestamp": "2026-10-01T19:04:55.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.74, "accuracy_std_dev": 0.03370161735650745, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0978457, "avg_duration_per_task_ms": 271590.2111111111, "avg_output_tokens_per_task": 12524.18888888889, "last_run_timestamp": "2026-10-31T20:07:04.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Nebius", "accuracy": 0.74, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06914866, "avg_duration_per_task_ms": 152605.49, "avg_output_tokens_per_task": 14774.68, "last_run_timestamp": "2026-10-31T12:25:52.830Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.74, "accuracy_std_dev": 0.019860653454128848, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.01624762875, "avg_duration_per_task_ms": 161764.35125, "avg_output_tokens_per_task": 4237.58, "last_run_timestamp": "2026-10-31T11:33:42.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.7399329261744967, "accuracy_std_dev": 0.017150616176167584, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.24555212751677855, "avg_duration_per_task_ms": 357614.23825503356, "avg_output_tokens_per_task": 12408.355704697986, "last_run_timestamp": "2026-10-31T12:24:39.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7395555333333333, "accuracy_std_dev": 0.016947170784004716, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.013297554666666666, "avg_duration_per_task_ms": 100947.12933333333, "avg_output_tokens_per_task": 10746.964, "last_run_timestamp": "2026-10-31T12:45:41.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.7394444166666667, "accuracy_std_dev": 0.04237910291928153, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.01083521, "avg_duration_per_task_ms": 286660.46166666667, "avg_output_tokens_per_task": 15769.041666666666, "last_run_timestamp": "2026-10-31T11:54:55.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7393938181818183, "accuracy_std_dev": 0.027627475312607935, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.34960052363636374, "avg_duration_per_task_ms": 219975.07454545455, "avg_output_tokens_per_task": 10861.59090909091, "last_run_timestamp": "2026-10-31T12:46:18.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.7392380571428574, "accuracy_std_dev": 0.02911476386493665, "run_count": 35, "total_questions": 1750, "avg_cost_per_task_usd": 0.369135528, "avg_duration_per_task_ms": 199052.34971428572, "avg_output_tokens_per_task": 11910.057714285715, "last_run_timestamp": "2026-10-31T13:17:20.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.739166625, "accuracy_std_dev": 0.02933284919223455, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0108853675, "avg_duration_per_task_ms": 219104.3875, "avg_output_tokens_per_task": 14027.29, "last_run_timestamp": "2026-10-31T19:46:05.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.738889, "accuracy_std_dev": 0.024695368688615833, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01095846, "avg_duration_per_task_ms": 321436.35333333333, "avg_output_tokens_per_task": 22570.04, "last_run_timestamp": "2026-10-31T12:52:06.048Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.7386668, "accuracy_std_dev": 0.03166503013357166, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.009092168000000001, "avg_duration_per_task_ms": 101935.708, "avg_output_tokens_per_task": 5027.116, "last_run_timestamp": "2026-10-17T11:17:02.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.738484909090909, "accuracy_std_dev": 0.024261459951778173, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.017931623636363636, "avg_duration_per_task_ms": 332207.99818181817, "avg_output_tokens_per_task": 23710.247272727273, "last_run_timestamp": "2026-10-31T13:20:49.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7383335, "accuracy_std_dev": 0.0016665000000000152, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08170183999999998, "avg_duration_per_task_ms": 327224.56, "avg_output_tokens_per_task": 8775.96, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.738333375, "accuracy_std_dev": 0.03244649655948039, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.049700385, "avg_duration_per_task_ms": 377861.96, "avg_output_tokens_per_task": 11236.51, "last_run_timestamp": "2026-10-31T21:59:10.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.73833325, "accuracy_std_dev": 0.017240319347027768, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.063434195, "avg_duration_per_task_ms": 520281.665, "avg_output_tokens_per_task": 14632.895, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.73833325, "accuracy_std_dev": 0.03210231473254696, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03571221, "avg_duration_per_task_ms": 388203.89, "avg_output_tokens_per_task": 8383.965, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.7382021797752809, "accuracy_std_dev": 0.021880590311850887, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.020498339325842697, "avg_duration_per_task_ms": 307841.11235955055, "avg_output_tokens_per_task": 10994.101123595505, "last_run_timestamp": "2026-10-17T11:09:59.070Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7380952857142857, "accuracy_std_dev": 0.031414080354035444, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08033971999999999, "avg_duration_per_task_ms": 526120.5771428571, "avg_output_tokens_per_task": 11993.431428571428, "last_run_timestamp": "2026-10-24T23:08:37.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7380950401002507, "accuracy_std_dev": 0.022649397541656617, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.053091303258145366, "avg_duration_per_task_ms": 777194.3383458647, "avg_output_tokens_per_task": 14178.676691729323, "last_run_timestamp": "2026-10-24T19:40:41.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.7380000999999999, "accuracy_std_dev": 0.02844855708274146, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.016056571999999998, "avg_duration_per_task_ms": 534586.188, "avg_output_tokens_per_task": 17634.696, "last_run_timestamp": "2026-10-31T12:03:50.388Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Mistral", "accuracy": 0.738, "accuracy_std_dev": 0.030954985239860806, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.019570816, "avg_duration_per_task_ms": 88243.608, "avg_output_tokens_per_task": 5445.588, "last_run_timestamp": "2026-10-31T11:13:51.572Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7378222664756448, "accuracy_std_dev": 0.026214713234349613, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.13115203724928365, "avg_duration_per_task_ms": 65255.4899713467, "avg_output_tokens_per_task": 9539.232091690545, "last_run_timestamp": "2026-10-31T19:12:18.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "SiliconFlow", "accuracy": 0.7377779999999999, "accuracy_std_dev": 0.027933050471917086, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02205162, "avg_duration_per_task_ms": 230910.26, "avg_output_tokens_per_task": 11297.92, "last_run_timestamp": "2026-10-31T12:27:40.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.7377779999999999, "accuracy_std_dev": 0.020608136952831695, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.009694766666666667, "avg_duration_per_task_ms": 179217.14666666667, "avg_output_tokens_per_task": 12873.993333333334, "last_run_timestamp": "2026-10-31T11:06:39.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.7377776666666667, "accuracy_std_dev": 0.020366893981273733, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3511924911111111, "avg_duration_per_task_ms": 195777.59333333332, "avg_output_tokens_per_task": 9122.282222222222, "last_run_timestamp": "2026-10-31T11:28:52.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7377776666666667, "accuracy_std_dev": 0.030952013836833894, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.18000499999999997, "avg_duration_per_task_ms": 513788.9733333333, "avg_output_tokens_per_task": 9651.956666666667, "last_run_timestamp": "2026-10-31T22:21:14.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7376126486486487, "accuracy_std_dev": 0.027927098584230564, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.011551577702702704, "avg_duration_per_task_ms": 61368.75337837838, "avg_output_tokens_per_task": 4262.564189189189, "last_run_timestamp": "2026-10-31T12:25:51.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.737499875, "accuracy_std_dev": 0.022592425906913488, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01732587, "avg_duration_per_task_ms": 193821.065, "avg_output_tokens_per_task": 4493.1475, "last_run_timestamp": "2026-10-31T12:42:00.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.7374243541247485, "accuracy_std_dev": 0.021992186721879213, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.34455653319919516, "avg_duration_per_task_ms": 409342.0885311871, "avg_output_tokens_per_task": 9871.545271629779, "last_run_timestamp": "2026-10-24T12:29:09.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.7373334, "accuracy_std_dev": 0.020482933819157834, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.039714615999999994, "avg_duration_per_task_ms": 174773.632, "avg_output_tokens_per_task": 27822.08, "last_run_timestamp": "2026-10-21T12:36:51.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.7372221666666667, "accuracy_std_dev": 0.025452663641988886, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07899650500000001, "avg_duration_per_task_ms": 156175.89, "avg_output_tokens_per_task": 8784.876666666667, "last_run_timestamp": "2026-10-31T20:18:12.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.736667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.47698523000000004, "avg_duration_per_task_ms": 128774.34, "avg_output_tokens_per_task": 7206.5, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7366668, "accuracy_std_dev": 0.022900761755015906, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.063241744, "avg_duration_per_task_ms": 1530537.112, "avg_output_tokens_per_task": 25222.716, "last_run_timestamp": "2026-10-03T08:06:43.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Io Net", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02539905, "avg_duration_per_task_ms": 216631.23, "avg_output_tokens_per_task": 14336.27, "last_run_timestamp": "2026-10-24T12:02:36.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.7366665, "accuracy_std_dev": 0.014529701381996819, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.057118495000000005, "avg_duration_per_task_ms": 215486.77, "avg_output_tokens_per_task": 9145.625, "last_run_timestamp": "2026-10-04T00:00:34.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.7365744652049571, "accuracy_std_dev": 0.02859181594676679, "run_count": 21, "total_questions": 1049, "avg_cost_per_task_usd": 0.13010050238322207, "avg_duration_per_task_ms": 159321.12011439467, "avg_output_tokens_per_task": 9364.21639656816, "last_run_timestamp": "2026-10-24T19:48:18.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.7363635454545455, "accuracy_std_dev": 0.027022348334677756, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.2857510236363636, "avg_duration_per_task_ms": 338121.0927272727, "avg_output_tokens_per_task": 10844.636363636364, "last_run_timestamp": "2026-10-31T11:38:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.7362961111111112, "accuracy_std_dev": 0.02157064127952337, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06493803333333333, "avg_duration_per_task_ms": 247546.98, "avg_output_tokens_per_task": 12690.464444444444, "last_run_timestamp": "2026-10-31T20:18:31.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.736, "accuracy_std_dev": 0.011623577573191504, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.02217724, "avg_duration_per_task_ms": 74896.184, "avg_output_tokens_per_task": 4355.024, "last_run_timestamp": "2026-10-31T12:05:37.260Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.7359999333333334, "accuracy_std_dev": 0.0231964569463145, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.013030309333333335, "avg_duration_per_task_ms": 73246.75466666666, "avg_output_tokens_per_task": 10779.536, "last_run_timestamp": "2026-10-31T11:42:11.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.7359999, "accuracy_std_dev": 0.025157337786220563, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.008132438, "avg_duration_per_task_ms": 403849.82, "avg_output_tokens_per_task": 14485.44, "last_run_timestamp": "2026-10-17T12:09:02.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7359998000000001, "accuracy_std_dev": 0.019596013200648757, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.3276280200000001, "avg_duration_per_task_ms": 156068.02, "avg_output_tokens_per_task": 21546.7, "last_run_timestamp": "2026-10-01T19:05:00.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7359261111111111, "accuracy_std_dev": 0.02318882707227026, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04132428666666666, "avg_duration_per_task_ms": 135748.28, "avg_output_tokens_per_task": 25779.64, "last_run_timestamp": "2026-10-31T12:16:10.183Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.04157400066013479, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.48569376, "avg_duration_per_task_ms": 125885.67333333334, "avg_output_tokens_per_task": 7296.606666666667, "last_run_timestamp": "2026-10-02T23:08:08.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.7355555555555555, "accuracy_std_dev": 0.016629543844289166, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11909873777777778, "avg_duration_per_task_ms": 359004.24666666664, "avg_output_tokens_per_task": 9944.664444444445, "last_run_timestamp": "2026-10-31T20:34:53.249Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7354165, "accuracy_std_dev": 0.0429125354005097, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.006874592499999999, "avg_duration_per_task_ms": 421319.53, "avg_output_tokens_per_task": 14336.97, "last_run_timestamp": "2026-10-10T15:31:23.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7352380000000001, "accuracy_std_dev": 0.01698343330593503, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3052330542857143, "avg_duration_per_task_ms": 191492.84285714285, "avg_output_tokens_per_task": 10672.025714285714, "last_run_timestamp": "2026-10-01T19:05:18.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.735000125, "accuracy_std_dev": 0.04186617734650937, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0273002175, "avg_duration_per_task_ms": 144987.4475, "avg_output_tokens_per_task": 12792.885, "last_run_timestamp": "2026-10-31T18:53:34.804Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7349398099062917, "accuracy_std_dev": 0.02455646627618342, "run_count": 15, "total_questions": 747, "avg_cost_per_task_usd": 0.05549164792503348, "avg_duration_per_task_ms": 157974.1485943775, "avg_output_tokens_per_task": 30263.555555555555, "last_run_timestamp": "2026-10-31T12:41:59.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7349397911646587, "accuracy_std_dev": 0.047428285455095254, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.2969604819277108, "avg_duration_per_task_ms": 386565.9397590361, "avg_output_tokens_per_task": 11351.530120481928, "last_run_timestamp": "2026-10-31T12:42:56.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7348147777777778, "accuracy_std_dev": 0.03043361346631854, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09666358000000001, "avg_duration_per_task_ms": 320248.93111111113, "avg_output_tokens_per_task": 11007.382222222222, "last_run_timestamp": "2026-10-31T20:47:51.733Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.734694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.25339910204081634, "avg_duration_per_task_ms": 146531.16326530612, "avg_output_tokens_per_task": 29831.102040816328, "last_run_timestamp": "2026-10-01T08:03:06.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.7344445, "accuracy_std_dev": 0.029165456061066488, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.028961041666666656, "avg_duration_per_task_ms": 102116.81166666666, "avg_output_tokens_per_task": 10295.041666666666, "last_run_timestamp": "2026-10-31T19:09:49.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.7344443333333334, "accuracy_std_dev": 0.03241329935981356, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.30671481166666664, "avg_duration_per_task_ms": 338981.285, "avg_output_tokens_per_task": 9637.978333333333, "last_run_timestamp": "2026-10-31T13:04:39.070Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.734, "accuracy_std_dev": 0.030905008584370283, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.049636292000000005, "avg_duration_per_task_ms": 244020.536, "avg_output_tokens_per_task": 10552.6, "last_run_timestamp": "2026-10-31T19:47:34.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.733796, "accuracy_std_dev": 0.0059017228557995535, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.20467558333333333, "avg_duration_per_task_ms": 370954.7361111111, "avg_output_tokens_per_task": 23212.9375, "last_run_timestamp": "2026-10-24T19:18:46.938Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.7337622316176471, "accuracy_std_dev": 0.042021571236176135, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.02766205882352941, "avg_duration_per_task_ms": 113870.94669117648, "avg_output_tokens_per_task": 10219.617647058823, "last_run_timestamp": "2026-10-31T19:48:53.852Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.7333335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12301775, "avg_duration_per_task_ms": 155307.55, "avg_output_tokens_per_task": 18812.75, "last_run_timestamp": "2026-10-01T19:05:47.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Together", "accuracy": 0.7333335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08901925000000001, "avg_duration_per_task_ms": 93612.32, "avg_output_tokens_per_task": 8634.62, "last_run_timestamp": "2026-10-31T11:29:29.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08394800999999999, "avg_duration_per_task_ms": 134148.76, "avg_output_tokens_per_task": 8723.94, "last_run_timestamp": "2026-10-17T19:55:39.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7333335, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12687428, "avg_duration_per_task_ms": 110480.49, "avg_output_tokens_per_task": 5152.54, "last_run_timestamp": "2026-10-01T19:04:55.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.7333334, "accuracy_std_dev": 0.03451234883690184, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.30572972400000004, "avg_duration_per_task_ms": 282521.796, "avg_output_tokens_per_task": 10813.856, "last_run_timestamp": "2026-10-31T12:51:11.835Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008480786666666665, "avg_duration_per_task_ms": 397846.0266666667, "avg_output_tokens_per_task": 19127.326666666668, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Mancer 2", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06451945333333334, "avg_duration_per_task_ms": 876085.0533333333, "avg_output_tokens_per_task": 15144.64, "last_run_timestamp": "2026-10-31T12:31:12.958Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Crusoe", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.03569390445004426, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021479246666666667, "avg_duration_per_task_ms": 120781.28, "avg_output_tokens_per_task": 11581.666666666666, "last_run_timestamp": "2026-10-31T11:22:01.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.05760637824832321, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04898614, "avg_duration_per_task_ms": 106829.63333333333, "avg_output_tokens_per_task": 9897.326666666666, "last_run_timestamp": "2026-10-10T11:51:48.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Amazon Bedrock", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.024944382578492966, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09260214, "avg_duration_per_task_ms": 326558.9733333333, "avg_output_tokens_per_task": 4729.153333333334, "last_run_timestamp": "2026-10-31T12:28:59.426Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cohere", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.024898453333333334, "avg_duration_per_task_ms": 37126.9, "avg_output_tokens_per_task": 4153.473333333333, "last_run_timestamp": "2026-10-31T12:07:50.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.73333325, "accuracy_std_dev": 0.02357013176007932, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06276583000000001, "avg_duration_per_task_ms": 510113.315, "avg_output_tokens_per_task": 14674.185, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.7333332, "accuracy_std_dev": 0.032659904062320846, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.012294704000000002, "avg_duration_per_task_ms": 85675.876, "avg_output_tokens_per_task": 10698.828, "last_run_timestamp": "2026-10-31T11:35:21.238Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12365041999999998, "avg_duration_per_task_ms": 156560.04, "avg_output_tokens_per_task": 19704.13, "last_run_timestamp": "2026-10-01T19:04:26.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10928178, "avg_duration_per_task_ms": 75479.44, "avg_output_tokens_per_task": 18824.05, "last_run_timestamp": "2026-10-01T19:05:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.732916625, "accuracy_std_dev": 0.018888392049996622, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0482453475, "avg_duration_per_task_ms": 162563.185, "avg_output_tokens_per_task": 29218.5525, "last_run_timestamp": "2026-10-31T12:25:59.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7327394365256125, "accuracy_std_dev": 0.030733543577653163, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.06566840534521158, "avg_duration_per_task_ms": 152213.53006681515, "avg_output_tokens_per_task": 11216.12026726058, "last_run_timestamp": "2026-10-31T19:24:16.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7327272727272728, "accuracy_std_dev": 0.06263266295137199, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.15801903636363634, "avg_duration_per_task_ms": 1429178.5054545454, "avg_output_tokens_per_task": 28482.28181818182, "last_run_timestamp": "2026-10-24T20:45:53.033Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.7325926666666667, "accuracy_std_dev": 0.03841877184687483, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.042171924444444446, "avg_duration_per_task_ms": 580922.6555555556, "avg_output_tokens_per_task": 9134.031111111111, "last_run_timestamp": "2026-10-31T19:41:08.087Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7325, "accuracy_std_dev": 0.04596937990662913, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012367795, "avg_duration_per_task_ms": 953069.875, "avg_output_tokens_per_task": 22737, "last_run_timestamp": "2026-10-17T10:32:07.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.7323944162754304, "accuracy_std_dev": 0.034116380301140836, "run_count": 13, "total_questions": 639, "avg_cost_per_task_usd": 0.06478910172143976, "avg_duration_per_task_ms": 133446.72456964006, "avg_output_tokens_per_task": 8336.646322378716, "last_run_timestamp": "2026-10-31T19:10:53.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.732, "accuracy_std_dev": 0.02124973202654565, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.036015575999999994, "avg_duration_per_task_ms": 148623.836, "avg_output_tokens_per_task": 32577.608, "last_run_timestamp": "2026-10-31T12:40:24.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.7319445, "accuracy_std_dev": 0.011663532651674044, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.22145578333333332, "avg_duration_per_task_ms": 246837.50166666668, "avg_output_tokens_per_task": 10031.816666666668, "last_run_timestamp": "2026-10-31T11:24:41.391Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.7318517777777778, "accuracy_std_dev": 0.03154899079836093, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.16188553555555557, "avg_duration_per_task_ms": 219860.70222222223, "avg_output_tokens_per_task": 9557.291111111112, "last_run_timestamp": "2026-10-31T19:57:59.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.7316666666666667, "accuracy_std_dev": 0.02436222789939834, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.3637274400000001, "avg_duration_per_task_ms": 227467.92083333334, "avg_output_tokens_per_task": 10893.904166666667, "last_run_timestamp": "2026-10-31T12:43:31.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.7316666666666667, "accuracy_std_dev": 0.04261151817942603, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.028175713333333335, "avg_duration_per_task_ms": 114994.48666666666, "avg_output_tokens_per_task": 9908.751666666667, "last_run_timestamp": "2026-10-31T19:22:00.212Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Crusoe", "accuracy": 0.7316665, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01274185, "avg_duration_per_task_ms": 154997.2, "avg_output_tokens_per_task": 13696.96, "last_run_timestamp": "2026-10-31T19:54:14.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.7313333, "accuracy_std_dev": 0.04022720859331405, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.069409406, "avg_duration_per_task_ms": 292502.6, "avg_output_tokens_per_task": 12732.744, "last_run_timestamp": "2026-10-31T19:48:30.443Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.7313332, "accuracy_std_dev": 0.020231941744676914, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.012560144, "avg_duration_per_task_ms": 74716.332, "avg_output_tokens_per_task": 10525.228, "last_run_timestamp": "2026-10-31T11:19:23.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7311827782258065, "accuracy_std_dev": 0.07837008853695393, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.009883709677419354, "avg_duration_per_task_ms": 170348.1431451613, "avg_output_tokens_per_task": 4275.5161290322585, "last_run_timestamp": "2026-10-31T12:31:51.998Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.7311637573964497, "accuracy_std_dev": 0.02813377440238986, "run_count": 17, "total_questions": 845, "avg_cost_per_task_usd": 0.024811487573964494, "avg_duration_per_task_ms": 209572.125443787, "avg_output_tokens_per_task": 21468.289940828403, "last_run_timestamp": "2026-10-31T11:27:16.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.7311111666666666, "accuracy_std_dev": 0.03022542058387644, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05052855666666667, "avg_duration_per_task_ms": 436577.48333333334, "avg_output_tokens_per_task": 9693.223333333333, "last_run_timestamp": "2026-10-24T20:04:11.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "OpenInference", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.020608136952831695, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028709900000000003, "avg_duration_per_task_ms": 638312.6933333334, "avg_output_tokens_per_task": 61738.18, "last_run_timestamp": "2026-10-31T12:28:49.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12928292, "avg_duration_per_task_ms": 109953.54, "avg_output_tokens_per_task": 5131.753333333333, "last_run_timestamp": "2026-10-02T23:08:37.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7310256538461538, "accuracy_std_dev": 0.03399061082116005, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.3526058969230769, "avg_duration_per_task_ms": 206233.06846153847, "avg_output_tokens_per_task": 10305.186923076923, "last_run_timestamp": "2026-10-31T12:42:19.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.7305555, "accuracy_std_dev": 0.026346395026960852, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06789053999999999, "avg_duration_per_task_ms": 79373.93333333333, "avg_output_tokens_per_task": 8526.28, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.7303704444444444, "accuracy_std_dev": 0.027417306521859138, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05101439333333333, "avg_duration_per_task_ms": 356298.62666666665, "avg_output_tokens_per_task": 10059.124444444444, "last_run_timestamp": "2026-10-31T21:01:26.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7300002499999999, "accuracy_std_dev": 0.028867484592314224, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03407485, "avg_duration_per_task_ms": 264292.92, "avg_output_tokens_per_task": 34254.08, "last_run_timestamp": "2026-10-31T13:21:05.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.73, "accuracy_std_dev": 0.0268328157299975, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.074570136, "avg_duration_per_task_ms": 246781.984, "avg_output_tokens_per_task": 30037.412, "last_run_timestamp": "2026-10-31T11:52:51.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.73, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12856867000000002, "avg_duration_per_task_ms": 121477.86, "avg_output_tokens_per_task": 19799.68, "last_run_timestamp": "2026-10-01T19:05:00.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.73, "accuracy_std_dev": 0.04333300000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.24468226999999998, "avg_duration_per_task_ms": 158524.06, "avg_output_tokens_per_task": 22552.35, "last_run_timestamp": "2026-10-01T19:04:55.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.73, "accuracy_std_dev": 0.0336238300093847, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07512531000000001, "avg_duration_per_task_ms": 210778.66, "avg_output_tokens_per_task": 11117.87, "last_run_timestamp": "2026-10-31T20:16:51.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.73, "accuracy_std_dev": 0.04167786071965147, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07906948333333333, "avg_duration_per_task_ms": 158873.66833333333, "avg_output_tokens_per_task": 8236.776666666667, "last_run_timestamp": "2026-10-31T19:53:09.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.7299999166666666, "accuracy_std_dev": 0.026034048220418122, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08839280833333334, "avg_duration_per_task_ms": 664802.0966666667, "avg_output_tokens_per_task": 9393.481666666667, "last_run_timestamp": "2026-10-31T19:40:42.793Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.7299998888888889, "accuracy_std_dev": 0.02922883744388389, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.014746135555555555, "avg_duration_per_task_ms": 251713.60666666666, "avg_output_tokens_per_task": 21386.666666666668, "last_run_timestamp": "2026-10-31T12:23:57.099Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.729931893877551, "accuracy_std_dev": 0.03636312637494197, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.018753575510204082, "avg_duration_per_task_ms": 310156.1551020408, "avg_output_tokens_per_task": 6681.187755102041, "last_run_timestamp": "2026-10-09T02:01:31.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.729654525083612, "accuracy_std_dev": 0.01804542249436126, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.18312612040133777, "avg_duration_per_task_ms": 543481.8829431438, "avg_output_tokens_per_task": 8776.65551839465, "last_run_timestamp": "2026-10-31T21:08:40.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7296295555555555, "accuracy_std_dev": 0.02759685177339019, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09735883111111111, "avg_duration_per_task_ms": 218490.78666666665, "avg_output_tokens_per_task": 12521.58, "last_run_timestamp": "2026-10-31T20:36:53.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.7294444166666667, "accuracy_std_dev": 0.03271169279849418, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.32659671333333334, "avg_duration_per_task_ms": 207735.03833333333, "avg_output_tokens_per_task": 10355.186666666666, "last_run_timestamp": "2026-10-31T12:49:56.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.7294443333333334, "accuracy_std_dev": 0.03346740998178512, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.033491596666666665, "avg_duration_per_task_ms": 413814.1533333333, "avg_output_tokens_per_task": 8947.89, "last_run_timestamp": "2026-10-31T21:01:10.349Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7293334, "accuracy_std_dev": 0.026533159130416407, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.012403696, "avg_duration_per_task_ms": 92314.648, "avg_output_tokens_per_task": 10764.472, "last_run_timestamp": "2026-10-31T12:41:01.831Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7292263868194842, "accuracy_std_dev": 0.037395839263812465, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.07898853868194843, "avg_duration_per_task_ms": 295613.8395415473, "avg_output_tokens_per_task": 13730.816618911174, "last_run_timestamp": "2026-09-26T08:12:00.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.7291665, "accuracy_std_dev": 0.023318429765959808, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0316769325, "avg_duration_per_task_ms": 89151.6375, "avg_output_tokens_per_task": 22436.51, "last_run_timestamp": "2026-10-31T11:37:01.556Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "ModelRun", "accuracy": 0.7288890000000001, "accuracy_std_dev": 0.03142665378305492, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02386675333333333, "avg_duration_per_task_ms": 151228.52, "avg_output_tokens_per_task": 22554.02666666667, "last_run_timestamp": "2026-10-28T06:40:49.113Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Novita", "accuracy": 0.728889, "accuracy_std_dev": 0.01911599421078241, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.017266966666666664, "avg_duration_per_task_ms": 122400.14, "avg_output_tokens_per_task": 11879.513333333334, "last_run_timestamp": "2026-10-31T11:33:22.952Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.728889, "accuracy_std_dev": 0.01257094435593444, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04855964666666667, "avg_duration_per_task_ms": 115027.81333333334, "avg_output_tokens_per_task": 15070.966666666667, "last_run_timestamp": "2026-10-31T12:30:34.970Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.7288886666666667, "accuracy_std_dev": 0.017497738831695454, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.033088113333333335, "avg_duration_per_task_ms": 134354.10666666666, "avg_output_tokens_per_task": 14586.926666666666, "last_run_timestamp": "2026-10-31T12:32:08.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.7283335, "accuracy_std_dev": 0.024626886091898288, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04604961333333333, "avg_duration_per_task_ms": 118630.39, "avg_output_tokens_per_task": 26239.556666666667, "last_run_timestamp": "2026-10-31T12:19:28.999Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.72833325, "accuracy_std_dev": 0.041197373838480274, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011627464999999998, "avg_duration_per_task_ms": 86463.365, "avg_output_tokens_per_task": 10669.235, "last_run_timestamp": "2026-10-31T12:31:32.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.72833325, "accuracy_std_dev": 0.03411588502717615, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.055064995000000005, "avg_duration_per_task_ms": 291890.35, "avg_output_tokens_per_task": 8526.425, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.7281586250000001, "accuracy_std_dev": 0.032945022045826604, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.10040358467741935, "avg_duration_per_task_ms": 94375.23588709677, "avg_output_tokens_per_task": 7576.909274193548, "last_run_timestamp": "2026-09-26T08:26:47.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimeIntellect", "accuracy": 0.728, "accuracy_std_dev": 0.02362657637492154, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.027497304, "avg_duration_per_task_ms": 93734.552, "avg_output_tokens_per_task": 12939.792, "last_run_timestamp": "2026-10-31T12:24:43.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.7277778333333333, "accuracy_std_dev": 0.029165443361489008, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00697867, "avg_duration_per_task_ms": 90387.44333333333, "avg_output_tokens_per_task": 9165.73, "last_run_timestamp": "2026-10-31T11:54:38.723Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.72777775, "accuracy_std_dev": 0.030164136379495988, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.34935106333333343, "avg_duration_per_task_ms": 233197.23, "avg_output_tokens_per_task": 10873.441666666668, "last_run_timestamp": "2026-10-31T12:28:19.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.7274047715780296, "accuracy_std_dev": 0.02610995768010187, "run_count": 23, "total_questions": 1147, "avg_cost_per_task_usd": 0.44006520662598075, "avg_duration_per_task_ms": 249359.09677419355, "avg_output_tokens_per_task": 10802.551002615519, "last_run_timestamp": "2026-10-31T12:24:25.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7273502461538462, "accuracy_std_dev": 0.043866981805083166, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.07634109230769232, "avg_duration_per_task_ms": 620581.2615384615, "avg_output_tokens_per_task": 7602.702564102564, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.7271428571428571, "accuracy_std_dev": 0.01855313018202104, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.31224118, "avg_duration_per_task_ms": 126481.89142857143, "avg_output_tokens_per_task": 5238.86, "last_run_timestamp": "2026-10-01T19:05:16.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.7270409540816326, "accuracy_std_dev": 0.02754153279934688, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.04606436989795917, "avg_duration_per_task_ms": 81576.88520408163, "avg_output_tokens_per_task": 19891.04081632653, "last_run_timestamp": "2026-10-31T12:26:53.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.726667, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.24019618, "avg_duration_per_task_ms": 932058.35, "avg_output_tokens_per_task": 24816.04, "last_run_timestamp": "2026-10-31T11:58:06.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.21341358, "avg_duration_per_task_ms": 101630.84, "avg_output_tokens_per_task": 8827.48, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 2.91364046, "avg_duration_per_task_ms": 331183.04, "avg_output_tokens_per_task": 12422.4, "last_run_timestamp": "2026-10-01T08:02:52.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05448868, "avg_duration_per_task_ms": 350679.6, "avg_output_tokens_per_task": 17629, "last_run_timestamp": "2026-10-01T08:04:38.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.7266667777777778, "accuracy_std_dev": 0.02591526553459167, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06854443777777779, "avg_duration_per_task_ms": 220265.38444444444, "avg_output_tokens_per_task": 13043.293333333333, "last_run_timestamp": "2026-10-31T19:39:42.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.30249026, "avg_duration_per_task_ms": 183822.13333333333, "avg_output_tokens_per_task": 10747.233333333334, "last_run_timestamp": "2026-10-02T23:13:07.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013071513333333333, "avg_duration_per_task_ms": 97597.57333333333, "avg_output_tokens_per_task": 4935.306666666666, "last_run_timestamp": "2026-10-02T23:07:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Modal", "accuracy": 0.7266665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018476000000000003, "avg_duration_per_task_ms": 104934.72, "avg_output_tokens_per_task": 10917.83, "last_run_timestamp": "2026-10-24T11:48:43.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7266665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.43167194999999997, "avg_duration_per_task_ms": 195075.98, "avg_output_tokens_per_task": 31329.14, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Io Net", "accuracy": 0.7266665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06725601, "avg_duration_per_task_ms": 640810.38, "avg_output_tokens_per_task": 15599.02, "last_run_timestamp": "2026-10-27T14:13:59.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.7266663333333333, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.024500659999999997, "avg_duration_per_task_ms": 72543.00666666667, "avg_output_tokens_per_task": 15175.193333333333, "last_run_timestamp": "2026-10-31T11:34:42.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Nebius", "accuracy": 0.7264369310344827, "accuracy_std_dev": 0.006057869592521776, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.012712737931034484, "avg_duration_per_task_ms": 160320.86896551723, "avg_output_tokens_per_task": 8003.303448275862, "last_run_timestamp": "2026-10-31T12:11:53.396Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.726330531092437, "accuracy_std_dev": 0.031188754641318872, "run_count": 12, "total_questions": 595, "avg_cost_per_task_usd": 0.07654581848739497, "avg_duration_per_task_ms": 189475.03361344538, "avg_output_tokens_per_task": 8239.426890756302, "last_run_timestamp": "2026-10-31T19:01:31.914Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7262963333333333, "accuracy_std_dev": 0.03679916650928086, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08815544444444444, "avg_duration_per_task_ms": 169827.67555555556, "avg_output_tokens_per_task": 10443.75111111111, "last_run_timestamp": "2026-10-31T19:56:54.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.7261110833333333, "accuracy_std_dev": 0.02924471198484249, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08110476333333332, "avg_duration_per_task_ms": 226307.265, "avg_output_tokens_per_task": 8776.583333333334, "last_run_timestamp": "2026-10-31T20:37:11.918Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.7256409230769231, "accuracy_std_dev": 0.024647466274717044, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.07311417384615386, "avg_duration_per_task_ms": 221051.06153846154, "avg_output_tokens_per_task": 8508.512307692308, "last_run_timestamp": "2026-10-31T20:10:58.659Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7253334, "accuracy_std_dev": 0.039305065549366525, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.00982467, "avg_duration_per_task_ms": 372894.758, "avg_output_tokens_per_task": 14244.68, "last_run_timestamp": "2026-10-17T13:13:33.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.7253333, "accuracy_std_dev": 0.02809501972610092, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.028140498, "avg_duration_per_task_ms": 228402.532, "avg_output_tokens_per_task": 9725.512, "last_run_timestamp": "2026-10-24T19:14:23.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7251952608695652, "accuracy_std_dev": 0.07868938349065195, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.04124413545150502, "avg_duration_per_task_ms": 274554.9113712375, "avg_output_tokens_per_task": 23809.344481605352, "last_run_timestamp": "2026-10-31T19:30:45.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Sail Research", "accuracy": 0.7244446666666666, "accuracy_std_dev": 0.01662961808207136, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00495835, "avg_duration_per_task_ms": 143781.20333333334, "avg_output_tokens_per_task": 12059.35, "last_run_timestamp": "2026-10-31T12:51:11.039Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7244444444444444, "accuracy_std_dev": 0.022662439694648512, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06561171777777777, "avg_duration_per_task_ms": 200024.29111111112, "avg_output_tokens_per_task": 9471.135555555555, "last_run_timestamp": "2026-10-31T20:17:42.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Phala", "accuracy": 0.7244443333333334, "accuracy_std_dev": 0.006285236475706843, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018751046666666667, "avg_duration_per_task_ms": 183683.96666666667, "avg_output_tokens_per_task": 11588.8, "last_run_timestamp": "2026-10-31T11:59:31.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.7244443333333334, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12152117999999999, "avg_duration_per_task_ms": 471173.63333333336, "avg_output_tokens_per_task": 54771.27333333333, "last_run_timestamp": "2026-10-02T23:27:09.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.724, "accuracy_std_dev": 0.02351363670723862, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.26574487199999997, "avg_duration_per_task_ms": 177434.862, "avg_output_tokens_per_task": 11402.85, "last_run_timestamp": "2026-10-31T12:56:09.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.7238889166666667, "accuracy_std_dev": 0.03479662118285417, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08805141666666667, "avg_duration_per_task_ms": 322075.905, "avg_output_tokens_per_task": 9307.315, "last_run_timestamp": "2026-10-31T19:57:59.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7237037777777777, "accuracy_std_dev": 0.0396418852639976, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10080122000000001, "avg_duration_per_task_ms": 210300.32, "avg_output_tokens_per_task": 11702.788888888888, "last_run_timestamp": "2026-10-31T20:14:27.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Makora", "accuracy": 0.723360643442623, "accuracy_std_dev": 0.019132270021092652, "run_count": 5, "total_questions": 244, "avg_cost_per_task_usd": 0.0756424631147541, "avg_duration_per_task_ms": 76631.30327868853, "avg_output_tokens_per_task": 8337.151639344262, "last_run_timestamp": "2026-10-31T11:46:18.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02063872, "avg_duration_per_task_ms": 638594.61, "avg_output_tokens_per_task": 13867.61, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7233335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11131322, "avg_duration_per_task_ms": 83286.18, "avg_output_tokens_per_task": 3736.91, "last_run_timestamp": "2026-10-01T19:04:40.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02918011, "avg_duration_per_task_ms": 177614.61, "avg_output_tokens_per_task": 35023.7, "last_run_timestamp": "2026-10-31T11:38:56.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33860155, "avg_duration_per_task_ms": 113561.71, "avg_output_tokens_per_task": 10711.4, "last_run_timestamp": "2026-10-01T19:04:58.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7229627777777778, "accuracy_std_dev": 0.03557083313858196, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09454113555555556, "avg_duration_per_task_ms": 220849.31555555554, "avg_output_tokens_per_task": 10002.34, "last_run_timestamp": "2026-10-31T20:09:58.111Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7228571428571429, "accuracy_std_dev": 0.019471637611287507, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10722213142857143, "avg_duration_per_task_ms": 326175.72, "avg_output_tokens_per_task": 14461.362857142858, "last_run_timestamp": "2026-10-17T19:16:53.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "BaseTen", "accuracy": 0.7228067157894738, "accuracy_std_dev": 0.002917999999999976, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.06760965263157895, "avg_duration_per_task_ms": 49831.26315789474, "avg_output_tokens_per_task": 5886.4315789473685, "last_run_timestamp": "2026-10-31T12:12:21.006Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.722650186440678, "accuracy_std_dev": 0.03806732610044826, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.024282243451463792, "avg_duration_per_task_ms": 277851.95069337444, "avg_output_tokens_per_task": 9180.11093990755, "last_run_timestamp": "2026-10-31T13:24:52.766Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.7225, "accuracy_std_dev": 0.029615564602924586, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05079202000000001, "avg_duration_per_task_ms": 151914.885, "avg_output_tokens_per_task": 28118.625, "last_run_timestamp": "2026-10-31T12:23:57.101Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.7223807857142858, "accuracy_std_dev": 0.05901878731650815, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.2836348157142858, "avg_duration_per_task_ms": 433854.6257142857, "avg_output_tokens_per_task": 10133.661428571428, "last_run_timestamp": "2026-10-31T12:33:51.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.011330936570096713, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018792733333333332, "avg_duration_per_task_ms": 301192.17333333334, "avg_output_tokens_per_task": 19692.3, "last_run_timestamp": "2026-10-01T16:00:06.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "SambaNova", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.011330936570096713, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03683, "avg_duration_per_task_ms": 122927.25333333333, "avg_output_tokens_per_task": 9303.18, "last_run_timestamp": "2026-10-31T12:49:19.749Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Particle AI", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.010745106666666667, "avg_duration_per_task_ms": 61185.806666666664, "avg_output_tokens_per_task": 4195.073333333334, "last_run_timestamp": "2026-10-31T11:48:12.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Nebius", "accuracy": 0.72166675, "accuracy_std_dev": 0.012801898733683979, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.024317345, "avg_duration_per_task_ms": 116922.04, "avg_output_tokens_per_task": 11395.83, "last_run_timestamp": "2026-10-31T12:46:14.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Darkbloom", "accuracy": 0.7214610958904109, "accuracy_std_dev": 0.02944405326868039, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.01699455479452055, "avg_duration_per_task_ms": 840567.9794520548, "avg_output_tokens_per_task": 10799.780821917808, "last_run_timestamp": "2026-10-31T13:36:58.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Reka", "accuracy": 0.7213334, "accuracy_std_dev": 0.01146975927559073, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.012899843999999999, "avg_duration_per_task_ms": 152142.784, "avg_output_tokens_per_task": 13063.108, "last_run_timestamp": "2026-10-31T13:10:04.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.721249875, "accuracy_std_dev": 0.02449118393135325, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.045012550000000005, "avg_duration_per_task_ms": 161807.095, "avg_output_tokens_per_task": 30651.7875, "last_run_timestamp": "2026-10-31T11:23:43.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.7211110833333333, "accuracy_std_dev": 0.02579616561771126, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.04659319666666666, "avg_duration_per_task_ms": 177612.27333333335, "avg_output_tokens_per_task": 9673.603333333333, "last_run_timestamp": "2026-10-31T20:01:58.593Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7211021935483871, "accuracy_std_dev": 0.04800607831972945, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.03239710483870968, "avg_duration_per_task_ms": 1093021.4314516129, "avg_output_tokens_per_task": 51597.49596774193, "last_run_timestamp": "2026-10-31T20:03:31.183Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.7210882448979592, "accuracy_std_dev": 0.0076780598851532815, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.009489710204081632, "avg_duration_per_task_ms": 145499.9755102041, "avg_output_tokens_per_task": 16260.81224489796, "last_run_timestamp": "2026-09-26T08:06:43.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7207407777777778, "accuracy_std_dev": 0.01349684513570796, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.052085299999999994, "avg_duration_per_task_ms": 388042.9088888889, "avg_output_tokens_per_task": 9288.517777777777, "last_run_timestamp": "2026-10-31T20:46:45.843Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.72, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.026063740000000002, "avg_duration_per_task_ms": 127597.92, "avg_output_tokens_per_task": 12968.38, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Io Net", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12671924, "avg_duration_per_task_ms": 566714.78, "avg_output_tokens_per_task": 16714.02, "last_run_timestamp": "2026-09-28T18:00:05.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00317077, "avg_duration_per_task_ms": 128719.23, "avg_output_tokens_per_task": 15118.03, "last_run_timestamp": "2026-10-01T08:00:27.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15154258, "avg_duration_per_task_ms": 105755.46, "avg_output_tokens_per_task": 13133.32, "last_run_timestamp": "2026-08-04T03:12:01.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.72, "accuracy_std_dev": 0.01247214674785381, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08792842000000001, "avg_duration_per_task_ms": 120135.21, "avg_output_tokens_per_task": 16462.21, "last_run_timestamp": "2026-10-31T11:24:22.174Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.402901, "avg_duration_per_task_ms": 196016.4, "avg_output_tokens_per_task": 16256.52, "last_run_timestamp": "2026-09-27T00:00:03.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.028463379999999996, "avg_duration_per_task_ms": 583244.64, "avg_output_tokens_per_task": 43866.54, "last_run_timestamp": "2026-08-08T08:27:52.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.7199999166666667, "accuracy_std_dev": 0.032546251989689814, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08362974499999999, "avg_duration_per_task_ms": 133531.81833333333, "avg_output_tokens_per_task": 9309.471666666666, "last_run_timestamp": "2026-10-31T19:18:12.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.7194444166666667, "accuracy_std_dev": 0.02063055611812381, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08198992666666667, "avg_duration_per_task_ms": 234565.96333333335, "avg_output_tokens_per_task": 9245.488333333333, "last_run_timestamp": "2026-10-31T20:11:34.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.71933335, "accuracy_std_dev": 0.026574804694061246, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.065825297, "avg_duration_per_task_ms": 115333.499, "avg_output_tokens_per_task": 8746.383, "last_run_timestamp": "2026-10-31T19:54:14.505Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.7193332, "accuracy_std_dev": 0.03143937799893632, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045152332, "avg_duration_per_task_ms": 326196.928, "avg_output_tokens_per_task": 13427.836, "last_run_timestamp": "2026-10-04T00:00:34.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.7192592222222223, "accuracy_std_dev": 0.03684397384701473, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07047637111111112, "avg_duration_per_task_ms": 259236.1, "avg_output_tokens_per_task": 27382.248888888887, "last_run_timestamp": "2026-10-31T19:22:37.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7186667999999999, "accuracy_std_dev": 0.028095019725211078, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.11915163599999999, "avg_duration_per_task_ms": 86040.636, "avg_output_tokens_per_task": 9027.512, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.7178963360655738, "accuracy_std_dev": 0.15543665632919412, "run_count": 5, "total_questions": 244, "avg_cost_per_task_usd": 0.02473220901639344, "avg_duration_per_task_ms": 313234.66803278687, "avg_output_tokens_per_task": 9615.508196721312, "last_run_timestamp": "2026-10-31T19:23:32.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.717778, "accuracy_std_dev": 0.046930323977857506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.045013793333333336, "avg_duration_per_task_ms": 196458.52, "avg_output_tokens_per_task": 18662.633333333335, "last_run_timestamp": "2026-10-31T19:03:35.054Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Ionstream", "accuracy": 0.7177779999999999, "accuracy_std_dev": 0.03499564699597175, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.039637780000000004, "avg_duration_per_task_ms": 223451.97333333333, "avg_output_tokens_per_task": 45456.886666666665, "last_run_timestamp": "2026-10-31T11:34:19.759Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.7177777777777777, "accuracy_std_dev": 0.02060811299128885, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05747164444444444, "avg_duration_per_task_ms": 334450.40222222224, "avg_output_tokens_per_task": 9986.046666666667, "last_run_timestamp": "2026-10-31T20:51:04.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7177776666666666, "accuracy_std_dev": 0.017497908167041637, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.057615893333333335, "avg_duration_per_task_ms": 509994.74666666664, "avg_output_tokens_per_task": 13843.966666666667, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.7175000624999999, "accuracy_std_dev": 0.03212396291121463, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.11736274687500002, "avg_duration_per_task_ms": 128613.640625, "avg_output_tokens_per_task": 8558.630625, "last_run_timestamp": "2026-10-31T19:56:11.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.717499875, "accuracy_std_dev": 0.03196146008256467, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09209893, "avg_duration_per_task_ms": 157130.905, "avg_output_tokens_per_task": 48565.055, "last_run_timestamp": "2026-10-01T19:06:49.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.717037, "accuracy_std_dev": 0.02668713823382175, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1317152511111111, "avg_duration_per_task_ms": 254884.2288888889, "avg_output_tokens_per_task": 10329.728888888889, "last_run_timestamp": "2026-10-31T20:31:35.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.716667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01519146, "avg_duration_per_task_ms": 819823.24, "avg_output_tokens_per_task": 11470.58, "last_run_timestamp": "2026-10-31T20:19:18.995Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7166669999999999, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1280142, "avg_duration_per_task_ms": 531647.04, "avg_output_tokens_per_task": 56970.52, "last_run_timestamp": "2026-10-01T19:04:42.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7166667777777778, "accuracy_std_dev": 0.026943095041124358, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.2115978688888889, "avg_duration_per_task_ms": 1320901.8111111112, "avg_output_tokens_per_task": 13916.793333333333, "last_run_timestamp": "2026-10-31T19:47:23.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Venice", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.022047789999999998, "avg_duration_per_task_ms": 63248.07, "avg_output_tokens_per_task": 8992.32, "last_run_timestamp": "2026-10-31T12:43:48.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StreamLake", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.015074639999999999, "avg_duration_per_task_ms": 124785.23, "avg_output_tokens_per_task": 11839.88, "last_run_timestamp": "2026-10-31T12:28:39.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.71669186, "avg_duration_per_task_ms": 277432.68, "avg_output_tokens_per_task": 22919.22, "last_run_timestamp": "2026-10-02T23:08:29.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.02603405888927041, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.13610847, "avg_duration_per_task_ms": 230555.42, "avg_output_tokens_per_task": 11694.86, "last_run_timestamp": "2026-10-31T19:46:31.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7159998000000001, "accuracy_std_dev": 0.03295793548085197, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.24450749400000002, "avg_duration_per_task_ms": 314057.314, "avg_output_tokens_per_task": 21575.564, "last_run_timestamp": "2026-10-31T12:46:35.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Together", "accuracy": 0.7155556666666666, "accuracy_std_dev": 0.03189478845969807, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01325666, "avg_duration_per_task_ms": 86099.67333333334, "avg_output_tokens_per_task": 8966.793333333333, "last_run_timestamp": "2026-10-31T13:08:15.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.7155554444444445, "accuracy_std_dev": 0.032203028767531616, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05172563555555556, "avg_duration_per_task_ms": 392299.5088888889, "avg_output_tokens_per_task": 11718.533333333333, "last_run_timestamp": "2026-10-31T20:58:51.733Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.7155554444444445, "accuracy_std_dev": 0.023094000077514658, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05959621777777777, "avg_duration_per_task_ms": 317385.8333333333, "avg_output_tokens_per_task": 9770.851111111111, "last_run_timestamp": "2026-10-31T20:35:24.659Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.715278, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.009582875, "avg_duration_per_task_ms": 164124.5, "avg_output_tokens_per_task": 18445.333333333332, "last_run_timestamp": "2026-10-10T11:44:02.317Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.7133333888888889, "accuracy_std_dev": 0.022553037627726643, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.03885568666666667, "avg_duration_per_task_ms": 225043.39444444445, "avg_output_tokens_per_task": 9769.26, "last_run_timestamp": "2026-10-31T20:38:36.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.7133333333333334, "accuracy_std_dev": 0.03126952723321681, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.04438405333333333, "avg_duration_per_task_ms": 115670.52333333333, "avg_output_tokens_per_task": 8401.7, "last_run_timestamp": "2026-10-31T19:54:42.760Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Particle AI", "accuracy": 0.7133330000000001, "accuracy_std_dev": 0.03265986323710907, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02237405333333333, "avg_duration_per_task_ms": 91226.86, "avg_output_tokens_per_task": 9082.033333333333, "last_run_timestamp": "2026-10-31T12:52:04.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7133330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12082516, "avg_duration_per_task_ms": 865226.27, "avg_output_tokens_per_task": 27649.46, "last_run_timestamp": "2026-08-15T08:15:24.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05452906, "avg_duration_per_task_ms": 272872.18, "avg_output_tokens_per_task": 39719.44, "last_run_timestamp": "2026-10-13T20:01:29.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08007529, "avg_duration_per_task_ms": 140906.22, "avg_output_tokens_per_task": 42659.23, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.006318460000000001, "avg_duration_per_task_ms": 234030.92, "avg_output_tokens_per_task": 14773.36, "last_run_timestamp": "2026-10-01T08:05:57.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Fireworks", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01617958, "avg_duration_per_task_ms": 141600.68, "avg_output_tokens_per_task": 14742.64, "last_run_timestamp": "2026-10-31T12:42:39.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.713333, "accuracy_std_dev": 0.024494897427831803, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.046480935, "avg_duration_per_task_ms": 164672.7, "avg_output_tokens_per_task": 8116.895, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01665106, "avg_duration_per_task_ms": 327560.6, "avg_output_tokens_per_task": 17227.24, "last_run_timestamp": "2026-10-01T08:00:41.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.712766, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.21022357446808512, "avg_duration_per_task_ms": 261220.25531914894, "avg_output_tokens_per_task": 27609.17021276596, "last_run_timestamp": "2026-10-01T08:02:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.7126666, "accuracy_std_dev": 0.019425055985504893, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.02303438, "avg_duration_per_task_ms": 273736.758, "avg_output_tokens_per_task": 10030.944, "last_run_timestamp": "2026-10-31T19:56:09.033Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.712381, "accuracy_std_dev": 0.016106167044245584, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11192505714285715, "avg_duration_per_task_ms": 96202.06, "avg_output_tokens_per_task": 3694.017142857143, "last_run_timestamp": "2026-10-01T19:04:52.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.7123077692307692, "accuracy_std_dev": 0.03859468338864592, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.08011615384615385, "avg_duration_per_task_ms": 106325.87538461538, "avg_output_tokens_per_task": 8598.36, "last_run_timestamp": "2026-10-31T19:45:55.386Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.71222225, "accuracy_std_dev": 0.14864397040699018, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.06236268333333334, "avg_duration_per_task_ms": 148178.685, "avg_output_tokens_per_task": 17104.778333333332, "last_run_timestamp": "2026-10-31T19:40:48.790Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.7117822371364653, "accuracy_std_dev": 0.056118924341169225, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.0462677941834452, "avg_duration_per_task_ms": 1033622.2237136465, "avg_output_tokens_per_task": 60076.49888143177, "last_run_timestamp": "2026-10-31T13:23:14.602Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7113334, "accuracy_std_dev": 0.036884212823916955, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.28188106999999996, "avg_duration_per_task_ms": 497814.602, "avg_output_tokens_per_task": 9499.378, "last_run_timestamp": "2026-10-31T22:14:51.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cohere", "accuracy": 0.7111113333333333, "accuracy_std_dev": 0.006285236475706792, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06754876666666666, "avg_duration_per_task_ms": 139695.35333333333, "avg_output_tokens_per_task": 33581.94, "last_run_timestamp": "2026-10-31T12:01:01.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.7111111111111111, "accuracy_std_dev": 0.012570963999492764, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.027520999999999997, "avg_duration_per_task_ms": 233912.26444444444, "avg_output_tokens_per_task": 10378.048888888889, "last_run_timestamp": "2026-10-31T19:47:12.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.7111111111111111, "accuracy_std_dev": 0.043659620580232664, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06395731555555556, "avg_duration_per_task_ms": 239858.9088888889, "avg_output_tokens_per_task": 8206.18888888889, "last_run_timestamp": "2026-10-31T20:09:59.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7110144347826087, "accuracy_std_dev": 0.024991420835482343, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.10159863652173912, "avg_duration_per_task_ms": 108613.41913043478, "avg_output_tokens_per_task": 9699.877391304348, "last_run_timestamp": "2026-10-31T19:37:51.988Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7104167499999999, "accuracy_std_dev": 0.11449707753448339, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.22292825749999998, "avg_duration_per_task_ms": 1269690.4575, "avg_output_tokens_per_task": 60617.9, "last_run_timestamp": "2026-10-10T19:16:51.412Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7100000833333333, "accuracy_std_dev": 0.030367993020224245, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.12156609333333333, "avg_duration_per_task_ms": 163832.635, "avg_output_tokens_per_task": 11504.31, "last_run_timestamp": "2026-10-31T19:55:04.254Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cohere", "accuracy": 0.71, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.015407820000000001, "avg_duration_per_task_ms": 62181.78, "avg_output_tokens_per_task": 11850.44, "last_run_timestamp": "2026-10-31T11:31:06.991Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "ModelRun", "accuracy": 0.71, "accuracy_std_dev": 0.01795033271279393, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0134254, "avg_duration_per_task_ms": 120168.52, "avg_output_tokens_per_task": 11670.58, "last_run_timestamp": "2026-10-31T12:25:39.138Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.71, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12724396999999998, "avg_duration_per_task_ms": 73218.95, "avg_output_tokens_per_task": 3826.45, "last_run_timestamp": "2026-10-01T19:04:38.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.7093333, "accuracy_std_dev": 0.07889230465267193, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.010360992, "avg_duration_per_task_ms": 240638.89, "avg_output_tokens_per_task": 9031.174, "last_run_timestamp": "2026-10-31T12:26:55.059Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.709166625, "accuracy_std_dev": 0.031786845419675976, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09062656499999999, "avg_duration_per_task_ms": 834592.7325, "avg_output_tokens_per_task": 9529.9875, "last_run_timestamp": "2026-10-31T21:09:07.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7090847895791583, "accuracy_std_dev": 0.018714036978695964, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.0960923246492986, "avg_duration_per_task_ms": 181290.61923847697, "avg_output_tokens_per_task": 10798.256513026052, "last_run_timestamp": "2026-10-17T19:53:39.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.708889, "accuracy_std_dev": 0.03311020878822723, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.025858879999999997, "avg_duration_per_task_ms": 230061.79333333333, "avg_output_tokens_per_task": 9903.024444444445, "last_run_timestamp": "2026-10-31T20:20:16.475Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Novita", "accuracy": 0.708889, "accuracy_std_dev": 0.0031423825355929936, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.017471633333333333, "avg_duration_per_task_ms": 219758.70666666667, "avg_output_tokens_per_task": 16040.473333333333, "last_run_timestamp": "2026-10-31T12:30:16.930Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.708889, "accuracy_std_dev": 0.033554753486735006, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07036646666666667, "avg_duration_per_task_ms": 317115.9511111111, "avg_output_tokens_per_task": 10689.18, "last_run_timestamp": "2026-10-31T21:34:09.712Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.7088888333333333, "accuracy_std_dev": 0.04374444021212551, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11762815500000003, "avg_duration_per_task_ms": 129475.63166666667, "avg_output_tokens_per_task": 9546.961666666666, "last_run_timestamp": "2026-10-31T21:47:15.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.7088556290726817, "accuracy_std_dev": 0.03423477982234403, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.021466571428571425, "avg_duration_per_task_ms": 619938.7243107769, "avg_output_tokens_per_task": 9440.741854636592, "last_run_timestamp": "2026-10-31T19:57:15.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7083335, "accuracy_std_dev": 0.041466338423473054, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.156516845, "avg_duration_per_task_ms": 345707.575, "avg_output_tokens_per_task": 10875.9, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7083333333333334, "accuracy_std_dev": 0.036119498172347615, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.10219764333333331, "avg_duration_per_task_ms": 96190.13, "avg_output_tokens_per_task": 9006.453333333333, "last_run_timestamp": "2026-10-31T19:33:25.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.7083331666666667, "accuracy_std_dev": 0.017400462957602263, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.25373078000000004, "avg_duration_per_task_ms": 176420.9, "avg_output_tokens_per_task": 10605.743333333334, "last_run_timestamp": "2026-10-31T19:48:27.256Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7082050769230769, "accuracy_std_dev": 0.030060181146036768, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.034733135384615386, "avg_duration_per_task_ms": 181977.0169230769, "avg_output_tokens_per_task": 9062.112307692307, "last_run_timestamp": "2026-10-31T20:27:27.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.7081482222222222, "accuracy_std_dev": 0.10187287917659142, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09577372888888888, "avg_duration_per_task_ms": 240048.98666666666, "avg_output_tokens_per_task": 22174.402222222223, "last_run_timestamp": "2026-10-31T19:46:05.991Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7076191428571428, "accuracy_std_dev": 0.061867939965308524, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10378842857142857, "avg_duration_per_task_ms": 217807.51714285713, "avg_output_tokens_per_task": 11023.12, "last_run_timestamp": "2026-10-17T19:18:15.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.707483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.004842469387755102, "avg_duration_per_task_ms": 161558.55102040817, "avg_output_tokens_per_task": 17024.020408163266, "last_run_timestamp": "2026-10-01T08:01:49.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.706667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.44632813, "avg_duration_per_task_ms": 132811.5, "avg_output_tokens_per_task": 20668.24, "last_run_timestamp": "2026-10-01T19:04:52.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.71176192, "avg_duration_per_task_ms": 285827.28, "avg_output_tokens_per_task": 22895.76, "last_run_timestamp": "2026-10-01T16:00:34.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1838298, "avg_duration_per_task_ms": 329811.9, "avg_output_tokens_per_task": 9540.84, "last_run_timestamp": "2026-08-15T00:16:18.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 3.04321209, "avg_duration_per_task_ms": 317897.29, "avg_output_tokens_per_task": 13272.53, "last_run_timestamp": "2026-10-02T23:13:16.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1042244, "avg_duration_per_task_ms": 226024.67, "avg_output_tokens_per_task": 3543.23, "last_run_timestamp": "2026-10-30T00:17:12.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "GMICloud", "accuracy": 0.70666675, "accuracy_std_dev": 0.027080004924067123, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.017216295, "avg_duration_per_task_ms": 228198.02, "avg_output_tokens_per_task": 15657.435, "last_run_timestamp": "2026-10-31T13:09:38.196Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.70666675, "accuracy_std_dev": 0.027080004924067123, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04331178, "avg_duration_per_task_ms": 178565.92, "avg_output_tokens_per_task": 8366.805, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DigitalOcean", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.015417799999999999, "avg_duration_per_task_ms": 75266.33333333333, "avg_output_tokens_per_task": 12018.02, "last_run_timestamp": "2026-10-31T11:20:39.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.5994524266666668, "avg_duration_per_task_ms": 636541.5466666666, "avg_output_tokens_per_task": 54354.86, "last_run_timestamp": "2026-10-02T23:22:41.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.01962587106064057, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08575882, "avg_duration_per_task_ms": 316120.8933333333, "avg_output_tokens_per_task": 32977.22666666667, "last_run_timestamp": "2026-10-02T23:12:53.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7061111666666666, "accuracy_std_dev": 0.023602995992152234, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.023716863333333334, "avg_duration_per_task_ms": 111467.81333333334, "avg_output_tokens_per_task": 29595.083333333332, "last_run_timestamp": "2026-10-31T19:52:48.234Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.7059259999999999, "accuracy_std_dev": 0.05168229207593467, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03188982666666667, "avg_duration_per_task_ms": 346128.37333333335, "avg_output_tokens_per_task": 10303.717777777778, "last_run_timestamp": "2026-10-31T20:10:49.937Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.7058029271255061, "accuracy_std_dev": 0.021947489401751626, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.012539862348178138, "avg_duration_per_task_ms": 294724.995951417, "avg_output_tokens_per_task": 13911.971659919029, "last_run_timestamp": "2026-10-31T12:54:14.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7057145714285713, "accuracy_std_dev": 0.030327119046712443, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.012995502857142857, "avg_duration_per_task_ms": 102610.46857142857, "avg_output_tokens_per_task": 4898.9228571428575, "last_run_timestamp": "2026-10-01T19:05:16.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.7051282307692308, "accuracy_std_dev": 0.03309201444725772, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.04285510153846153, "avg_duration_per_task_ms": 142444.70769230768, "avg_output_tokens_per_task": 8502.066153846154, "last_run_timestamp": "2026-10-31T20:18:13.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.7050002499999999, "accuracy_std_dev": 0.029580300314356156, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050410445, "avg_duration_per_task_ms": 854714.71, "avg_output_tokens_per_task": 8739.19, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.70499975, "accuracy_std_dev": 0.011902485735656231, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.13260128, "avg_duration_per_task_ms": 477326.39, "avg_output_tokens_per_task": 55745.89, "last_run_timestamp": "2026-10-01T19:05:16.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7045706153846154, "accuracy_std_dev": 0.020768805526558354, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.049576892976588634, "avg_duration_per_task_ms": 412685.872909699, "avg_output_tokens_per_task": 8662.418060200669, "last_run_timestamp": "2026-10-31T21:01:44.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.7040814081632653, "accuracy_std_dev": 0.029861000000000026, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.12006706122448979, "avg_duration_per_task_ms": 975720.112244898, "avg_output_tokens_per_task": 30678.173469387755, "last_run_timestamp": "2026-08-15T14:15:18.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Makora", "accuracy": 0.7039998000000001, "accuracy_std_dev": 0.025508095447524113, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.021704752, "avg_duration_per_task_ms": 119683.016, "avg_output_tokens_per_task": 11904.78, "last_run_timestamp": "2026-10-31T12:25:42.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7033335, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08534018, "avg_duration_per_task_ms": 347658.09, "avg_output_tokens_per_task": 32908.85, "last_run_timestamp": "2026-10-01T16:00:47.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.703333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05425128, "avg_duration_per_task_ms": 396980.46, "avg_output_tokens_per_task": 20673.9, "last_run_timestamp": "2026-10-01T08:05:20.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.7031330647149461, "accuracy_std_dev": 0.03140917629230264, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.07489512018489985, "avg_duration_per_task_ms": 165770.67642526966, "avg_output_tokens_per_task": 9459.0030816641, "last_run_timestamp": "2026-10-31T19:41:05.183Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.7025, "accuracy_std_dev": 0.025353913425741587, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11362007500000001, "avg_duration_per_task_ms": 91835.93333333333, "avg_output_tokens_per_task": 9769.808333333332, "last_run_timestamp": "2026-10-31T19:57:32.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.021999169459070246, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10818771333333332, "avg_duration_per_task_ms": 79804.25333333333, "avg_output_tokens_per_task": 3542.24, "last_run_timestamp": "2026-10-02T23:10:36.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13357544, "avg_duration_per_task_ms": 105586.74666666667, "avg_output_tokens_per_task": 5260.826666666667, "last_run_timestamp": "2026-10-02T23:07:17.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.027755546660882827, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04592378666666666, "avg_duration_per_task_ms": 539584.4533333334, "avg_output_tokens_per_task": 9026.931111111111, "last_run_timestamp": "2026-10-31T20:03:54.586Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.7019519504504504, "accuracy_std_dev": 0.09294428248162816, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.1323645135135135, "avg_duration_per_task_ms": 1692899.1238738738, "avg_output_tokens_per_task": 97433.60585585586, "last_run_timestamp": "2026-10-31T19:11:32.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.7018970975609756, "accuracy_std_dev": 0.11960704720475296, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.0732234268292683, "avg_duration_per_task_ms": 155630.15853658537, "avg_output_tokens_per_task": 15582.174796747968, "last_run_timestamp": "2026-10-03T16:16:30.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.701111, "accuracy_std_dev": 0.005665795148667743, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05215248, "avg_duration_per_task_ms": 222421.00666666665, "avg_output_tokens_per_task": 6776.133333333333, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.701111, "accuracy_std_dev": 0.02276020351988678, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.14173139733333334, "avg_duration_per_task_ms": 80397.728, "avg_output_tokens_per_task": 10481.090666666667, "last_run_timestamp": "2026-10-24T19:06:47.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.7010051055276382, "accuracy_std_dev": 0.03189651765083454, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.016903984924623115, "avg_duration_per_task_ms": 263739.4221105528, "avg_output_tokens_per_task": 8905.97487437186, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.7007407777777778, "accuracy_std_dev": 0.04138870710116677, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06795873333333333, "avg_duration_per_task_ms": 72240.74888888889, "avg_output_tokens_per_task": 9788.00888888889, "last_run_timestamp": "2026-10-31T19:23:52.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.7004591575757575, "accuracy_std_dev": 0.0304287066932544, "run_count": 37, "total_questions": 1815, "avg_cost_per_task_usd": 0.10116235151515152, "avg_duration_per_task_ms": 72387.2787878788, "avg_output_tokens_per_task": 7992.845730027549, "last_run_timestamp": "2026-10-31T20:07:10.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29769207999999997, "avg_duration_per_task_ms": 128375.6, "avg_output_tokens_per_task": 8500.86, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Google", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0355865, "avg_duration_per_task_ms": 154491.8, "avg_output_tokens_per_task": 12574.4, "last_run_timestamp": "2026-10-31T11:15:31.400Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00357748, "avg_duration_per_task_ms": 48793.08, "avg_output_tokens_per_task": 1567.76, "last_run_timestamp": "2026-10-01T23:50:45.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0030046400000000003, "avg_duration_per_task_ms": 98848.12, "avg_output_tokens_per_task": 17113.74, "last_run_timestamp": "2026-10-01T08:00:19.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12726106, "avg_duration_per_task_ms": 181226.44, "avg_output_tokens_per_task": 9838.56, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "StreamLake", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0161093, "avg_duration_per_task_ms": 215051.72, "avg_output_tokens_per_task": 15584.08, "last_run_timestamp": "2026-10-31T12:24:55.227Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07376152, "avg_duration_per_task_ms": 750952.84, "avg_output_tokens_per_task": 11282.28, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": 0.004714280910170736, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.634261665, "avg_duration_per_task_ms": 673734.145, "avg_output_tokens_per_task": 52923.67, "last_run_timestamp": "2026-10-01T16:01:02.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1012818, "avg_duration_per_task_ms": 144103.72, "avg_output_tokens_per_task": 3638.14, "last_run_timestamp": "2026-10-01T08:03:00.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "OpenAI", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05778106, "avg_duration_per_task_ms": 103456.94, "avg_output_tokens_per_task": 4693.4, "last_run_timestamp": "2026-10-28T00:06:02.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.6995834999999999, "accuracy_std_dev": 0.07598327392755594, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0480097725, "avg_duration_per_task_ms": 463483.8175, "avg_output_tokens_per_task": 15486.0875, "last_run_timestamp": "2026-10-31T19:37:26.520Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.6992592222222223, "accuracy_std_dev": 0.03968313867732682, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06891943333333334, "avg_duration_per_task_ms": 354863.96, "avg_output_tokens_per_task": 10019.886666666667, "last_run_timestamp": "2026-10-31T20:44:35.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.6987950281124499, "accuracy_std_dev": 0.0269759350636822, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.02643562248995984, "avg_duration_per_task_ms": 223792.88353413655, "avg_output_tokens_per_task": 9633.240963855422, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.6983333125, "accuracy_std_dev": 0.03425807589408435, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.127335795, "avg_duration_per_task_ms": 128403.7125, "avg_output_tokens_per_task": 8998.39, "last_run_timestamp": "2026-10-31T20:01:39.270Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Parasail", "accuracy": 0.6977776666666667, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013071946666666667, "avg_duration_per_task_ms": 88036.78, "avg_output_tokens_per_task": 11978.486666666666, "last_run_timestamp": "2026-10-31T11:55:17.583Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.6977776666666667, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.14221055333333335, "avg_duration_per_task_ms": 130884.03333333334, "avg_output_tokens_per_task": 4996.426666666666, "last_run_timestamp": "2026-10-30T00:14:00.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.6970371111111111, "accuracy_std_dev": 0.02933193693056708, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05237315555555556, "avg_duration_per_task_ms": 363352.25111111114, "avg_output_tokens_per_task": 9093.155555555555, "last_run_timestamp": "2026-10-31T22:10:13.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.696667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01321477, "avg_duration_per_task_ms": 116452.62, "avg_output_tokens_per_task": 5139.02, "last_run_timestamp": "2026-10-01T19:04:36.191Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.62488186, "avg_duration_per_task_ms": 152081.62, "avg_output_tokens_per_task": 3927.45, "last_run_timestamp": "2026-10-01T08:05:41.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Io Net", "accuracy": 0.6963472739726028, "accuracy_std_dev": 0.05315590618172168, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.01628368493150685, "avg_duration_per_task_ms": 275676.23287671234, "avg_output_tokens_per_task": 12531.828767123288, "last_run_timestamp": "2026-10-31T13:29:36.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.6958335, "accuracy_std_dev": 0.03781341986980283, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.132227115, "avg_duration_per_task_ms": 205681.995, "avg_output_tokens_per_task": 9407.695, "last_run_timestamp": "2026-10-01T22:50:26.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.6948196351351352, "accuracy_std_dev": 0.0038228551575432913, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.005936054054054054, "avg_duration_per_task_ms": 331167.7297297297, "avg_output_tokens_per_task": 9197.358108108108, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6946665000000001, "accuracy_std_dev": 0.013597418058219743, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.02175149, "avg_duration_per_task_ms": 498490.026, "avg_output_tokens_per_task": 18620.424, "last_run_timestamp": "2026-10-31T19:50:01.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6933334999999999, "accuracy_std_dev": 0.018257236010689022, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08853248000000001, "avg_duration_per_task_ms": 338024.125, "avg_output_tokens_per_task": 32557.16, "last_run_timestamp": "2026-10-01T16:01:04.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.014401903168987411, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03953364, "avg_duration_per_task_ms": 309829.13333333336, "avg_output_tokens_per_task": 32261.04, "last_run_timestamp": "2026-10-02T23:10:33.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05167764, "avg_duration_per_task_ms": 69227.88, "avg_output_tokens_per_task": 9318.98, "last_run_timestamp": "2026-08-08T08:31:07.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12999802000000002, "avg_duration_per_task_ms": 96250.54, "avg_output_tokens_per_task": 4982.06, "last_run_timestamp": "2026-10-01T08:05:35.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02754302, "avg_duration_per_task_ms": 209408.8, "avg_output_tokens_per_task": 14914.54, "last_run_timestamp": "2026-10-24T12:03:24.771Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.6918933673469387, "accuracy_std_dev": 0.11325810487735402, "run_count": 12, "total_questions": 588, "avg_cost_per_task_usd": 0.3106834897959183, "avg_duration_per_task_ms": 1590211.7602040817, "avg_output_tokens_per_task": 51719.284013605444, "last_run_timestamp": "2026-10-31T12:54:39.225Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6914284285714286, "accuracy_std_dev": 0.02217231732110459, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.10640983571428575, "avg_duration_per_task_ms": 75797.385, "avg_output_tokens_per_task": 9708.589285714286, "last_run_timestamp": "2026-10-31T19:55:55.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "BaseTen", "accuracy": 0.6909721458333333, "accuracy_std_dev": 0.0024635000000000074, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.0246803125, "avg_duration_per_task_ms": 48638.177083333336, "avg_output_tokens_per_task": 11672.458333333334, "last_run_timestamp": "2026-10-31T11:12:51.723Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.69, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10882898999999999, "avg_duration_per_task_ms": 135482.83, "avg_output_tokens_per_task": 51335.48, "last_run_timestamp": "2026-10-01T19:05:16.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Minimax", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01750064, "avg_duration_per_task_ms": 190406.46, "avg_output_tokens_per_task": 15714.76, "last_run_timestamp": "2026-10-31T12:31:30.661Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "PrimeIntellect", "accuracy": 0.69, "accuracy_std_dev": 0.016667000000000043, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01877389, "avg_duration_per_task_ms": 125629.58, "avg_output_tokens_per_task": 15732.46, "last_run_timestamp": "2026-10-31T11:20:36.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 2.99530482, "avg_duration_per_task_ms": 320001.13, "avg_output_tokens_per_task": 12475.35, "last_run_timestamp": "2026-10-01T08:05:20.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.69, "accuracy_std_dev": 0.04333300000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07366, "avg_duration_per_task_ms": 63569.01, "avg_output_tokens_per_task": 3649.5, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.6877778333333333, "accuracy_std_dev": 0.1755169025663119, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.0163179, "avg_duration_per_task_ms": 360518.31666666665, "avg_output_tokens_per_task": 11107.656666666666, "last_run_timestamp": "2026-10-31T11:27:39.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6874072222222223, "accuracy_std_dev": 0.042449001759700566, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.016434591111111112, "avg_duration_per_task_ms": 373578.1911111111, "avg_output_tokens_per_task": 19878.99111111111, "last_run_timestamp": "2026-10-31T20:18:22.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.6873796242774566, "accuracy_std_dev": 0.023430359614685144, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.021091329479768786, "avg_duration_per_task_ms": 336802.3034682081, "avg_output_tokens_per_task": 17067.51156069364, "last_run_timestamp": "2026-10-31T21:08:27.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00219582, "avg_duration_per_task_ms": 78121.92, "avg_output_tokens_per_task": 11659.46, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Azure", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06838028, "avg_duration_per_task_ms": 78879.18, "avg_output_tokens_per_task": 4783.1, "last_run_timestamp": "2026-10-28T00:05:23.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.6866666923076923, "accuracy_std_dev": 0.06431750303513466, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.11028715384615387, "avg_duration_per_task_ms": 78688.8476923077, "avg_output_tokens_per_task": 9043.872307692307, "last_run_timestamp": "2026-10-31T19:47:27.755Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Wafer", "accuracy": 0.6866665000000001, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01561806, "avg_duration_per_task_ms": 117250.14, "avg_output_tokens_per_task": 14519.81, "last_run_timestamp": "2026-10-24T12:06:50.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.6861537692307692, "accuracy_std_dev": 0.04555144565493437, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.09695205384615385, "avg_duration_per_task_ms": 189917.99538461538, "avg_output_tokens_per_task": 7426.601538461538, "last_run_timestamp": "2026-10-31T19:10:43.801Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.6854528208092485, "accuracy_std_dev": 0.016921495753349107, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.03335441329479769, "avg_duration_per_task_ms": 533340.3294797688, "avg_output_tokens_per_task": 18704.07225433526, "last_run_timestamp": "2026-10-17T19:21:01.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6850001666666666, "accuracy_std_dev": 0.0143694957278334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04138883, "avg_duration_per_task_ms": 242662.93333333332, "avg_output_tokens_per_task": 30866.096666666668, "last_run_timestamp": "2026-10-01T08:05:57.699Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.685000125, "accuracy_std_dev": 0.03693083437805562, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0358575875, "avg_duration_per_task_ms": 157897.08, "avg_output_tokens_per_task": 9891.505, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.6834170150753769, "accuracy_std_dev": 0.019760471659034357, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.006376396984924624, "avg_duration_per_task_ms": 294466.5879396985, "avg_output_tokens_per_task": 9614.21608040201, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "PrimeIntellect", "accuracy": 0.6833334999999999, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018626550000000002, "avg_duration_per_task_ms": 239747.89, "avg_output_tokens_per_task": 15949.04, "last_run_timestamp": "2026-10-31T19:36:36.917Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.07779388246014092, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00909372, "avg_duration_per_task_ms": 253755.2, "avg_output_tokens_per_task": 15941.146666666667, "last_run_timestamp": "2026-10-08T22:38:42.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.012619766977070334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10710260999999997, "avg_duration_per_task_ms": 135398.91, "avg_output_tokens_per_task": 4027.3166666666666, "last_run_timestamp": "2026-10-01T08:06:09.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.021998563367234276, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.014046139999999999, "avg_duration_per_task_ms": 71689.16, "avg_output_tokens_per_task": 5022.28, "last_run_timestamp": "2026-10-30T00:15:40.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09746202, "avg_duration_per_task_ms": 132162.47333333333, "avg_output_tokens_per_task": 3507.6066666666666, "last_run_timestamp": "2026-10-02T23:10:34.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.6813334, "accuracy_std_dev": 0.0026667999999999913, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08704178, "avg_duration_per_task_ms": 82752.556, "avg_output_tokens_per_task": 3576.016, "last_run_timestamp": "2026-10-01T16:01:04.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.68125, "accuracy_std_dev": 0.0202715728731147, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0174918175, "avg_duration_per_task_ms": 123091.5525, "avg_output_tokens_per_task": 21340.095, "last_run_timestamp": "2026-10-31T11:46:38.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.6811111666666666, "accuracy_std_dev": 0.008088865998326882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00903582, "avg_duration_per_task_ms": 105469.92666666667, "avg_output_tokens_per_task": 8459.866666666667, "last_run_timestamp": "2026-10-01T08:05:22.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.58024224, "avg_duration_per_task_ms": 141487.98, "avg_output_tokens_per_task": 3731.58, "last_run_timestamp": "2026-10-01T08:05:19.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.68, "accuracy_std_dev": 0.03810292105162893, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00883898, "avg_duration_per_task_ms": 114396.12, "avg_output_tokens_per_task": 8841.28, "last_run_timestamp": "2026-10-02T23:08:44.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.6799998571428572, "accuracy_std_dev": 0.0198405229080613, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.015568199999999999, "avg_duration_per_task_ms": 259045.84285714285, "avg_output_tokens_per_task": 18504.7, "last_run_timestamp": "2026-10-31T20:09:24.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.6793939999999999, "accuracy_std_dev": 0.03428396622064286, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.017193465454545455, "avg_duration_per_task_ms": 303188.84545454546, "avg_output_tokens_per_task": 19789.663636363635, "last_run_timestamp": "2026-10-31T21:04:41.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.678333, "accuracy_std_dev": 0.0050000000000000044, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0069528, "avg_duration_per_task_ms": 471037.61, "avg_output_tokens_per_task": 19230.47, "last_run_timestamp": "2026-10-31T13:27:50.961Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.67666675, "accuracy_std_dev": 0.028867484592314224, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.008505605000000001, "avg_duration_per_task_ms": 317641.53, "avg_output_tokens_per_task": 13432.655, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5967202500000001, "avg_duration_per_task_ms": 137724.15, "avg_output_tokens_per_task": 4015.19, "last_run_timestamp": "2026-10-02T23:06:01.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6757320878661087, "accuracy_std_dev": 0.02326584314053542, "run_count": 5, "total_questions": 239, "avg_cost_per_task_usd": 0.14251672803347282, "avg_duration_per_task_ms": 407673.5857740586, "avg_output_tokens_per_task": 10868.456066945606, "last_run_timestamp": "2026-10-17T19:51:53.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.675675972972973, "accuracy_std_dev": 0.02284471405428887, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04042408108108108, "avg_duration_per_task_ms": 518660.3783783784, "avg_output_tokens_per_task": 20990.527027027027, "last_run_timestamp": "2026-09-26T08:11:26.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.00831485358994507, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.014157986666666667, "avg_duration_per_task_ms": 117427.1, "avg_output_tokens_per_task": 13587.66, "last_run_timestamp": "2026-10-31T11:44:53.544Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.63842362, "avg_duration_per_task_ms": 88201.35333333333, "avg_output_tokens_per_task": 3898.48, "last_run_timestamp": "2026-10-30T00:17:00.667Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07120372000000001, "avg_duration_per_task_ms": 144319.38666666666, "avg_output_tokens_per_task": 2561.846666666667, "last_run_timestamp": "2026-10-08T22:25:52.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.6755554444444445, "accuracy_std_dev": 0.10270425478377884, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.017111031111111112, "avg_duration_per_task_ms": 257153.8, "avg_output_tokens_per_task": 10465.493333333334, "last_run_timestamp": "2026-10-31T19:58:02.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.675, "accuracy_std_dev": 0.011667000000000038, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1079227, "avg_duration_per_task_ms": 145751.42, "avg_output_tokens_per_task": 18113.26, "last_run_timestamp": "2026-10-04T05:20:00.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09937637, "avg_duration_per_task_ms": 637553.26, "avg_output_tokens_per_task": 9917.08, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.6292479799999999, "avg_duration_per_task_ms": 729264.79, "avg_output_tokens_per_task": 54931.75, "last_run_timestamp": "2026-10-01T16:01:02.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Tenstorrent", "accuracy": 0.6733333333333333, "accuracy_std_dev": 0.01962587106064057, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012727326666666665, "avg_duration_per_task_ms": 155983.29333333333, "avg_output_tokens_per_task": 15660.926666666666, "last_run_timestamp": "2026-10-31T12:26:47.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.6733330000000001, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.5819717300000002, "avg_duration_per_task_ms": 314318.37, "avg_output_tokens_per_task": 23725.55, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00928508, "avg_duration_per_task_ms": 121838.98, "avg_output_tokens_per_task": 8898.44, "last_run_timestamp": "2026-10-01T08:00:22.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.672, "accuracy_std_dev": 0.028409779224766934, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07962264, "avg_duration_per_task_ms": 80557.024, "avg_output_tokens_per_task": 3531.236, "last_run_timestamp": "2026-10-01T16:00:45.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.6719998, "accuracy_std_dev": 0.012927091279943813, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.015563263999999999, "avg_duration_per_task_ms": 230227.66, "avg_output_tokens_per_task": 10531.424, "last_run_timestamp": "2026-10-13T22:01:15.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.6713710161290322, "accuracy_std_dev": 0.029936692690409203, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.02159695766129032, "avg_duration_per_task_ms": 287822.4233870968, "avg_output_tokens_per_task": 16757.65927419355, "last_run_timestamp": "2026-10-31T20:14:20.109Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.6706668, "accuracy_std_dev": 0.015548970672041286, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.142799016, "avg_duration_per_task_ms": 115124.684, "avg_output_tokens_per_task": 5068.812, "last_run_timestamp": "2026-10-01T08:04:38.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.6693334, "accuracy_std_dev": 0.07285906090281427, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.09112218200000001, "avg_duration_per_task_ms": 124279.178, "avg_output_tokens_per_task": 8535.108, "last_run_timestamp": "2026-10-31T19:19:57.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.6690909090909091, "accuracy_std_dev": 0.14703195632270777, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.12102286727272728, "avg_duration_per_task_ms": 359186.4090909091, "avg_output_tokens_per_task": 8810.841818181818, "last_run_timestamp": "2026-10-31T19:48:58.813Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.6686666, "accuracy_std_dev": 0.08499931398452577, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.05116424000000001, "avg_duration_per_task_ms": 341233.54, "avg_output_tokens_per_task": 7834.636, "last_run_timestamp": "2026-10-03T16:13:53.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.6684915255474452, "accuracy_std_dev": 0.1213625557738751, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.040750651459854015, "avg_duration_per_task_ms": 1768695.9981751824, "avg_output_tokens_per_task": 66240.04562043796, "last_run_timestamp": "2026-10-31T11:55:35.800Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.66833325, "accuracy_std_dev": 0.005527532106419614, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.027697885000000002, "avg_duration_per_task_ms": 410531.75, "avg_output_tokens_per_task": 10158.81, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.038762819999999996, "avg_duration_per_task_ms": 314594.08, "avg_output_tokens_per_task": 31952.48, "last_run_timestamp": "2026-10-01T08:00:24.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Sail Research", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.013241780000000002, "avg_duration_per_task_ms": 134332.46, "avg_output_tokens_per_task": 19207.38, "last_run_timestamp": "2026-10-31T19:46:36.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.66666675, "accuracy_std_dev": 0.016996797086142445, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.008543925, "avg_duration_per_task_ms": 288167.26, "avg_output_tokens_per_task": 9621.62, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "PrimeIntellect", "accuracy": 0.6666666666666666, "accuracy_std_dev": 0.0524933858267454, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03491572666666667, "avg_duration_per_task_ms": 274075.64666666667, "avg_output_tokens_per_task": 23860.97333333333, "last_run_timestamp": "2026-10-31T19:43:40.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "auto-routing", "accuracy": 0.6666665, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.013174700000000001, "avg_duration_per_task_ms": 181950.26, "avg_output_tokens_per_task": 13269.31, "last_run_timestamp": "2026-10-31T12:42:39.997Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.6666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08889954, "avg_duration_per_task_ms": 108433.06, "avg_output_tokens_per_task": 12861.73, "last_run_timestamp": "2026-10-01T19:04:42.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6661110555555556, "accuracy_std_dev": 0.029580425001590006, "run_count": 36, "total_questions": 1800, "avg_cost_per_task_usd": 0.12436681222222222, "avg_duration_per_task_ms": 119159.24222222222, "avg_output_tokens_per_task": 12659.702222222222, "last_run_timestamp": "2026-10-31T12:26:00.225Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.66583325, "accuracy_std_dev": 0.03639120582211448, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.006095729999999999, "avg_duration_per_task_ms": 325369.79, "avg_output_tokens_per_task": 18643.27, "last_run_timestamp": "2026-10-31T19:46:03.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.6632998585858586, "accuracy_std_dev": 0.0034015000000000017, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.07516567676767677, "avg_duration_per_task_ms": 78894.82828282828, "avg_output_tokens_per_task": 13736.282828282829, "last_run_timestamp": "2026-08-15T14:12:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.662963, "accuracy_std_dev": 0.06555468465673864, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05622624222222223, "avg_duration_per_task_ms": 819076.5222222222, "avg_output_tokens_per_task": 18202.83777777778, "last_run_timestamp": "2026-10-05T20:43:11.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.6616665833333334, "accuracy_std_dev": 0.0293288743461068, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11567571166666667, "avg_duration_per_task_ms": 123049.12333333334, "avg_output_tokens_per_task": 12632.496666666666, "last_run_timestamp": "2026-10-31T12:42:18.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6613334000000001, "accuracy_std_dev": 0.01758814752724117, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10186603200000001, "avg_duration_per_task_ms": 156255.32, "avg_output_tokens_per_task": 13208.728, "last_run_timestamp": "2026-10-01T19:04:42.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.6604760000000001, "accuracy_std_dev": 0.025350253033394797, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.008717225714285714, "avg_duration_per_task_ms": 574345.4257142857, "avg_output_tokens_per_task": 17634.762857142858, "last_run_timestamp": "2026-10-31T20:07:05.983Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.6600000833333333, "accuracy_std_dev": 0.02652737928838283, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11395726666666665, "avg_duration_per_task_ms": 127166.50666666667, "avg_output_tokens_per_task": 12326.945, "last_run_timestamp": "2026-10-31T11:44:30.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.6583334999999999, "accuracy_std_dev": 0.028037782049406136, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02298765, "avg_duration_per_task_ms": 355829.865, "avg_output_tokens_per_task": 10214.775, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.6580179999999999, "accuracy_std_dev": 0.027893658429732626, "run_count": 37, "total_questions": 1850, "avg_cost_per_task_usd": 0.12081404162162164, "avg_duration_per_task_ms": 121162.63837837838, "avg_output_tokens_per_task": 12580.847027027026, "last_run_timestamp": "2026-10-31T12:17:41.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6579165, "accuracy_std_dev": 0.013326680635101876, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.023098354999999997, "avg_duration_per_task_ms": 154244.335, "avg_output_tokens_per_task": 17007.6275, "last_run_timestamp": "2026-10-31T18:35:38.514Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.65666675, "accuracy_std_dev": 0.0074535971978300555, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02864462, "avg_duration_per_task_ms": 223453.205, "avg_output_tokens_per_task": 10372.695, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.6564246703910614, "accuracy_std_dev": 0.022405218838489775, "run_count": 18, "total_questions": 895, "avg_cost_per_task_usd": 0.01014628938547486, "avg_duration_per_task_ms": 302924.7284916201, "avg_output_tokens_per_task": 17044.198882681565, "last_run_timestamp": "2026-10-31T19:05:31.409Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6562239298245615, "accuracy_std_dev": 0.018946289616967236, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.01692722807017544, "avg_duration_per_task_ms": 312804.7944862155, "avg_output_tokens_per_task": 22811.754385964912, "last_run_timestamp": "2026-10-31T12:28:49.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6557823102040816, "accuracy_std_dev": 0.009318408855593308, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.014797057142857142, "avg_duration_per_task_ms": 277907.2775510204, "avg_output_tokens_per_task": 20245.579591836733, "last_run_timestamp": "2026-10-10T11:18:54.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6555556666666666, "accuracy_std_dev": 0.03807065880409451, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021039175555555557, "avg_duration_per_task_ms": 248791.5977777778, "avg_output_tokens_per_task": 26269.908888888887, "last_run_timestamp": "2026-10-31T12:25:21.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.6555556666666666, "accuracy_std_dev": 0.012570472951413636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.030086579999999998, "avg_duration_per_task_ms": 125366.48, "avg_output_tokens_per_task": 16133.046666666667, "last_run_timestamp": "2026-10-02T23:09:54.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.6546664, "accuracy_std_dev": 0.03512514515614133, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.01491196, "avg_duration_per_task_ms": 86567.944, "avg_output_tokens_per_task": 19737.508, "last_run_timestamp": "2026-10-31T19:08:06.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.654279400900901, "accuracy_std_dev": 0.11858723742179567, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.025987085585585587, "avg_duration_per_task_ms": 1029787.9324324324, "avg_output_tokens_per_task": 34152.32432432433, "last_run_timestamp": "2026-10-17T21:28:32.796Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.6533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00488981, "avg_duration_per_task_ms": 111736.2, "avg_output_tokens_per_task": 17377.29, "last_run_timestamp": "2026-10-01T08:04:38.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6533333333333333, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.2856746933333333, "avg_duration_per_task_ms": 183725.1, "avg_output_tokens_per_task": 8628.813333333334, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00923216, "avg_duration_per_task_ms": 145753.66, "avg_output_tokens_per_task": 8192.38, "last_run_timestamp": "2026-09-23T06:00:08.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.6518518888888889, "accuracy_std_dev": 0.03247467220412599, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.012659444444444444, "avg_duration_per_task_ms": 517342.6377777778, "avg_output_tokens_per_task": 18101.524444444443, "last_run_timestamp": "2026-10-31T19:41:05.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6518188195991091, "accuracy_std_dev": 0.02538535107955173, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.012973, "avg_duration_per_task_ms": 774691.5612472161, "avg_output_tokens_per_task": 26354.719376391982, "last_run_timestamp": "2026-10-31T19:22:53.448Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "AkashML", "accuracy": 0.65166675, "accuracy_std_dev": 0.028819327042585528, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01676374, "avg_duration_per_task_ms": 406825.29, "avg_output_tokens_per_task": 27732.185, "last_run_timestamp": "2026-10-31T19:20:36.815Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.65166675, "accuracy_std_dev": 0.04506177549972369, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04474220500000001, "avg_duration_per_task_ms": 212693.005, "avg_output_tokens_per_task": 3236.785, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.6505556666666666, "accuracy_std_dev": 0.024107646962230795, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11465736166666668, "avg_duration_per_task_ms": 137405.46666666667, "avg_output_tokens_per_task": 12547.148333333333, "last_run_timestamp": "2026-10-31T11:42:13.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.6502015887096775, "accuracy_std_dev": 0.03709283547317459, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.01248410685483871, "avg_duration_per_task_ms": 1080016.7157258065, "avg_output_tokens_per_task": 22928.112903225807, "last_run_timestamp": "2026-10-31T14:30:58.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.6500001111111111, "accuracy_std_dev": 0.032356116448892955, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03375578888888889, "avg_duration_per_task_ms": 60946.455555555556, "avg_output_tokens_per_task": 14806.375555555556, "last_run_timestamp": "2026-10-31T18:54:40.763Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.6488889999999999, "accuracy_std_dev": 0.019115994210782406, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08501352666666667, "avg_duration_per_task_ms": 97755.09333333334, "avg_output_tokens_per_task": 12128.533333333333, "last_run_timestamp": "2026-10-02T23:10:27.235Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.64833325, "accuracy_std_dev": 0.021794475600195134, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.022383950000000007, "avg_duration_per_task_ms": 245046.31, "avg_output_tokens_per_task": 9447.4, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.647177495967742, "accuracy_std_dev": 0.07108568891563195, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.025456885080645163, "avg_duration_per_task_ms": 277301.2076612903, "avg_output_tokens_per_task": 25224.120967741936, "last_run_timestamp": "2026-10-17T19:25:29.604Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.6466665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0295618, "avg_duration_per_task_ms": 121397.75, "avg_output_tokens_per_task": 15953.93, "last_run_timestamp": "2026-10-01T19:04:28.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.6455556666666666, "accuracy_std_dev": 0.09964144484545016, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11177372666666668, "avg_duration_per_task_ms": 710817.72, "avg_output_tokens_per_task": 87845.5, "last_run_timestamp": "2026-09-29T19:45:03.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.6451851111111111, "accuracy_std_dev": 0.12616958039827408, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05096927111111111, "avg_duration_per_task_ms": 818258.1133333333, "avg_output_tokens_per_task": 25672.8, "last_run_timestamp": "2026-10-31T19:22:14.979Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.6416665, "accuracy_std_dev": 0.02833350000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0220575, "avg_duration_per_task_ms": 236336.73, "avg_output_tokens_per_task": 9832.45, "last_run_timestamp": "2026-08-15T14:15:19.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.6413691205357143, "accuracy_std_dev": 0.06151143594868467, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.024266383928571424, "avg_duration_per_task_ms": 398610.2589285714, "avg_output_tokens_per_task": 15573.261160714286, "last_run_timestamp": "2026-10-31T19:07:04.114Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.6413336000000001, "accuracy_std_dev": 0.014847377621654282, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.03101342, "avg_duration_per_task_ms": 232783.988, "avg_output_tokens_per_task": 15818.512, "last_run_timestamp": "2026-10-01T19:05:16.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.64, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01684376, "avg_duration_per_task_ms": 1178898.4, "avg_output_tokens_per_task": 43776.78, "last_run_timestamp": "2026-10-09T00:39:20.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.638298, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.020538510638297872, "avg_duration_per_task_ms": 686659.4680851063, "avg_output_tokens_per_task": 47094.31914893617, "last_run_timestamp": "2026-10-09T00:23:16.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.6377776666666667, "accuracy_std_dev": 0.032810732962790655, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.20948366666666668, "avg_duration_per_task_ms": 978205.8133333334, "avg_output_tokens_per_task": 27155.76, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.636978650507328, "accuracy_std_dev": 0.041368248430863674, "run_count": 18, "total_questions": 887, "avg_cost_per_task_usd": 0.020942316798196168, "avg_duration_per_task_ms": 316972.6538895152, "avg_output_tokens_per_task": 16888.583990980835, "last_run_timestamp": "2026-10-31T20:19:17.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.6355553333333334, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008414553333333331, "avg_duration_per_task_ms": 1275373.5733333332, "avg_output_tokens_per_task": 17754.266666666666, "last_run_timestamp": "2026-10-31T21:55:54.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.6333332500000001, "accuracy_std_dev": 0.016996502910525478, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.024200355, "avg_duration_per_task_ms": 140351.54, "avg_output_tokens_per_task": 10016.095, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.6333332500000001, "accuracy_std_dev": 0.03299835012826401, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021382605000000002, "avg_duration_per_task_ms": 229847.125, "avg_output_tokens_per_task": 9805.265, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6331658693467337, "accuracy_std_dev": 0.03188449882289982, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.03419022110552764, "avg_duration_per_task_ms": 256833.6432160804, "avg_output_tokens_per_task": 16666.94974874372, "last_run_timestamp": "2026-10-31T19:04:42.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.6307017947368421, "accuracy_std_dev": 0.04099517109977711, "run_count": 4, "total_questions": 190, "avg_cost_per_task_usd": 0.02501245263157895, "avg_duration_per_task_ms": 66232.84736842105, "avg_output_tokens_per_task": 14146.842105263158, "last_run_timestamp": "2026-10-24T18:40:18.531Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.6283335, "accuracy_std_dev": 0.18833350000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.15963215, "avg_duration_per_task_ms": 1567590.01, "avg_output_tokens_per_task": 36431.88, "last_run_timestamp": "2026-10-17T19:19:25.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.6283332500000001, "accuracy_std_dev": 0.023746444179866183, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.028451149999999998, "avg_duration_per_task_ms": 210873.285, "avg_output_tokens_per_task": 9700.365, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.6273407438202249, "accuracy_std_dev": 0.028511029028200886, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.04094152584269662, "avg_duration_per_task_ms": 84933.8, "avg_output_tokens_per_task": 20903.849438202247, "last_run_timestamp": "2026-10-31T18:59:19.181Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.6266666666666667, "accuracy_std_dev": 0.14676766724392065, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03268331333333334, "avg_duration_per_task_ms": 135271.32666666666, "avg_output_tokens_per_task": 12787.166666666666, "last_run_timestamp": "2026-10-31T11:51:01.680Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.6266666666666667, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12269806, "avg_duration_per_task_ms": 610304.8666666667, "avg_output_tokens_per_task": 27046.16, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.62, "accuracy_std_dev": 0.024343133492264774, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06804065333333334, "avg_duration_per_task_ms": 32072.04888888889, "avg_output_tokens_per_task": 21507.428888888888, "last_run_timestamp": "2026-10-31T19:45:42.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.6161905714285714, "accuracy_std_dev": 0.1331358561415016, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.049797854285714274, "avg_duration_per_task_ms": 468411.09142857144, "avg_output_tokens_per_task": 9093.771428571428, "last_run_timestamp": "2026-10-31T19:32:40.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.6155556666666666, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12338818000000001, "avg_duration_per_task_ms": 626122.66, "avg_output_tokens_per_task": 27741.31333333333, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6076188571428572, "accuracy_std_dev": 0.03274303997331158, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.046068719999999994, "avg_duration_per_task_ms": 191059.44, "avg_output_tokens_per_task": 3405.154285714286, "last_run_timestamp": "2026-10-08T22:20:40.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.606667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03212956, "avg_duration_per_task_ms": 335001.52, "avg_output_tokens_per_task": 8851.18, "last_run_timestamp": "2026-10-01T08:03:01.921Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6066668000000001, "accuracy_std_dev": 0.026666399999999996, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046513716, "avg_duration_per_task_ms": 209556.196, "avg_output_tokens_per_task": 3673.824, "last_run_timestamp": "2026-10-01T16:00:32.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6066666666666667, "accuracy_std_dev": 0.04343290554406878, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09464461999999998, "avg_duration_per_task_ms": 58154.904444444444, "avg_output_tokens_per_task": 3592.282222222222, "last_run_timestamp": "2026-10-31T19:06:25.403Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.6049998333333334, "accuracy_std_dev": 0.01979055610989468, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04771943666666667, "avg_duration_per_task_ms": 93553.64666666667, "avg_output_tokens_per_task": 4645.643333333333, "last_run_timestamp": "2026-10-03T16:14:40.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.6026666, "accuracy_std_dev": 0.023701878031919768, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045193232, "avg_duration_per_task_ms": 105920.528, "avg_output_tokens_per_task": 4778.932, "last_run_timestamp": "2026-10-03T16:07:50.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.6000001666666667, "accuracy_std_dev": 0.03771239112642186, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.19008634000000002, "avg_duration_per_task_ms": 186975.22, "avg_output_tokens_per_task": 11452.75, "last_run_timestamp": "2026-10-01T19:04:55.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.59966491959799, "accuracy_std_dev": 0.08352485507909829, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.028319288944723613, "avg_duration_per_task_ms": 1740003.0175879397, "avg_output_tokens_per_task": 55612.85427135678, "last_run_timestamp": "2026-10-31T19:40:44.417Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.593333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04552234, "avg_duration_per_task_ms": 32685.39, "avg_output_tokens_per_task": 3125.98, "last_run_timestamp": "2026-09-26T08:22:32.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.593333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01896666, "avg_duration_per_task_ms": 306972.98, "avg_output_tokens_per_task": 7436.64, "last_run_timestamp": "2026-10-01T08:02:59.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.591111, "accuracy_std_dev": 0.006285295405415193, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029060479999999996, "avg_duration_per_task_ms": 77755.87, "avg_output_tokens_per_task": 7763.98, "last_run_timestamp": "2026-10-01T19:05:46.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.588889, "accuracy_std_dev": 0.008314764498569197, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.14882857333333332, "avg_duration_per_task_ms": 490464.7866666667, "avg_output_tokens_per_task": 24032.806666666667, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.5873334, "accuracy_std_dev": 0.015691608516656286, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.020115355999999997, "avg_duration_per_task_ms": 100935.536, "avg_output_tokens_per_task": 4366.388, "last_run_timestamp": "2026-10-03T16:14:27.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.583333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015297799999999999, "avg_duration_per_task_ms": 833284.24, "avg_output_tokens_per_task": 38596.74, "last_run_timestamp": "2026-10-01T08:04:13.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.58, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20638665, "avg_duration_per_task_ms": 179102.02, "avg_output_tokens_per_task": 12109.41, "last_run_timestamp": "2026-10-01T19:04:42.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5789978786127168, "accuracy_std_dev": 0.05180400838836742, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.18764681502890174, "avg_duration_per_task_ms": 2044873.4393063583, "avg_output_tokens_per_task": 41085.294797687864, "last_run_timestamp": "2026-10-17T19:42:10.129Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.57483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.017800938775510203, "avg_duration_per_task_ms": 142107.1224489796, "avg_output_tokens_per_task": 3993.1632653061224, "last_run_timestamp": "2026-08-08T16:03:49.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.5733883127572016, "accuracy_std_dev": 0.039571612153158485, "run_count": 5, "total_questions": 243, "avg_cost_per_task_usd": 0.08109196296296296, "avg_duration_per_task_ms": 99593.68724279835, "avg_output_tokens_per_task": 6510.271604938272, "last_run_timestamp": "2026-10-03T16:05:18.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.5666665000000001, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.028862480000000003, "avg_duration_per_task_ms": 75034.33, "avg_output_tokens_per_task": 7701.06, "last_run_timestamp": "2026-10-01T19:04:40.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.5648772953020134, "accuracy_std_dev": 0.0424873867714486, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1486826308724832, "avg_duration_per_task_ms": 18002.88590604027, "avg_output_tokens_per_task": 2546.3288590604025, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.5633334999999999, "accuracy_std_dev": 0.09666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07923399999999998, "avg_duration_per_task_ms": 1068653.19, "avg_output_tokens_per_task": 62040.81, "last_run_timestamp": "2026-08-15T14:08:35.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.560133587973274, "accuracy_std_dev": 0.2552451920126611, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.015204305122494433, "avg_duration_per_task_ms": 265953.0779510022, "avg_output_tokens_per_task": 7739.80846325167, "last_run_timestamp": "2026-10-17T11:22:12.284Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.557500125, "accuracy_std_dev": 0.07859452244501124, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.032340572500000005, "avg_duration_per_task_ms": 393952.61, "avg_output_tokens_per_task": 16997.9425, "last_run_timestamp": "2026-10-24T19:54:36.551Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.55, "accuracy_std_dev": 0.036667000000000005, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.2373302, "avg_duration_per_task_ms": 164429.2, "avg_output_tokens_per_task": 12508.34, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.546667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03543453, "avg_duration_per_task_ms": 61133.82, "avg_output_tokens_per_task": 6656.99, "last_run_timestamp": "2026-10-01T19:04:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.5409356315789473, "accuracy_std_dev": 0.048637113976719215, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.00998410275689223, "avg_duration_per_task_ms": 184598.81453634085, "avg_output_tokens_per_task": 15438.807017543859, "last_run_timestamp": "2026-10-31T19:08:29.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5370371414141414, "accuracy_std_dev": 0.020577999999999985, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.02047550505050505, "avg_duration_per_task_ms": 144559.1616161616, "avg_output_tokens_per_task": 38541.121212121216, "last_run_timestamp": "2026-10-01T19:05:00.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.534014, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.01161230612244898, "avg_duration_per_task_ms": 94666.5306122449, "avg_output_tokens_per_task": 16582.326530612245, "last_run_timestamp": "2026-10-01T19:04:42.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.5333335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10219375, "avg_duration_per_task_ms": 375443.12, "avg_output_tokens_per_task": 17495.06, "last_run_timestamp": "2026-08-15T14:16:11.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.5330330315315315, "accuracy_std_dev": 0.017762103976002412, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.016922781531531532, "avg_duration_per_task_ms": 1163584.25, "avg_output_tokens_per_task": 75958.21846846846, "last_run_timestamp": "2026-10-31T19:53:39.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.5296297777777778, "accuracy_std_dev": 0.01914500153377888, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06659953333333334, "avg_duration_per_task_ms": 283737.4666666667, "avg_output_tokens_per_task": 4050.251111111111, "last_run_timestamp": "2026-10-31T19:28:47.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.5285472267002518, "accuracy_std_dev": 0.03054949850959095, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.0026572493702770776, "avg_duration_per_task_ms": 371704.879093199, "avg_output_tokens_per_task": 6936.692695214106, "last_run_timestamp": "2026-10-31T19:13:20.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.522831205479452, "accuracy_std_dev": 0.027543964303555765, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.11947870547945207, "avg_duration_per_task_ms": 69921.19178082192, "avg_output_tokens_per_task": 2362.404109589041, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.519759618556701, "accuracy_std_dev": 0.11063848893355334, "run_count": 4, "total_questions": 194, "avg_cost_per_task_usd": 0.05320747422680413, "avg_duration_per_task_ms": 383087.36597938143, "avg_output_tokens_per_task": 23944.39175257732, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.517500125, "accuracy_std_dev": 0.024930426803594342, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0282700725, "avg_duration_per_task_ms": 444815, "avg_output_tokens_per_task": 58343.125, "last_run_timestamp": "2026-10-03T16:09:19.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.5173608125000001, "accuracy_std_dev": 0.025072499999999998, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.06539028125, "avg_duration_per_task_ms": 2881049.1770833335, "avg_output_tokens_per_task": 39867.583333333336, "last_run_timestamp": "2026-08-15T14:12:30.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.513889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.025741249999999997, "avg_duration_per_task_ms": 587337.8541666666, "avg_output_tokens_per_task": 10115.354166666666, "last_run_timestamp": "2026-08-08T16:03:43.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.5133333333333333, "accuracy_std_dev": 0.028803034820811636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12443134666666665, "avg_duration_per_task_ms": 52357.68, "avg_output_tokens_per_task": 2755.9866666666667, "last_run_timestamp": "2026-10-02T23:07:04.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.513333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03238348, "avg_duration_per_task_ms": 271942.88, "avg_output_tokens_per_task": 7084.96, "last_run_timestamp": "2026-10-01T08:05:50.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.4995211379310344, "accuracy_std_dev": 0.03521937973703148, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.16297116666666667, "avg_duration_per_task_ms": 2016445.1580459771, "avg_output_tokens_per_task": 38482.971264367814, "last_run_timestamp": "2026-10-17T19:35:49.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.4990475714285714, "accuracy_std_dev": 0.026047140242574162, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.025615628571428573, "avg_duration_per_task_ms": 596502.9742857143, "avg_output_tokens_per_task": 52103.73714285714, "last_run_timestamp": "2026-09-26T16:00:25.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.4972901544715447, "accuracy_std_dev": 0.018392858501059588, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.021434772357723578, "avg_duration_per_task_ms": 1275972.138211382, "avg_output_tokens_per_task": 109100.26829268293, "last_run_timestamp": "2026-10-31T19:48:14.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.493333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14235776, "avg_duration_per_task_ms": 443941.74, "avg_output_tokens_per_task": 8833.66, "last_run_timestamp": "2026-08-06T00:00:02.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.48850570114942526, "accuracy_std_dev": 0.017439968302957548, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.031982068965517245, "avg_duration_per_task_ms": 642915.867816092, "avg_output_tokens_per_task": 59401.63505747126, "last_run_timestamp": "2026-10-23T09:26:25.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.4857621557788945, "accuracy_std_dev": 0.02560762584874622, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.028534012562814074, "avg_duration_per_task_ms": 352717.891959799, "avg_output_tokens_per_task": 60736.11306532663, "last_run_timestamp": "2026-10-03T16:14:03.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.48333349999999997, "accuracy_std_dev": 0.03666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04059527, "avg_duration_per_task_ms": 669013.45, "avg_output_tokens_per_task": 93900.79, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.48000024999999996, "accuracy_std_dev": 0.01414201777284628, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0673682, "avg_duration_per_task_ms": 287537.155, "avg_output_tokens_per_task": 4110.14, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011001320000000002, "avg_duration_per_task_ms": 80205.64, "avg_output_tokens_per_task": 3762.88, "last_run_timestamp": "2026-10-08T22:08:03.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.48, "accuracy_std_dev": 0.0312693389057076, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.050673248000000004, "avg_duration_per_task_ms": 91008.444, "avg_output_tokens_per_task": 2749.592, "last_run_timestamp": "2026-10-03T16:07:50.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07899766, "avg_duration_per_task_ms": 132704.5, "avg_output_tokens_per_task": 3161.38, "last_run_timestamp": "2026-10-04T03:00:16.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.47916700000000007, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.04362447916666667, "avg_duration_per_task_ms": 66953.94791666667, "avg_output_tokens_per_task": 2828.96875, "last_run_timestamp": "2026-08-15T14:08:14.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.47874697986577186, "accuracy_std_dev": 0.00975008705374242, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.16965920805369125, "avg_duration_per_task_ms": 2424316.9261744968, "avg_output_tokens_per_task": 45379.409395973154, "last_run_timestamp": "2026-10-17T19:38:30.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.47666650000000005, "accuracy_std_dev": 0.0166665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12559784000000002, "avg_duration_per_task_ms": 52928.96, "avg_output_tokens_per_task": 2668.55, "last_run_timestamp": "2026-10-01T19:04:26.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.4754382631578947, "accuracy_std_dev": 0.054999999999999966, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.01757382105263158, "avg_duration_per_task_ms": 167109.45263157896, "avg_output_tokens_per_task": 28512.663157894738, "last_run_timestamp": "2026-10-01T19:06:10.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.475, "accuracy_std_dev": 0.00986024566123988, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021282230000000003, "avg_duration_per_task_ms": 677068.455, "avg_output_tokens_per_task": 19681.38, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.47333349999999996, "accuracy_std_dev": 0.008164965814380372, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.024006935, "avg_duration_per_task_ms": 869541.065, "avg_output_tokens_per_task": 22816.51, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.47222181249999995, "accuracy_std_dev": 0.0011595000000000077, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.07886530208333332, "avg_duration_per_task_ms": 55709.895833333336, "avg_output_tokens_per_task": 3493.7291666666665, "last_run_timestamp": "2026-08-15T14:15:58.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.472, "accuracy_std_dev": 0.023999999999999994, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.034465088000000005, "avg_duration_per_task_ms": 43484.756, "avg_output_tokens_per_task": 3636.224, "last_run_timestamp": "2026-10-03T16:17:11.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.4715242060301507, "accuracy_std_dev": 0.16796757633971357, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.06913073618090453, "avg_duration_per_task_ms": 563511.0577889447, "avg_output_tokens_per_task": 9865.51256281407, "last_run_timestamp": "2026-10-31T19:59:00.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4706666, "accuracy_std_dev": 0.020483090046182005, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023913492, "avg_duration_per_task_ms": 139319.672, "avg_output_tokens_per_task": 3726.148, "last_run_timestamp": "2026-10-03T16:13:56.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.47, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03551402, "avg_duration_per_task_ms": 437816.58, "avg_output_tokens_per_task": 20922.04, "last_run_timestamp": "2026-10-08T23:09:09.727Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.46820784971098267, "accuracy_std_dev": 0.04097451831844482, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.027368598265895955, "avg_duration_per_task_ms": 2298741.832369942, "avg_output_tokens_per_task": 136175.1647398844, "last_run_timestamp": "2026-10-24T19:05:04.624Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.4680002, "accuracy_std_dev": 0.03798255037987838, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.019017924000000002, "avg_duration_per_task_ms": 96994.328, "avg_output_tokens_per_task": 3628.408, "last_run_timestamp": "2026-10-03T16:12:20.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.46555555555555556, "accuracy_std_dev": 0.1375265612956365, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.027833837777777776, "avg_duration_per_task_ms": 236061.30666666667, "avg_output_tokens_per_task": 18727.45111111111, "last_run_timestamp": "2026-10-31T19:46:58.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4644446666666666, "accuracy_std_dev": 0.029979285029648195, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04325750666666667, "avg_duration_per_task_ms": 738509.7133333334, "avg_output_tokens_per_task": 100007.46, "last_run_timestamp": "2026-10-02T23:22:06.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.4640002, "accuracy_std_dev": 0.04144079793343752, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.024943207999999998, "avg_duration_per_task_ms": 59536.72, "avg_output_tokens_per_task": 3736.6, "last_run_timestamp": "2026-10-03T16:10:49.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.46166674999999996, "accuracy_std_dev": 0.04010416645047619, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04448468, "avg_duration_per_task_ms": 738816.635, "avg_output_tokens_per_task": 95390.315, "last_run_timestamp": "2026-10-01T19:06:01.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011152899999999999, "avg_duration_per_task_ms": 87357.46, "avg_output_tokens_per_task": 9087.9, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 158124.84, "avg_output_tokens_per_task": 4515, "last_run_timestamp": "2026-10-01T19:06:51.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.46, "accuracy_std_dev": 0.006667000000000006, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03575541, "avg_duration_per_task_ms": 402821.3, "avg_output_tokens_per_task": 3754.72, "last_run_timestamp": "2026-08-15T08:15:42.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4566668333333333, "accuracy_std_dev": 0.03590122251129556, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.021435916666666662, "avg_duration_per_task_ms": 222009.41666666666, "avg_output_tokens_per_task": 3511.05, "last_run_timestamp": "2026-10-08T22:47:27.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.4541387986577181, "accuracy_std_dev": 0.02966298114934956, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.023203442953020136, "avg_duration_per_task_ms": 2013894.1275167784, "avg_output_tokens_per_task": 14181.58389261745, "last_run_timestamp": "2026-10-30T21:07:22.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.45333349999999994, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0063000800000000004, "avg_duration_per_task_ms": 38540.88, "avg_output_tokens_per_task": 1215.72, "last_run_timestamp": "2026-08-15T08:16:13.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.4524997500000001, "accuracy_std_dev": 0.02520749723172652, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02146402, "avg_duration_per_task_ms": 650786.745, "avg_output_tokens_per_task": 19908.01, "last_run_timestamp": "2026-10-09T00:21:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.45121959349593493, "accuracy_std_dev": 0.052473835908574476, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.05198310569105691, "avg_duration_per_task_ms": 74014.12195121951, "avg_output_tokens_per_task": 3228.670731707317, "last_run_timestamp": "2026-10-03T16:07:05.221Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.45111100000000004, "accuracy_std_dev": 0.01571332689152746, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023389446666666668, "avg_duration_per_task_ms": 54689.44666666666, "avg_output_tokens_per_task": 2398.8933333333334, "last_run_timestamp": "2026-10-02T23:07:02.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.45, "accuracy_std_dev": 0.041518865477893636, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05302523142857142, "avg_duration_per_task_ms": 144621.62857142856, "avg_output_tokens_per_task": 3704.842857142857, "last_run_timestamp": "2026-10-17T19:11:29.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.45, "accuracy_std_dev": 0.0033330000000000026, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08659784, "avg_duration_per_task_ms": 80537.63, "avg_output_tokens_per_task": 4189.08, "last_run_timestamp": "2026-08-15T08:11:06.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.4497487512562814, "accuracy_std_dev": 0.043480269304018805, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.015482552763819097, "avg_duration_per_task_ms": 129064.97989949748, "avg_output_tokens_per_task": 3318.743718592965, "last_run_timestamp": "2026-10-31T18:58:15.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.4490197823529411, "accuracy_std_dev": 0.021932270978759, "run_count": 7, "total_questions": 340, "avg_cost_per_task_usd": 0.05236282352941176, "avg_duration_per_task_ms": 148084.36176470589, "avg_output_tokens_per_task": 72432.0411764706, "last_run_timestamp": "2026-10-31T19:03:34.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.4466668, "accuracy_std_dev": 0.08033274135344816, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.20203069199999998, "avg_duration_per_task_ms": 115064.372, "avg_output_tokens_per_task": 3550.864, "last_run_timestamp": "2026-10-03T16:19:14.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.44458325000000004, "accuracy_std_dev": 0.02852565974587617, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.039974685, "avg_duration_per_task_ms": 271844.9625, "avg_output_tokens_per_task": 3550.7, "last_run_timestamp": "2026-10-31T19:56:10.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.44333300000000003, "accuracy_std_dev": 0.009999999999999981, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.023192820000000003, "avg_duration_per_task_ms": 58714.73, "avg_output_tokens_per_task": 2330.08, "last_run_timestamp": "2026-10-01T19:04:38.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.4416665, "accuracy_std_dev": 0.017873086533948188, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.104832345, "avg_duration_per_task_ms": 890298.59, "avg_output_tokens_per_task": 21334.155, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.44, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03130282, "avg_duration_per_task_ms": 680376.23, "avg_output_tokens_per_task": 16688.33, "last_run_timestamp": "2026-10-01T16:00:49.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.43999974999999997, "accuracy_std_dev": 0.047081383440671115, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050863820000000004, "avg_duration_per_task_ms": 37018.795, "avg_output_tokens_per_task": 2487.295, "last_run_timestamp": "2026-10-03T16:12:29.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4377778333333333, "accuracy_std_dev": 0.040490818956963014, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.053731056666666666, "avg_duration_per_task_ms": 129281.94, "avg_output_tokens_per_task": 3456.7733333333335, "last_run_timestamp": "2026-10-03T16:19:54.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.43631422764227645, "accuracy_std_dev": 0.03840736243170051, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.057368723577235776, "avg_duration_per_task_ms": 251181.19512195123, "avg_output_tokens_per_task": 2298.849593495935, "last_run_timestamp": "2026-10-05T20:09:54.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.4353741734693877, "accuracy_std_dev": 0.03760835138885378, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.04635305102040816, "avg_duration_per_task_ms": 63193.6581632653, "avg_output_tokens_per_task": 2275.091836734694, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.435, "accuracy_std_dev": 0.020749805890658335, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.074401825, "avg_duration_per_task_ms": 778012.545, "avg_output_tokens_per_task": 20313.29, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.4322221666666667, "accuracy_std_dev": 0.01651776073823432, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.21270556333333332, "avg_duration_per_task_ms": 46305.06, "avg_output_tokens_per_task": 2266.36, "last_run_timestamp": "2026-10-02T23:10:05.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.431111, "accuracy_std_dev": 0.00314238253559302, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.24174756666666666, "avg_duration_per_task_ms": 46576.24, "avg_output_tokens_per_task": 2473.9733333333334, "last_run_timestamp": "2026-10-02T23:09:02.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.4301802162162162, "accuracy_std_dev": 0.02412263108922131, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.06743897297297298, "avg_duration_per_task_ms": 360892.5135135135, "avg_output_tokens_per_task": 3318.9054054054054, "last_run_timestamp": "2026-10-08T22:54:33.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.42916675, "accuracy_std_dev": 0.025967332577442378, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.027418319999999996, "avg_duration_per_task_ms": 84447.825, "avg_output_tokens_per_task": 2784.11, "last_run_timestamp": "2026-10-03T16:12:20.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.4283335, "accuracy_std_dev": 0.02598098663927142, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.016813855, "avg_duration_per_task_ms": 118682.965, "avg_output_tokens_per_task": 2808.965, "last_run_timestamp": "2026-10-03T16:14:29.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.428333375, "accuracy_std_dev": 0.02990715029862215, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.007658255, "avg_duration_per_task_ms": 180128.6975, "avg_output_tokens_per_task": 4519.6725, "last_run_timestamp": "2026-10-31T19:46:05.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.428, "accuracy_std_dev": 0.052391746594287164, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.49939273599999995, "avg_duration_per_task_ms": 111261.628, "avg_output_tokens_per_task": 3387.876, "last_run_timestamp": "2026-10-03T16:14:53.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.42799980000000004, "accuracy_std_dev": 0.012220332784339386, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.039161852, "avg_duration_per_task_ms": 146989.988, "avg_output_tokens_per_task": 3492.532, "last_run_timestamp": "2026-10-03T16:18:20.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.4273334, "accuracy_std_dev": 0.03115546180110319, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.024095914, "avg_duration_per_task_ms": 401744.36, "avg_output_tokens_per_task": 3568.138, "last_run_timestamp": "2026-10-31T19:22:37.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.42666699999999996, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02972828, "avg_duration_per_task_ms": 273042.46, "avg_output_tokens_per_task": 3546.89, "last_run_timestamp": "2026-08-15T08:02:48.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.42666675, "accuracy_std_dev": 0.014142017772846282, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050337635, "avg_duration_per_task_ms": 158009.49, "avg_output_tokens_per_task": 2226.425, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.42666650000000006, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.21223042999999997, "avg_duration_per_task_ms": 51685.1, "avg_output_tokens_per_task": 2189.48, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.4256754864864865, "accuracy_std_dev": 0.024259060786069653, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.03375386486486487, "avg_duration_per_task_ms": 525936.777027027, "avg_output_tokens_per_task": 5145.756756756757, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.4255553333333334, "accuracy_std_dev": 0.034462310778910675, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029985173333333334, "avg_duration_per_task_ms": 822675.42, "avg_output_tokens_per_task": 18074.12, "last_run_timestamp": "2026-10-08T22:41:10.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.42400019999999994, "accuracy_std_dev": 0.014514259655938366, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.024904308, "avg_duration_per_task_ms": 223225.116, "avg_output_tokens_per_task": 3537.724, "last_run_timestamp": "2026-10-03T16:13:22.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.4236811784511785, "accuracy_std_dev": 0.029725846878880795, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.05290695286195286, "avg_duration_per_task_ms": 528585.1717171717, "avg_output_tokens_per_task": 22153.03367003367, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.42154647297297293, "accuracy_std_dev": 0.02867673607820049, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.0049315878378378385, "avg_duration_per_task_ms": 343548.7364864865, "avg_output_tokens_per_task": 4828.531531531532, "last_run_timestamp": "2026-10-31T19:55:29.787Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41888899999999996, "accuracy_std_dev": 0.014614444065147779, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01463076, "avg_duration_per_task_ms": 76549.69333333333, "avg_output_tokens_per_task": 8814.213333333333, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.4175171377551021, "accuracy_std_dev": 0.02667947172423725, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.057072859693877556, "avg_duration_per_task_ms": 211166.0357142857, "avg_output_tokens_per_task": 4534.619897959184, "last_run_timestamp": "2026-10-03T08:12:06.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.416667, "accuracy_std_dev": 0.03, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.19647352999999998, "avg_duration_per_task_ms": 53166.02, "avg_output_tokens_per_task": 2086.91, "last_run_timestamp": "2026-10-01T16:01:13.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.411667, "accuracy_std_dev": 0.049749371855331, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.055448005, "avg_duration_per_task_ms": 142634.75, "avg_output_tokens_per_task": 3380.275, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.40836190721649485, "accuracy_std_dev": 0.024997748289707123, "run_count": 6, "total_questions": 291, "avg_cost_per_task_usd": 0.011245838487972509, "avg_duration_per_task_ms": 117422.78006872852, "avg_output_tokens_per_task": 3161.364261168385, "last_run_timestamp": "2026-10-03T16:17:21.424Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.406667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06754938, "avg_duration_per_task_ms": 114529.78, "avg_output_tokens_per_task": 3789.7, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.4066664000000001, "accuracy_std_dev": 0.025647109432448717, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07750359200000001, "avg_duration_per_task_ms": 118177.072, "avg_output_tokens_per_task": 3117.652, "last_run_timestamp": "2026-10-03T16:12:07.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.40533320000000006, "accuracy_std_dev": 0.01808612973966516, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.06744982, "avg_duration_per_task_ms": 383181.876, "avg_output_tokens_per_task": 10145.868, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.40389273722627733, "accuracy_std_dev": 0.01580206672840261, "run_count": 3, "total_questions": 137, "avg_cost_per_task_usd": 0.008183423357664232, "avg_duration_per_task_ms": 89294.24087591241, "avg_output_tokens_per_task": 1500.5474452554745, "last_run_timestamp": "2026-10-03T16:15:51.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.40380291275167784, "accuracy_std_dev": 0.008840018715415086, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.07104265771812081, "avg_duration_per_task_ms": 235544.13422818793, "avg_output_tokens_per_task": 8668.979865771813, "last_run_timestamp": "2026-10-02T15:44:48.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.40333325000000003, "accuracy_std_dev": 0.01914841159698371, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.023807599999999998, "avg_duration_per_task_ms": 77145.375, "avg_output_tokens_per_task": 2775.135, "last_run_timestamp": "2026-10-01T19:05:00.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.4026668, "accuracy_std_dev": 0.027519660851107883, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.027686715999999997, "avg_duration_per_task_ms": 276072.088, "avg_output_tokens_per_task": 4225.576, "last_run_timestamp": "2026-10-03T16:13:56.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.401667, "accuracy_std_dev": 0.025000000000000022, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01796061, "avg_duration_per_task_ms": 1329879.6, "avg_output_tokens_per_task": 125682.59, "last_run_timestamp": "2026-10-31T20:05:39.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.4, "accuracy_std_dev": 0.034769053930183376, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.076876536, "avg_duration_per_task_ms": 340404.576, "avg_output_tokens_per_task": 15764.94, "last_run_timestamp": "2026-10-03T16:12:29.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.39999980000000007, "accuracy_std_dev": 0.008432845839928543, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.036944032, "avg_duration_per_task_ms": 191861.204, "avg_output_tokens_per_task": 3680.784, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.39833349999999995, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06903712000000001, "avg_duration_per_task_ms": 315593.59, "avg_output_tokens_per_task": 7308.28, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.3948148888888889, "accuracy_std_dev": 0.12040172902084868, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021518880000000004, "avg_duration_per_task_ms": 219251.11555555556, "avg_output_tokens_per_task": 3342.9933333333333, "last_run_timestamp": "2026-10-31T19:18:14.606Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.39466680000000004, "accuracy_std_dev": 0.019505181079907976, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.0621838, "avg_duration_per_task_ms": 95587.668, "avg_output_tokens_per_task": 2912.132, "last_run_timestamp": "2026-10-03T16:12:33.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.39166674999999995, "accuracy_std_dev": 0.00866020592639114, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015426675, "avg_duration_per_task_ms": 99199.855, "avg_output_tokens_per_task": 615.135, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.39132881081081083, "accuracy_std_dev": 0.025676864524422683, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.058435516891891894, "avg_duration_per_task_ms": 376057.5371621622, "avg_output_tokens_per_task": 3843.489864864865, "last_run_timestamp": "2026-10-08T23:21:09.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.3911292096774194, "accuracy_std_dev": 0.3156298317748815, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.005442725806451613, "avg_duration_per_task_ms": 343542.48387096776, "avg_output_tokens_per_task": 10219.729838709678, "last_run_timestamp": "2026-10-24T19:32:27.020Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.014510460000000001, "avg_duration_per_task_ms": 64332.54, "avg_output_tokens_per_task": 4106.96, "last_run_timestamp": "2026-10-09T15:37:38.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0592903, "avg_duration_per_task_ms": 203536.16, "avg_output_tokens_per_task": 5170.96, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.3866665, "accuracy_std_dev": 0.006666499999999992, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020927889999999998, "avg_duration_per_task_ms": 183788.07, "avg_output_tokens_per_task": 3842.25, "last_run_timestamp": "2026-08-15T08:02:44.458Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.38333325, "accuracy_std_dev": 0.007453597197830037, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.028690270000000004, "avg_duration_per_task_ms": 83258.84, "avg_output_tokens_per_task": 4293.12, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.3826666, "accuracy_std_dev": 0.01083190350030869, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.026093336, "avg_duration_per_task_ms": 200624.988, "avg_output_tokens_per_task": 4302.312, "last_run_timestamp": "2026-10-03T16:03:16.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.37745103619909504, "accuracy_std_dev": 0.12198880856754195, "run_count": 9, "total_questions": 442, "avg_cost_per_task_usd": 0.05260786199095024, "avg_duration_per_task_ms": 782967.8257918552, "avg_output_tokens_per_task": 9060.183257918552, "last_run_timestamp": "2026-10-31T19:55:08.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.3773334, "accuracy_std_dev": 0.04552913987590804, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.05450766399999999, "avg_duration_per_task_ms": 92940.624, "avg_output_tokens_per_task": 3847.676, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.37599980000000005, "accuracy_std_dev": 0.032276543943861154, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.055068684, "avg_duration_per_task_ms": 70747.116, "avg_output_tokens_per_task": 4411.06, "last_run_timestamp": "2026-10-03T16:05:34.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.37416568080094226, "accuracy_std_dev": 0.022198267854897723, "run_count": 17, "total_questions": 849, "avg_cost_per_task_usd": 0.013129518256772676, "avg_duration_per_task_ms": 40276.608951707894, "avg_output_tokens_per_task": 755.5041224970554, "last_run_timestamp": "2026-10-31T19:10:53.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.3688886666666667, "accuracy_std_dev": 0.01791611454963257, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03368235333333333, "avg_duration_per_task_ms": 150468.61333333334, "avg_output_tokens_per_task": 4460.053333333333, "last_run_timestamp": "2026-10-23T08:14:36.871Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.36833325, "accuracy_std_dev": 0.17853565236861657, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015193219999999999, "avg_duration_per_task_ms": 127682.09, "avg_output_tokens_per_task": 2917.505, "last_run_timestamp": "2026-10-03T16:12:29.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.364498756097561, "accuracy_std_dev": 0.08717599304625098, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.42022371138211384, "avg_duration_per_task_ms": 436503.325203252, "avg_output_tokens_per_task": 17041.613821138213, "last_run_timestamp": "2026-10-10T18:31:45.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.36, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.041000339999999996, "avg_duration_per_task_ms": 322345.3, "avg_output_tokens_per_task": 4212.74, "last_run_timestamp": "2026-10-04T03:00:10.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.35714285714285715, "accuracy_std_dev": 0.029979379745928993, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.07267795758928572, "avg_duration_per_task_ms": 400336.21875, "avg_output_tokens_per_task": 8020.354910714285, "last_run_timestamp": "2026-10-08T23:34:52.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.3542132751677853, "accuracy_std_dev": 0.13859537037210243, "run_count": 18, "total_questions": 894, "avg_cost_per_task_usd": 0.036926778523489925, "avg_duration_per_task_ms": 548291.3724832215, "avg_output_tokens_per_task": 133944.0715883669, "last_run_timestamp": "2026-10-31T20:07:06.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.3518517777777778, "accuracy_std_dev": 0.03277707804337855, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09174267777777778, "avg_duration_per_task_ms": 26796.88, "avg_output_tokens_per_task": 1065.5333333333333, "last_run_timestamp": "2026-10-31T18:48:48.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.35175891959798994, "accuracy_std_dev": 0.022258587996757, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.0788611256281407, "avg_duration_per_task_ms": 211842.61055276383, "avg_output_tokens_per_task": 7349.442211055276, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.333333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.051354699999999996, "avg_duration_per_task_ms": 358714.54, "avg_output_tokens_per_task": 69285.14, "last_run_timestamp": "2026-10-04T03:00:10.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.3281016860986547, "accuracy_std_dev": 0.044158645918249494, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.047258513452914805, "avg_duration_per_task_ms": 86946.02466367713, "avg_output_tokens_per_task": 1321.1367713004483, "last_run_timestamp": "2026-10-31T19:07:35.559Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.326667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02872976, "avg_duration_per_task_ms": 133695.86, "avg_output_tokens_per_task": 3408.56, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.32416675, "accuracy_std_dev": 0.025806488741932716, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07527415750000001, "avg_duration_per_task_ms": 136836.5875, "avg_output_tokens_per_task": 7681.4025, "last_run_timestamp": "2026-10-03T08:23:09.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.313333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15186704, "avg_duration_per_task_ms": 32635.52, "avg_output_tokens_per_task": 1294.54, "last_run_timestamp": "2026-10-04T03:00:16.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.30927829896907216, "accuracy_std_dev": 0.041318422501349206, "run_count": 4, "total_questions": 194, "avg_cost_per_task_usd": 0.016999237113402062, "avg_duration_per_task_ms": 169063.57731958764, "avg_output_tokens_per_task": 1549.3092783505156, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.30666649999999995, "accuracy_std_dev": 0.006666499999999992, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01779966, "avg_duration_per_task_ms": 55498.41, "avg_output_tokens_per_task": 3406.81, "last_run_timestamp": "2026-10-01T19:04:58.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.2847618571428572, "accuracy_std_dev": 0.02910620321918911, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1710796942857143, "avg_duration_per_task_ms": 139494.6, "avg_output_tokens_per_task": 1877.2057142857143, "last_run_timestamp": "2026-10-17T19:19:08.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.2844446666666667, "accuracy_std_dev": 0.02997950740459592, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.025635820000000004, "avg_duration_per_task_ms": 67806.75333333333, "avg_output_tokens_per_task": 4107.166666666667, "last_run_timestamp": "2026-10-02T23:13:47.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.27745670520231214, "accuracy_std_dev": 0.31828543520589203, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.009253416184971098, "avg_duration_per_task_ms": 52282.53468208093, "avg_output_tokens_per_task": 9707.71098265896, "last_run_timestamp": "2026-10-31T19:36:09.009Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.2733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07945186, "avg_duration_per_task_ms": 632916.72, "avg_output_tokens_per_task": 5086.04, "last_run_timestamp": "2026-10-01T16:00:45.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.24333325000000003, "accuracy_std_dev": 0.08749614912204708, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17476605, "avg_duration_per_task_ms": 545108.995, "avg_output_tokens_per_task": 11742.65, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.24133339999999998, "accuracy_std_dev": 0.06591572936591082, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.013381287999999998, "avg_duration_per_task_ms": 21869.236, "avg_output_tokens_per_task": 1091.82, "last_run_timestamp": "2026-10-03T16:13:42.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.24, "accuracy_std_dev": 0.033333, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07327475, "avg_duration_per_task_ms": 662655.63, "avg_output_tokens_per_task": 5322.46, "last_run_timestamp": "2026-10-01T16:00:49.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.23180078160919543, "accuracy_std_dev": 0.19038523699508184, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.19703842528735632, "avg_duration_per_task_ms": 1194250.8362068965, "avg_output_tokens_per_task": 7552.1781609195405, "last_run_timestamp": "2026-10-03T08:09:23.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.22244883673469384, "accuracy_std_dev": 0.060049660254159636, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.004490742857142857, "avg_duration_per_task_ms": 16706.061224489797, "avg_output_tokens_per_task": 1551.4979591836734, "last_run_timestamp": "2026-10-24T19:30:38.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.2116665, "accuracy_std_dev": 0.12643621800042107, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.018232810000000002, "avg_duration_per_task_ms": 80959.52, "avg_output_tokens_per_task": 1422.615, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.20799989999999996, "accuracy_std_dev": 0.23746439227827398, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.06683555599999999, "avg_duration_per_task_ms": 145217.8, "avg_output_tokens_per_task": 16818.696, "last_run_timestamp": "2026-10-31T19:46:05.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.19360285858585857, "accuracy_std_dev": 0.23547614599731753, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.008070734848484848, "avg_duration_per_task_ms": 691070.9520202021, "avg_output_tokens_per_task": 53901.46464646464, "last_run_timestamp": "2026-10-31T19:13:59.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.186667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01693508, "avg_duration_per_task_ms": 26941.4, "avg_output_tokens_per_task": 1041.58, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.17942213775510205, "accuracy_std_dev": 0.009739601557558703, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.010118459183673469, "avg_duration_per_task_ms": 155344.85714285713, "avg_output_tokens_per_task": 4996.790816326531, "last_run_timestamp": "2026-10-01T19:04:52.650Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.17133340000000002, "accuracy_std_dev": 0.261806292768986, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.006590812, "avg_duration_per_task_ms": 17587.43, "avg_output_tokens_per_task": 3014.246, "last_run_timestamp": "2026-10-17T19:48:24.252Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.14, "accuracy_std_dev": 0.014142135623730954, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0072234, "avg_duration_per_task_ms": 76455.195, "avg_output_tokens_per_task": 2322.375, "last_run_timestamp": "2026-10-01T19:04:58.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.111111, "accuracy_std_dev": 0.01571332689152746, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.007342933333333333, "avg_duration_per_task_ms": 57269.24, "avg_output_tokens_per_task": 2382.0866666666666, "last_run_timestamp": "2026-10-02T23:07:59.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.11, "accuracy_std_dev": 0.009999999999999995, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.006762440000000001, "avg_duration_per_task_ms": 64306.02, "avg_output_tokens_per_task": 2312.51, "last_run_timestamp": "2026-10-01T19:04:36.194Z" } ] }