{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-10-31", "retrieved_at": "2026-10-31T01:45:40.293Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's non-zero-score runs from the newest 90 days: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2230, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.964646, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.09468125757575757, "avg_duration_per_task_ms": 101980.28282828283, "avg_output_tokens_per_task": 8439.414141414141, "last_run_timestamp": "2026-10-14T15:00:03.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-4-argon", "model_slug": "google/gemini-4-argon", "model_name": "Gemini 4 Argon", "provider_name": "auto-routing", "accuracy": 0.959596, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.029260161616161615, "avg_duration_per_task_ms": 40252.82323232323, "avg_output_tokens_per_task": 2870.0252525252527, "last_run_timestamp": "2026-10-23T07:38:37.725Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.95637, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.06822796858638744, "avg_duration_per_task_ms": 132197.8062827225, "avg_output_tokens_per_task": 18049.068062827224, "last_run_timestamp": "2026-10-01T08:31:33.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.954545, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.332555398989899, "avg_duration_per_task_ms": 75446.5505050505, "avg_output_tokens_per_task": 4530.388888888889, "last_run_timestamp": "2026-10-01T08:27:35.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07423223737373738, "avg_duration_per_task_ms": 86748.28787878787, "avg_output_tokens_per_task": 19646.409090909092, "last_run_timestamp": "2026-10-01T08:29:11.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.95202, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20364760606060606, "avg_duration_per_task_ms": 135706.18686868687, "avg_output_tokens_per_task": 16847.588383838385, "last_run_timestamp": "2026-10-01T19:14:26.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.9511785, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.33620659090909094, "avg_duration_per_task_ms": 70544.52272727272, "avg_output_tokens_per_task": 4588.830808080808, "last_run_timestamp": "2026-10-02T23:23:01.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.9511779999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20247136363636364, "avg_duration_per_task_ms": 137578.30555555556, "avg_output_tokens_per_task": 16748.56818181818, "last_run_timestamp": "2026-10-01T19:12:42.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.949495, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.35495546717171716, "avg_duration_per_task_ms": 72102.17676767676, "avg_output_tokens_per_task": 4622.60101010101, "last_run_timestamp": "2026-10-01T08:29:03.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.9482614885496183, "accuracy_std_dev": 0.0012429999999999941, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.07536056743002545, "avg_duration_per_task_ms": 132814.94147582698, "avg_output_tokens_per_task": 18975.51145038168, "last_run_timestamp": "2026-10-15T21:00:04.244Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.947917, "accuracy_std_dev": null, "run_count": 1, "total_questions": 96, "avg_cost_per_task_usd": 0.009331666666666667, "avg_duration_per_task_ms": 232878.71875, "avg_output_tokens_per_task": 10278.75, "last_run_timestamp": "2026-10-14T18:00:04.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.947811, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.021484267676767678, "avg_duration_per_task_ms": 55623.856060606064, "avg_output_tokens_per_task": 1890.328282828283, "last_run_timestamp": "2026-10-30T00:03:33.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9472503333333333, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11162585016835018, "avg_duration_per_task_ms": 48234.978114478115, "avg_output_tokens_per_task": 2076.191919191919, "last_run_timestamp": "2026-10-02T23:16:05.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra", "model_slug": "sakana/fugu-ultra", "model_name": "Fugu Ultra", "provider_name": "auto-routing", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 1.148739303030303, "avg_duration_per_task_ms": 328779.10101010103, "avg_output_tokens_per_task": 33271.606060606064, "last_run_timestamp": "2026-09-15T01:07:47.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.9461276666666666, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1181814292929293, "avg_duration_per_task_ms": 36670.56565656565, "avg_output_tokens_per_task": 2059.811447811448, "last_run_timestamp": "2026-10-30T00:02:55.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20203182828282829, "avg_duration_per_task_ms": 133595.55303030304, "avg_output_tokens_per_task": 16712.858585858587, "last_run_timestamp": "2026-10-01T19:16:14.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11333201010101009, "avg_duration_per_task_ms": 57056.792929292926, "avg_output_tokens_per_task": 2108.121212121212, "last_run_timestamp": "2026-10-01T08:25:16.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.944444, "accuracy_std_dev": 0.0027491439879836154, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020607563973063972, "avg_duration_per_task_ms": 47519.37542087542, "avg_output_tokens_per_task": 1902.2373737373737, "last_run_timestamp": "2026-10-02T23:20:22.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02136650505050505, "avg_duration_per_task_ms": 52411.32323232323, "avg_output_tokens_per_task": 1978.1313131313132, "last_run_timestamp": "2026-10-01T08:24:51.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9431713315363881, "accuracy_std_dev": 0.006147978549897494, "run_count": 4, "total_questions": 742, "avg_cost_per_task_usd": 0.10982969407008086, "avg_duration_per_task_ms": 117613.10242587602, "avg_output_tokens_per_task": 1949.2371967654988, "last_run_timestamp": "2026-10-01T08:26:36.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9427606666666666, "accuracy_std_dev": 0.0036369299629708685, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.029872336700336698, "avg_duration_per_task_ms": 28492.575757575756, "avg_output_tokens_per_task": 7819.474747474747, "last_run_timestamp": "2026-10-01T16:16:03.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.941077, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.15125092424242426, "avg_duration_per_task_ms": 97603.77272727272, "avg_output_tokens_per_task": 11559.585858585859, "last_run_timestamp": "2026-10-01T16:10:33.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.939394, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.31252995454545457, "avg_duration_per_task_ms": 150832.55050505052, "avg_output_tokens_per_task": 10288.712121212122, "last_run_timestamp": "2026-10-01T19:12:11.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.939394, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.32760574494949496, "avg_duration_per_task_ms": 78455.45202020202, "avg_output_tokens_per_task": 11977.780303030304, "last_run_timestamp": "2026-10-01T16:36:19.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.9393935, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.15067959090909092, "avg_duration_per_task_ms": 95544.60353535354, "avg_output_tokens_per_task": 11515.699494949495, "last_run_timestamp": "2026-10-02T23:21:48.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.9391951653543308, "accuracy_std_dev": 0.009884880879403648, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.026313035433070862, "avg_duration_per_task_ms": 56131.107611548556, "avg_output_tokens_per_task": 7729.320209973754, "last_run_timestamp": "2026-10-15T21:01:11.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.938528, "accuracy_std_dev": 0.004357499999999959, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.0293307974025974, "avg_duration_per_task_ms": 59248.22337662338, "avg_output_tokens_per_task": 7683.532467532467, "last_run_timestamp": "2026-10-01T16:17:32.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.9382713333333333, "accuracy_std_dev": 0.004627357405210382, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022108563973063967, "avg_duration_per_task_ms": 32928.50084175084, "avg_output_tokens_per_task": 1914.9974747474748, "last_run_timestamp": "2026-10-01T08:40:50.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.9360266666666668, "accuracy_std_dev": 0.0036369299629708854, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3141816919191919, "avg_duration_per_task_ms": 142474.1195286195, "avg_output_tokens_per_task": 10342.412457912458, "last_run_timestamp": "2026-10-02T23:25:41.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Io Net", "accuracy": 0.9353121872146118, "accuracy_std_dev": 0.0026674999999999893, "run_count": 2, "total_questions": 219, "avg_cost_per_task_usd": 0.014858730593607304, "avg_duration_per_task_ms": 395134.2511415525, "avg_output_tokens_per_task": 29583.762557077625, "last_run_timestamp": "2026-10-24T08:19:41.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash", "model_slug": "google/gemini-3.8-flash", "model_name": "Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.9343430000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023175040404040403, "avg_duration_per_task_ms": 44381.02525252525, "avg_output_tokens_per_task": 6131.126262626262, "last_run_timestamp": "2026-10-23T07:37:43.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9336215714285715, "accuracy_std_dev": 0.008810277924222438, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.3281882835497835, "avg_duration_per_task_ms": 164668.6948051948, "avg_output_tokens_per_task": 10091.254689754689, "last_run_timestamp": "2026-10-01T19:14:07.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra-v2", "model_slug": "sakana/fugu-ultra-v2", "model_name": "Fugu Ultra V2", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.5025571414141414, "avg_duration_per_task_ms": 194915.41919191918, "avg_output_tokens_per_task": 13806.227272727272, "last_run_timestamp": "2026-09-15T01:07:48.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "typesafe/jev-router", "model_slug": "typesafe/jev-router", "model_name": "TypeSafe: Jev Router", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03490100505050505, "avg_duration_per_task_ms": 57907.71885521885, "avg_output_tokens_per_task": 6070.841750841751, "last_run_timestamp": "2026-10-09T23:23:36.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.9309762, "accuracy_std_dev": 0.005106374267520954, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.057293524242424246, "avg_duration_per_task_ms": 66775.28888888888, "avg_output_tokens_per_task": 11798.8, "last_run_timestamp": "2026-10-01T19:12:42.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.9308640666666669, "accuracy_std_dev": 0.007021551143602255, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.026242806060606066, "avg_duration_per_task_ms": 18771.915488215487, "avg_output_tokens_per_task": 2231.500336700337, "last_run_timestamp": "2026-10-24T08:30:19.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.9306396, "accuracy_std_dev": 0.006683344480123684, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025279440404040406, "avg_duration_per_task_ms": 20334.968686868688, "avg_output_tokens_per_task": 2296.2020202020203, "last_run_timestamp": "2026-10-24T08:11:43.558Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.9303028, "accuracy_std_dev": 0.00377921650081072, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02488049898989899, "avg_duration_per_task_ms": 19525.145454545454, "avg_output_tokens_per_task": 2247.960606060606, "last_run_timestamp": "2026-10-24T08:10:23.014Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9301345, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.044492378787878796, "avg_duration_per_task_ms": 69170.04040404041, "avg_output_tokens_per_task": 11715.780303030304, "last_run_timestamp": "2026-10-01T19:12:42.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.928451, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.3476244823232323, "avg_duration_per_task_ms": 112460.52777777778, "avg_output_tokens_per_task": 10405.60101010101, "last_run_timestamp": "2026-10-01T19:13:06.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12326839393939394, "avg_duration_per_task_ms": 361758.7878787879, "avg_output_tokens_per_task": 20322.222222222223, "last_run_timestamp": "2026-10-01T16:09:08.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.9280388242811503, "accuracy_std_dev": 0.010739874350082303, "run_count": 12, "total_questions": 2191, "avg_cost_per_task_usd": 0.1293467288909174, "avg_duration_per_task_ms": 132537.31081697854, "avg_output_tokens_per_task": 8542.681424007302, "last_run_timestamp": "2026-10-24T08:42:17.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.9276095000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04436604545454545, "avg_duration_per_task_ms": 61670.28282828283, "avg_output_tokens_per_task": 11683.290404040405, "last_run_timestamp": "2026-10-01T19:11:04.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9276091428571428, "accuracy_std_dev": 0.005010400338398178, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10054259307359308, "avg_duration_per_task_ms": 55310.95093795094, "avg_output_tokens_per_task": 3496.1031746031745, "last_run_timestamp": "2026-10-01T19:42:48.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.9271917641601563, "accuracy_std_dev": 0.007748903788582068, "run_count": 33, "total_questions": 6144, "avg_cost_per_task_usd": 0.17010933951822915, "avg_duration_per_task_ms": 162569.57161458334, "avg_output_tokens_per_task": 9313.541341145834, "last_run_timestamp": "2026-10-24T08:50:28.834Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.9271882499999999, "accuracy_std_dev": 0.006560703749408305, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.025096623737373737, "avg_duration_per_task_ms": 19787.91414141414, "avg_output_tokens_per_task": 2272.02904040404, "last_run_timestamp": "2026-10-24T08:11:40.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.9264870000000001, "accuracy_std_dev": 0.007815979699734826, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03737200673400674, "avg_duration_per_task_ms": 60770.02693602694, "avg_output_tokens_per_task": 3580.875420875421, "last_run_timestamp": "2026-10-02T23:20:36.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9261662857142858, "accuracy_std_dev": 0.0030420094728079157, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026267576479076477, "avg_duration_per_task_ms": 19507.747474747473, "avg_output_tokens_per_task": 2314.836940836941, "last_run_timestamp": "2026-10-29T00:04:34.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.9259260000000001, "accuracy_std_dev": 0.004610436378478734, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.10788488484848485, "avg_duration_per_task_ms": 187765.80707070706, "avg_output_tokens_per_task": 9499.632323232323, "last_run_timestamp": "2026-10-24T08:37:09.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.925795996178344, "accuracy_std_dev": 0.007145577022492671, "run_count": 8, "total_questions": 1570, "avg_cost_per_task_usd": 0.1095707592356688, "avg_duration_per_task_ms": 200872.2719745223, "avg_output_tokens_per_task": 9447.89617834395, "last_run_timestamp": "2026-10-24T08:14:36.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Darkbloom", "accuracy": 0.925214, "accuracy_std_dev": null, "run_count": 1, "total_questions": 78, "avg_cost_per_task_usd": 0.0012695128205128204, "avg_duration_per_task_ms": 237789.78205128206, "avg_output_tokens_per_task": 6844.423076923077, "last_run_timestamp": "2026-10-30T01:38:27.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DeepInfra", "accuracy": 0.9247310806451613, "accuracy_std_dev": 0.013119500000000006, "run_count": 2, "total_questions": 372, "avg_cost_per_task_usd": 0.005701161290322581, "avg_duration_per_task_ms": 222245.45967741936, "avg_output_tokens_per_task": 19159.182795698925, "last_run_timestamp": "2026-10-24T08:13:25.170Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9246375695652174, "accuracy_std_dev": 0.006863146440316182, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.12292555942028985, "avg_duration_per_task_ms": 340480.1579710145, "avg_output_tokens_per_task": 19662.684057971015, "last_run_timestamp": "2026-10-01T16:09:31.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.924599700084246, "accuracy_std_dev": 0.006819818440072705, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.02751449789385004, "avg_duration_per_task_ms": 24957.139848357205, "avg_output_tokens_per_task": 2420.7127211457455, "last_run_timestamp": "2026-10-01T08:24:52.927Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9245788000000001, "accuracy_std_dev": 0.005259575777569916, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1499325777777778, "avg_duration_per_task_ms": 85124.36363636363, "avg_output_tokens_per_task": 15611.38383838384, "last_run_timestamp": "2026-10-01T19:13:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03863523484848485, "avg_duration_per_task_ms": 63161.229797979795, "avg_output_tokens_per_task": 3708.2954545454545, "last_run_timestamp": "2026-10-01T19:12:28.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "unbiased/pareto-26.10-preview", "model_slug": "unbiased/pareto-26.10-preview", "model_name": "Pareto 26.10 Preview", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0030723434343434344, "avg_duration_per_task_ms": 37807.181818181816, "avg_output_tokens_per_task": 903.9090909090909, "last_run_timestamp": "2026-10-15T14:24:31.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1436656515151515, "avg_duration_per_task_ms": 89621.648989899, "avg_output_tokens_per_task": 15839.800505050505, "last_run_timestamp": "2026-10-01T19:18:40.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.9230042575935954, "accuracy_std_dev": 0.028834365810508387, "run_count": 22, "total_questions": 4247, "avg_cost_per_task_usd": 0.12647735083588418, "avg_duration_per_task_ms": 173368.8584883447, "avg_output_tokens_per_task": 9831.030138921591, "last_run_timestamp": "2026-10-24T08:45:15.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9225589999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14588472727272728, "avg_duration_per_task_ms": 63933.833333333336, "avg_output_tokens_per_task": 16085.368686868687, "last_run_timestamp": "2026-10-01T19:12:44.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9220161548643283, "accuracy_std_dev": 0.015489389982329555, "run_count": 9, "total_questions": 1511, "avg_cost_per_task_usd": 0.1115360423560556, "avg_duration_per_task_ms": 205913.44606221045, "avg_output_tokens_per_task": 8295.92322964924, "last_run_timestamp": "2026-10-24T08:11:11.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9219976666666666, "accuracy_std_dev": 0.0028614651647170097, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.08309612457912459, "avg_duration_per_task_ms": 49364.952861952865, "avg_output_tokens_per_task": 3748.658249158249, "last_run_timestamp": "2026-10-30T00:00:06.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.9214363999999999, "accuracy_std_dev": 0.007595474019001951, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.13534853400673405, "avg_duration_per_task_ms": 180051.0377104377, "avg_output_tokens_per_task": 11618.102693602694, "last_run_timestamp": "2026-10-24T09:13:18.915Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9214090472319009, "accuracy_std_dev": 0.0068651349363197224, "run_count": 14, "total_questions": 2583, "avg_cost_per_task_usd": 0.14661487727448702, "avg_duration_per_task_ms": 125761.86372435153, "avg_output_tokens_per_task": 9628.765001935733, "last_run_timestamp": "2026-10-24T08:48:19.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.92087525, "accuracy_std_dev": 0.004610545920766882, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.14864516792929292, "avg_duration_per_task_ms": 124980.99747474748, "avg_output_tokens_per_task": 9671.253787878788, "last_run_timestamp": "2026-10-24T08:58:50.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9208750000000001, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.026268675084175085, "avg_duration_per_task_ms": 25466.75420875421, "avg_output_tokens_per_task": 2308.39898989899, "last_run_timestamp": "2026-10-30T00:00:05.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.9208749999999999, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025590255892255893, "avg_duration_per_task_ms": 33502.22222222222, "avg_output_tokens_per_task": 2400.506734006734, "last_run_timestamp": "2026-10-02T23:17:42.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "ParetoAlpha", "accuracy": 0.9203848188976379, "accuracy_std_dev": 0.004243859645122607, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.15456283070866142, "avg_duration_per_task_ms": 390118.6771653543, "avg_output_tokens_per_task": 10091.72965879265, "last_run_timestamp": "2026-10-30T06:35:32.515Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.9203701999999999, "accuracy_std_dev": 0.006736210266908262, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.17098447676767675, "avg_duration_per_task_ms": 297723.4101010101, "avg_output_tokens_per_task": 9720.665151515152, "last_run_timestamp": "2026-10-28T03:40:38.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DigitalOcean", "accuracy": 0.9200335, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.021749656565656566, "avg_duration_per_task_ms": 75214.94191919192, "avg_output_tokens_per_task": 20570.64898989899, "last_run_timestamp": "2026-10-24T08:29:16.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02528019191919192, "avg_duration_per_task_ms": 32477.28787878788, "avg_output_tokens_per_task": 2369.5, "last_run_timestamp": "2026-10-01T08:24:24.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Azure", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01357689898989899, "avg_duration_per_task_ms": 18304.29292929293, "avg_output_tokens_per_task": 2104.2979797979797, "last_run_timestamp": "2026-10-28T00:00:04.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.918981375, "accuracy_std_dev": 0.00679620392089402, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12805398863636364, "avg_duration_per_task_ms": 541652.8712121212, "avg_output_tokens_per_task": 37116.541666666664, "last_run_timestamp": "2026-10-03T09:05:48.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.9182648918918918, "accuracy_std_dev": 0.012947711305692437, "run_count": 12, "total_questions": 2294, "avg_cost_per_task_usd": 0.1290118766346992, "avg_duration_per_task_ms": 117697.42414995641, "avg_output_tokens_per_task": 8571.878378378378, "last_run_timestamp": "2026-10-24T08:44:47.181Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9181606468965516, "accuracy_std_dev": 0.010622287016200703, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.13311453471264367, "avg_duration_per_task_ms": 278187.9034482759, "avg_output_tokens_per_task": 10469.161379310344, "last_run_timestamp": "2026-10-24T08:16:43.073Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9180774433042584, "accuracy_std_dev": 0.01435885589070384, "run_count": 10, "total_questions": 1949, "avg_cost_per_task_usd": 0.24670780708055412, "avg_duration_per_task_ms": 414365.66136480245, "avg_output_tokens_per_task": 36895.286813750645, "last_run_timestamp": "2026-10-24T08:18:32.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.9171716, "accuracy_std_dev": 0.0037495390703391955, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.02583652727272727, "avg_duration_per_task_ms": 19190.16717171717, "avg_output_tokens_per_task": 2225.3469696969696, "last_run_timestamp": "2026-10-24T08:28:10.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.9165345411718132, "accuracy_std_dev": 0.00728827618680496, "run_count": 13, "total_questions": 2526, "avg_cost_per_task_usd": 0.14153307442596988, "avg_duration_per_task_ms": 283509.3226444972, "avg_output_tokens_per_task": 9267.701108471892, "last_run_timestamp": "2026-10-24T08:44:42.687Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cohere", "accuracy": 0.915825, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014131747474747476, "avg_duration_per_task_ms": 59960.4595959596, "avg_output_tokens_per_task": 21225.651515151516, "last_run_timestamp": "2026-10-28T21:00:06.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "SiliconFlow", "accuracy": 0.9158245, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026372479797979794, "avg_duration_per_task_ms": 193752.02525252526, "avg_output_tokens_per_task": 21866.449494949495, "last_run_timestamp": "2026-10-24T08:46:05.254Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.9152636666666667, "accuracy_std_dev": 0.006963814615568124, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.09072644781144781, "avg_duration_per_task_ms": 657647.413580247, "avg_output_tokens_per_task": 40463.242985409655, "last_run_timestamp": "2026-10-24T08:13:28.067Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.9150769353846153, "accuracy_std_dev": 0.01273628821599266, "run_count": 12, "total_questions": 1625, "avg_cost_per_task_usd": 0.11317312553846154, "avg_duration_per_task_ms": 182427.93907692307, "avg_output_tokens_per_task": 7798.748307692308, "last_run_timestamp": "2026-10-30T00:27:14.164Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Baidu", "accuracy": 0.9147832881355932, "accuracy_std_dev": 0.01218950000000002, "run_count": 2, "total_questions": 354, "avg_cost_per_task_usd": 0.02172540395480226, "avg_duration_per_task_ms": 124806.83333333333, "avg_output_tokens_per_task": 19165.833333333332, "last_run_timestamp": "2026-10-24T08:44:43.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.9144541630847027, "accuracy_std_dev": 0.007898223803049383, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.09740494732406238, "avg_duration_per_task_ms": 296054.32490518334, "avg_output_tokens_per_task": 25118.074589127686, "last_run_timestamp": "2026-10-29T22:45:19.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Mancer 2", "accuracy": 0.9142155073529412, "accuracy_std_dev": 0.012753000000000014, "run_count": 2, "total_questions": 272, "avg_cost_per_task_usd": 0.014779014705882354, "avg_duration_per_task_ms": 301053.6323529412, "avg_output_tokens_per_task": 14660.216911764706, "last_run_timestamp": "2026-10-24T08:32:55.335Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra", "model_slug": "openai/gpt-6.1-terra", "model_name": "GPT-6.1-Terra", "provider_name": "auto-routing", "accuracy": 0.9141410000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0075055656565656565, "avg_duration_per_task_ms": 21128.91414141414, "avg_output_tokens_per_task": 1395.4343434343434, "last_run_timestamp": "2026-10-27T03:32:35.022Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.91386075, "accuracy_std_dev": 0.013535011243222777, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.034369089225589226, "avg_duration_per_task_ms": 228439.68476430976, "avg_output_tokens_per_task": 28471.782828282827, "last_run_timestamp": "2026-10-24T16:21:24.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.913200685767098, "accuracy_std_dev": 0.02916766954216023, "run_count": 22, "total_questions": 4328, "avg_cost_per_task_usd": 0.1485330272643253, "avg_duration_per_task_ms": 403035.12731053605, "avg_output_tokens_per_task": 9145.585720887246, "last_run_timestamp": "2026-10-24T08:17:56.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9131792857142857, "accuracy_std_dev": 0.008577350568532835, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.15955438744588746, "avg_duration_per_task_ms": 565778.9992784993, "avg_output_tokens_per_task": 39975.08946608946, "last_run_timestamp": "2026-10-24T08:28:29.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Alibaba", "accuracy": 0.913019, "accuracy_std_dev": 0.006496204173720757, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.027868947811447813, "avg_duration_per_task_ms": 190559.73569023568, "avg_output_tokens_per_task": 21553.755892255893, "last_run_timestamp": "2026-10-28T03:31:51.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "GMICloud", "accuracy": 0.9118966666666666, "accuracy_std_dev": 0.0028614651647170097, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020073099326599324, "avg_duration_per_task_ms": 146602.46801346802, "avg_output_tokens_per_task": 22818.382154882154, "last_run_timestamp": "2026-10-27T00:36:02.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.9118396713596139, "accuracy_std_dev": 0.015391401769750214, "run_count": 13, "total_questions": 2486, "avg_cost_per_task_usd": 0.1705156729686243, "avg_duration_per_task_ms": 504972.3576025744, "avg_output_tokens_per_task": 11645.380128720837, "last_run_timestamp": "2026-10-24T08:25:10.474Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Decart", "accuracy": 0.911616, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007887881313131314, "avg_duration_per_task_ms": 185118.85353535353, "avg_output_tokens_per_task": 21770.37626262626, "last_run_timestamp": "2026-10-24T08:29:45.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.9115664705882354, "accuracy_std_dev": 0.014309678991415931, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.13998793790849673, "avg_duration_per_task_ms": 602578.7964943553, "avg_output_tokens_per_task": 39991.1972667855, "last_run_timestamp": "2026-10-28T04:18:14.838Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.911111096, "accuracy_std_dev": 0.009498497417355022, "run_count": 4, "total_questions": 750, "avg_cost_per_task_usd": 0.136844476, "avg_duration_per_task_ms": 321154.29333333333, "avg_output_tokens_per_task": 35843.412, "last_run_timestamp": "2026-10-24T08:11:11.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "OpenAI", "accuracy": 0.910774, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.011462444444444444, "avg_duration_per_task_ms": 27557.444444444445, "avg_output_tokens_per_task": 2133.969696969697, "last_run_timestamp": "2026-10-28T00:00:04.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "AtlasCloud", "accuracy": 0.9099324999999999, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02111154797979798, "avg_duration_per_task_ms": 145444.96212121213, "avg_output_tokens_per_task": 21297.967171717173, "last_run_timestamp": "2026-10-24T08:04:44.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Fireworks", "accuracy": 0.9097220343749999, "accuracy_std_dev": 0.007467330848435752, "run_count": 5, "total_questions": 960, "avg_cost_per_task_usd": 0.023086179166666665, "avg_duration_per_task_ms": 81909.69166666667, "avg_output_tokens_per_task": 20192.428125, "last_run_timestamp": "2026-10-28T00:00:04.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Nebius", "accuracy": 0.909652, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025303424242424242, "avg_duration_per_task_ms": 91659.5925925926, "avg_output_tokens_per_task": 20911.968013468013, "last_run_timestamp": "2026-10-24T08:24:39.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9095115, "accuracy_std_dev": 0.002759842885020819, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0416229191919192, "avg_duration_per_task_ms": 138307.48484848486, "avg_output_tokens_per_task": 27969.503787878788, "last_run_timestamp": "2026-10-01T16:11:23.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.9094272, "accuracy_std_dev": 0.00547072746168187, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025930728282828285, "avg_duration_per_task_ms": 177886.11212121212, "avg_output_tokens_per_task": 28121.050505050505, "last_run_timestamp": "2026-10-03T09:34:35.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Particle AI", "accuracy": 0.9090910000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017200570707070706, "avg_duration_per_task_ms": 67652.35606060606, "avg_output_tokens_per_task": 17022.85606060606, "last_run_timestamp": "2026-10-28T22:43:29.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1305062777777778, "avg_duration_per_task_ms": 540279.6767676767, "avg_output_tokens_per_task": 38448.40404040404, "last_run_timestamp": "2026-10-01T09:14:23.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0054125, "avg_duration_per_task_ms": 235839.91414141413, "avg_output_tokens_per_task": 10749.717171717171, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2221867676767677, "avg_duration_per_task_ms": 298883.5202020202, "avg_output_tokens_per_task": 29352.99494949495, "last_run_timestamp": "2026-09-27T00:06:19.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.9090908333333333, "accuracy_std_dev": 0.008189992133017543, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03826549410774411, "avg_duration_per_task_ms": 134585.28703703705, "avg_output_tokens_per_task": 28750.484848484848, "last_run_timestamp": "2026-10-27T00:36:05.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.908733465037911, "accuracy_std_dev": 0.006850022329140406, "run_count": 12, "total_questions": 2374, "avg_cost_per_task_usd": 0.030451491996630154, "avg_duration_per_task_ms": 408677.97219882056, "avg_output_tokens_per_task": 31497.24094355518, "last_run_timestamp": "2026-10-24T16:30:24.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.9084910475789473, "accuracy_std_dev": 0.04945617807940908, "run_count": 12, "total_questions": 2375, "avg_cost_per_task_usd": 0.14244542484210523, "avg_duration_per_task_ms": 331138.315368421, "avg_output_tokens_per_task": 10400.368421052632, "last_run_timestamp": "2026-10-24T08:48:54.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9080194545454545, "accuracy_std_dev": 0.015874287061234574, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.035897662534435264, "avg_duration_per_task_ms": 311230.74242424243, "avg_output_tokens_per_task": 29833.370982552802, "last_run_timestamp": "2026-10-24T16:24:24.501Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.9077097088435375, "accuracy_std_dev": 0.01602150486845725, "run_count": 4, "total_questions": 735, "avg_cost_per_task_usd": 0.12614392789115647, "avg_duration_per_task_ms": 1486736.511564626, "avg_output_tokens_per_task": 50215.49115646259, "last_run_timestamp": "2026-10-24T08:43:13.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.9074072500000001, "accuracy_std_dev": 0.008828586845441347, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.009289530303030303, "avg_duration_per_task_ms": 172567.80050505052, "avg_output_tokens_per_task": 18481.776515151516, "last_run_timestamp": "2026-10-24T08:44:46.341Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Venice", "accuracy": 0.907407, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025121388888888888, "avg_duration_per_task_ms": 92618.0808080808, "avg_output_tokens_per_task": 20743.772727272728, "last_run_timestamp": "2026-10-29T00:00:06.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Ionstream", "accuracy": 0.907407, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018446676767676768, "avg_duration_per_task_ms": 140631.38888888888, "avg_output_tokens_per_task": 20189.31818181818, "last_run_timestamp": "2026-10-24T08:54:33.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.907407, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.012070199494949497, "avg_duration_per_task_ms": 98657.74747474748, "avg_output_tokens_per_task": 21841.54797979798, "last_run_timestamp": "2026-10-24T08:21:56.264Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.907407, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.010421979797979798, "avg_duration_per_task_ms": 107076.06565656565, "avg_output_tokens_per_task": 18400.97474747475, "last_run_timestamp": "2026-10-24T08:14:51.110Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9070186923076924, "accuracy_std_dev": 0.024866682548789026, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.03558504700854701, "avg_duration_per_task_ms": 231138.5062160062, "avg_output_tokens_per_task": 29778.67948717949, "last_run_timestamp": "2026-10-27T00:51:48.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/switchyard", "model_slug": "nvidia/switchyard", "model_name": "NVIDIA: Switchyard", "provider_name": "auto-routing", "accuracy": 0.9067937377877597, "accuracy_std_dev": 0.03179893182614709, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.01609888826501965, "avg_duration_per_task_ms": 58211.32790567097, "avg_output_tokens_per_task": 3297.93037619315, "last_run_timestamp": "2026-10-16T18:32:13.741Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.9067057499999999, "accuracy_std_dev": 0.02181467158101399, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.15476828703703704, "avg_duration_per_task_ms": 601020.0946969697, "avg_output_tokens_per_task": 39038.01388888889, "last_run_timestamp": "2026-10-24T08:50:20.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.9066696814159291, "accuracy_std_dev": 0.02221528527057198, "run_count": 11, "total_questions": 2034, "avg_cost_per_task_usd": 0.150266023107178, "avg_duration_per_task_ms": 558942.4459193707, "avg_output_tokens_per_task": 37986.727138643066, "last_run_timestamp": "2026-10-24T09:27:08.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.9065654999999999, "accuracy_std_dev": 0.00811767264984244, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03409049431818181, "avg_duration_per_task_ms": 218247.69949494948, "avg_output_tokens_per_task": 27597.452651515152, "last_run_timestamp": "2026-10-28T03:27:34.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Sail Research", "accuracy": 0.9065654999999999, "accuracy_std_dev": 0.008117672649842422, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0029748573232323233, "avg_duration_per_task_ms": 218585.40025252526, "avg_output_tokens_per_task": 19647.71212121212, "last_run_timestamp": "2026-10-24T08:58:07.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9064251666666665, "accuracy_std_dev": 0.020964442642377977, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03777529545454546, "avg_duration_per_task_ms": 260085.33417508416, "avg_output_tokens_per_task": 31644.78872053872, "last_run_timestamp": "2026-10-24T16:42:35.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "BaseTen", "accuracy": 0.9063117514792899, "accuracy_std_dev": 0.0007095000000000296, "run_count": 2, "total_questions": 338, "avg_cost_per_task_usd": 0.0210026775147929, "avg_duration_per_task_ms": 63992.53550295858, "avg_output_tokens_per_task": 17351.381656804733, "last_run_timestamp": "2026-10-24T08:13:28.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.9060604000000001, "accuracy_std_dev": 0.0032644069966840917, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02883772424242424, "avg_duration_per_task_ms": 166818.5292929293, "avg_output_tokens_per_task": 28846.466666666667, "last_run_timestamp": "2026-10-24T08:33:28.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Wafer", "accuracy": 0.9060041666666666, "accuracy_std_dev": 0.005181136857764807, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008250873737373737, "avg_duration_per_task_ms": 82060.31565656565, "avg_output_tokens_per_task": 16356.271043771043, "last_run_timestamp": "2026-10-29T22:25:59.214Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.9059828461538462, "accuracy_std_dev": 0.02626261808627545, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.04094654972804972, "avg_duration_per_task_ms": 283710.56487956486, "avg_output_tokens_per_task": 31884.02874902875, "last_run_timestamp": "2026-10-24T16:17:39.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.9058946753684209, "accuracy_std_dev": 0.029872349099497063, "run_count": 12, "total_questions": 2375, "avg_cost_per_task_usd": 0.10017258273684211, "avg_duration_per_task_ms": 566875.4896842105, "avg_output_tokens_per_task": 39376.88463157895, "last_run_timestamp": "2026-10-24T08:54:07.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StreamLake", "accuracy": 0.905724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.026723479797979795, "avg_duration_per_task_ms": 119393.66666666667, "avg_output_tokens_per_task": 22072.065656565657, "last_run_timestamp": "2026-10-26T03:00:10.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "auto-routing", "accuracy": 0.905724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06900820707070707, "avg_duration_per_task_ms": 101485.72222222222, "avg_output_tokens_per_task": 16020.974747474747, "last_run_timestamp": "2026-10-24T08:27:04.527Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.9057143142857144, "accuracy_std_dev": 0.001963571554591265, "run_count": 4, "total_questions": 700, "avg_cost_per_task_usd": 0.18660921714285714, "avg_duration_per_task_ms": 435582.91714285716, "avg_output_tokens_per_task": 41735.86142857143, "last_run_timestamp": "2026-10-24T08:46:32.813Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Modal", "accuracy": 0.9056120000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.06647105102040816, "avg_duration_per_task_ms": 64791.01530612245, "avg_output_tokens_per_task": 15327.612244897959, "last_run_timestamp": "2026-10-24T08:14:36.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.905376, "accuracy_std_dev": null, "run_count": 1, "total_questions": 155, "avg_cost_per_task_usd": 0.09219292258064517, "avg_duration_per_task_ms": 892432.9419354838, "avg_output_tokens_per_task": 36790.02580645162, "last_run_timestamp": "2026-10-02T06:44:44.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.905323592920354, "accuracy_std_dev": 0.03949844256599099, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.1419002410450906, "avg_duration_per_task_ms": 207365.2397808681, "avg_output_tokens_per_task": 9535.341761483354, "last_run_timestamp": "2026-10-24T08:46:33.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Makora", "accuracy": 0.9053030000000001, "accuracy_std_dev": 0.004187691070267732, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.026182512626262627, "avg_duration_per_task_ms": 138194.00252525252, "avg_output_tokens_per_task": 21714.06186868687, "last_run_timestamp": "2026-10-24T08:02:23.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.9051626666666666, "accuracy_std_dev": 0.0055553002519115975, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.009504934343434343, "avg_duration_per_task_ms": 118639.06902356903, "avg_output_tokens_per_task": 16467.956228956227, "last_run_timestamp": "2026-10-24T08:45:01.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.9049724513274336, "accuracy_std_dev": 0.008906991898924113, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.020146505689001264, "avg_duration_per_task_ms": 644404.02528445, "avg_output_tokens_per_task": 42703.32616940582, "last_run_timestamp": "2026-10-28T03:59:43.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Modal", "accuracy": 0.904882, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.022279229797979798, "avg_duration_per_task_ms": 122775.77777777778, "avg_output_tokens_per_task": 22121.87626262626, "last_run_timestamp": "2026-10-24T08:33:42.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.9044612500000001, "accuracy_std_dev": 0.009172855468037184, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03501683964646465, "avg_duration_per_task_ms": 199099.8143939394, "avg_output_tokens_per_task": 29016.732954545456, "last_run_timestamp": "2026-10-24T08:11:11.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.904461125, "accuracy_std_dev": 0.008737580020198659, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02118971275252525, "avg_duration_per_task_ms": 207950.51199494948, "avg_output_tokens_per_task": 28934.199494949495, "last_run_timestamp": "2026-10-24T08:11:11.332Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.9040405, "accuracy_std_dev": 0.011784499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13174327272727274, "avg_duration_per_task_ms": 249918.9898989899, "avg_output_tokens_per_task": 43791.378787878784, "last_run_timestamp": "2026-10-01T19:18:35.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "Kimi K3.1 Pro", "provider_name": "auto-routing", "accuracy": 0.9040400000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.021051848484848486, "avg_duration_per_task_ms": 58149.92929292929, "avg_output_tokens_per_task": 2035.7424242424242, "last_run_timestamp": "2026-10-26T19:26:14.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.9040400000000001, "accuracy_std_dev": 0.004761657064510192, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03280091414141414, "avg_duration_per_task_ms": 814621.6686868687, "avg_output_tokens_per_task": 37346.743434343436, "last_run_timestamp": "2026-10-24T08:23:31.992Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Makora", "accuracy": 0.9039929892328398, "accuracy_std_dev": 0.013615822071674532, "run_count": 4, "total_questions": 743, "avg_cost_per_task_usd": 0.061841156123822336, "avg_duration_per_task_ms": 74303.45625841184, "avg_output_tokens_per_task": 14413.555854643339, "last_run_timestamp": "2026-10-24T08:25:42.800Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.9036307954401521, "accuracy_std_dev": 0.011636066678482626, "run_count": 16, "total_questions": 3158, "avg_cost_per_task_usd": 0.027261241925269155, "avg_duration_per_task_ms": 186560.3065231159, "avg_output_tokens_per_task": 25898.152311589613, "last_run_timestamp": "2026-10-28T03:34:20.229Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.9036194166666667, "accuracy_std_dev": 0.01817259761499133, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.034217859006734005, "avg_duration_per_task_ms": 319774.7689393939, "avg_output_tokens_per_task": 30123.76936026936, "last_run_timestamp": "2026-10-24T16:12:49.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimeIntellect", "accuracy": 0.90361925, "accuracy_std_dev": 0.005630970448111022, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.026822416666666668, "avg_duration_per_task_ms": 92441.06818181818, "avg_output_tokens_per_task": 22174.864898989897, "last_run_timestamp": "2026-10-29T23:05:04.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.9036160645842126, "accuracy_std_dev": 0.011349068793175144, "run_count": 12, "total_questions": 2369, "avg_cost_per_task_usd": 0.028033349514563106, "avg_duration_per_task_ms": 292905.91599831154, "avg_output_tokens_per_task": 23207.665681722247, "last_run_timestamp": "2026-10-24T16:03:25.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.903508802631579, "accuracy_std_dev": 0.0984941779678147, "run_count": 3, "total_questions": 456, "avg_cost_per_task_usd": 0.11285829166666667, "avg_duration_per_task_ms": 423954.7916666667, "avg_output_tokens_per_task": 16846.184210526317, "last_run_timestamp": "2026-10-01T16:04:59.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.9033163440134907, "accuracy_std_dev": 0.008355089002931883, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.015168129286115795, "avg_duration_per_task_ms": 392884.92523889826, "avg_output_tokens_per_task": 28918.40921866217, "last_run_timestamp": "2026-10-24T08:18:49.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.9031985, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14226843434343436, "avg_duration_per_task_ms": 149341.0782828283, "avg_output_tokens_per_task": 9356.287878787878, "last_run_timestamp": "2026-10-01T19:14:55.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.9031626720031235, "accuracy_std_dev": 0.019975453712141735, "run_count": 13, "total_questions": 2561, "avg_cost_per_task_usd": 0.11988065286997265, "avg_duration_per_task_ms": 343221.44474814524, "avg_output_tokens_per_task": 8205.648574775478, "last_run_timestamp": "2026-10-24T08:49:03.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9026110979153505, "accuracy_std_dev": 0.006660054560587301, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.028997156664560955, "avg_duration_per_task_ms": 168876.1749842072, "avg_output_tokens_per_task": 28895.867340492736, "last_run_timestamp": "2026-10-24T08:51:57.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.902357, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007275146464646465, "avg_duration_per_task_ms": 217667.1313131313, "avg_output_tokens_per_task": 25683.540404040403, "last_run_timestamp": "2026-08-08T09:30:52.456Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.902357, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06939250673400674, "avg_duration_per_task_ms": 257971.41245791246, "avg_output_tokens_per_task": 16102.877104377105, "last_run_timestamp": "2026-10-30T00:00:05.460Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.902357, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1437863712121212, "avg_duration_per_task_ms": 149585.63636363635, "avg_output_tokens_per_task": 9456.126262626263, "last_run_timestamp": "2026-10-01T19:14:23.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Mistral", "accuracy": 0.9023569999999999, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02541817676767677, "avg_duration_per_task_ms": 99436.77525252526, "avg_output_tokens_per_task": 21016.33585858586, "last_run_timestamp": "2026-10-30T16:01:57.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.9023568, "accuracy_std_dev": 0.005733794394639539, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.05410575151515151, "avg_duration_per_task_ms": 31371.09595959596, "avg_output_tokens_per_task": 2496.787878787879, "last_run_timestamp": "2026-10-24T08:04:44.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.9023566666666667, "accuracy_std_dev": 0.003636621356271357, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.053918992424242425, "avg_duration_per_task_ms": 26868.823232323233, "avg_output_tokens_per_task": 2480.0361952861954, "last_run_timestamp": "2026-10-24T08:14:51.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Nebius", "accuracy": 0.9017956666666668, "accuracy_std_dev": 0.003459479954495376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.014097868686868686, "avg_duration_per_task_ms": 101233.50336700337, "avg_output_tokens_per_task": 13920.223905723906, "last_run_timestamp": "2026-10-24T08:29:16.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.9017254583333333, "accuracy_std_dev": 0.0058683477301193655, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.2056731323653199, "avg_duration_per_task_ms": 97891.09090909091, "avg_output_tokens_per_task": 7837.397516835017, "last_run_timestamp": "2026-10-24T08:44:41.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.9016089931034483, "accuracy_std_dev": 0.017035826057784066, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.10899811586206896, "avg_duration_per_task_ms": 561470.3995402299, "avg_output_tokens_per_task": 37112.96183908046, "last_run_timestamp": "2026-10-24T08:01:41.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-1017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "Deepseek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.901515, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006598151515151515, "avg_duration_per_task_ms": 72990.44444444444, "avg_output_tokens_per_task": 11623.267676767677, "last_run_timestamp": "2026-10-23T06:09:54.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Together", "accuracy": 0.901515, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.022678441919191918, "avg_duration_per_task_ms": 49322.41161616162, "avg_output_tokens_per_task": 18730.81818181818, "last_run_timestamp": "2026-10-24T08:41:23.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.901515, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.16319313383838385, "avg_duration_per_task_ms": 88415.74242424243, "avg_output_tokens_per_task": 9762.232323232323, "last_run_timestamp": "2026-10-01T19:12:25.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.9009352959183673, "accuracy_std_dev": 0.0002650000000000152, "run_count": 2, "total_questions": 392, "avg_cost_per_task_usd": 0.013316757653061225, "avg_duration_per_task_ms": 199818.51020408163, "avg_output_tokens_per_task": 22103.426020408162, "last_run_timestamp": "2026-10-24T08:16:43.075Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.9007668055555554, "accuracy_std_dev": 0.005776883838096259, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.21669425, "avg_duration_per_task_ms": 93977.99873737374, "avg_output_tokens_per_task": 7906.665684624018, "last_run_timestamp": "2026-10-24T08:14:03.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Google", "accuracy": 0.9006734999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.020602151515151518, "avg_duration_per_task_ms": 398490.0909090909, "avg_output_tokens_per_task": 16966.28787878788, "last_run_timestamp": "2026-10-24T08:44:49.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Together", "accuracy": 0.900673, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07405119191919192, "avg_duration_per_task_ms": 94171.00505050505, "avg_output_tokens_per_task": 17264.035353535353, "last_run_timestamp": "2026-10-24T08:14:51.112Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "GMICloud", "accuracy": 0.900673, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11481898484848484, "avg_duration_per_task_ms": 1609821.202020202, "avg_output_tokens_per_task": 45736.868686868685, "last_run_timestamp": "2026-10-27T09:00:04.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.9002912013585639, "accuracy_std_dev": 0.008613175564538058, "run_count": 12, "total_questions": 2061, "avg_cost_per_task_usd": 0.024704503153808833, "avg_duration_per_task_ms": 166852.90538573507, "avg_output_tokens_per_task": 20435.77389616691, "last_run_timestamp": "2026-10-24T16:26:43.118Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.8999997999999999, "accuracy_std_dev": 0.0027351667883330276, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03378601313131313, "avg_duration_per_task_ms": 836485.5171717171, "avg_output_tokens_per_task": 38467.88484848485, "last_run_timestamp": "2026-10-24T08:37:44.264Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.8999869065420562, "accuracy_std_dev": 0.0055303889812583175, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.012645620327102805, "avg_duration_per_task_ms": 176614.65965732088, "avg_output_tokens_per_task": 26316.880841121496, "last_run_timestamp": "2026-10-24T08:45:10.619Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Crusoe", "accuracy": 0.899832, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00798121212121212, "avg_duration_per_task_ms": 376079.0606060606, "avg_output_tokens_per_task": 28057.621212121212, "last_run_timestamp": "2026-10-26T03:00:10.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.8998315555555554, "accuracy_std_dev": 0.004532988273904737, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.05740673849607183, "avg_duration_per_task_ms": 25170.2278338945, "avg_output_tokens_per_task": 2482.2177328843995, "last_run_timestamp": "2026-10-24T08:41:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.8998315, "accuracy_std_dev": 0.007575499999999957, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02710091414141414, "avg_duration_per_task_ms": 156920.601010101, "avg_output_tokens_per_task": 22433.31818181818, "last_run_timestamp": "2026-10-24T08:34:01.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Cohere", "accuracy": 0.8998315, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11557131818181818, "avg_duration_per_task_ms": 1030735.8762626263, "avg_output_tokens_per_task": 45994.38383838384, "last_run_timestamp": "2026-10-24T08:14:03.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.8995506666666668, "accuracy_std_dev": 0.005722799567422267, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11900113804713805, "avg_duration_per_task_ms": 846691.8215488215, "avg_output_tokens_per_task": 47399.26936026936, "last_run_timestamp": "2026-10-24T08:02:41.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8994372877637131, "accuracy_std_dev": 0.01977995405637288, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.03544181097046414, "avg_duration_per_task_ms": 152645.21687763714, "avg_output_tokens_per_task": 26597.288607594935, "last_run_timestamp": "2026-10-03T09:28:39.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8993638888888889, "accuracy_std_dev": 0.004257353623619148, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.058306676767676764, "avg_duration_per_task_ms": 37477.73063973064, "avg_output_tokens_per_task": 2520.0830527497196, "last_run_timestamp": "2026-10-24T08:44:59.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8993638888888887, "accuracy_std_dev": 0.004029367042778795, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.030411617283950615, "avg_duration_per_task_ms": 237553.6509539843, "avg_output_tokens_per_task": 26633.1290684624, "last_run_timestamp": "2026-10-03T10:19:34.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.8993264, "accuracy_std_dev": 0.006511664598242117, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13052546969696968, "avg_duration_per_task_ms": 30874.333333333332, "avg_output_tokens_per_task": 2403.130303030303, "last_run_timestamp": "2026-10-01T08:24:30.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.8992834083438684, "accuracy_std_dev": 0.01959300954191881, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.12703601896333752, "avg_duration_per_task_ms": 516266.79730299197, "avg_output_tokens_per_task": 38334.854193004634, "last_run_timestamp": "2026-10-24T08:37:37.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.8991558025316456, "accuracy_std_dev": 0.04033796787732645, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.09046152742616034, "avg_duration_per_task_ms": 742340.6286919832, "avg_output_tokens_per_task": 35318.2, "last_run_timestamp": "2026-10-24T16:21:36.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.89913, "accuracy_std_dev": 0.005890503218458193, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.05769051388888888, "avg_duration_per_task_ms": 26119.287037037036, "avg_output_tokens_per_task": 2529.9574915824915, "last_run_timestamp": "2026-10-24T08:24:00.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "BaseTen", "accuracy": 0.8989899999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07316585858585858, "avg_duration_per_task_ms": 97804.9696969697, "avg_output_tokens_per_task": 16993.328282828283, "last_run_timestamp": "2026-10-24T08:12:37.563Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.8989897837837839, "accuracy_std_dev": 0.006188632325591686, "run_count": 37, "total_questions": 7326, "avg_cost_per_task_usd": 0.22242387441987438, "avg_duration_per_task_ms": 94205.91345891346, "avg_output_tokens_per_task": 8109.393939393939, "last_run_timestamp": "2026-10-24T08:20:25.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.8986389583333334, "accuracy_std_dev": 0.007699096453909172, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.12657329903198655, "avg_duration_per_task_ms": 58006.606060606064, "avg_output_tokens_per_task": 4727.50441919192, "last_run_timestamp": "2026-10-24T08:33:28.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna", "model_slug": "openai/gpt-6-luna", "model_name": "GPT-6-Luna", "provider_name": "auto-routing", "accuracy": 0.898477, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.003442197969543147, "avg_duration_per_task_ms": 73312.30456852792, "avg_output_tokens_per_task": 6831.9390862944165, "last_run_timestamp": "2026-10-08T13:29:57.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.8984635624999999, "accuracy_std_dev": 0.01886644038355125, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.07989067045454545, "avg_duration_per_task_ms": 218000.37657828283, "avg_output_tokens_per_task": 30348.85164141414, "last_run_timestamp": "2026-10-28T05:05:35.330Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.8983774545454545, "accuracy_std_dev": 0.004829955304962313, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.2299064136822773, "avg_duration_per_task_ms": 110399.28466483012, "avg_output_tokens_per_task": 8983.414600550965, "last_run_timestamp": "2026-10-24T08:27:26.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.8983162, "accuracy_std_dev": 0.008080975000580047, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1253350707070707, "avg_duration_per_task_ms": 994044.4858585858, "avg_output_tokens_per_task": 49876.97171717172, "last_run_timestamp": "2026-10-24T08:12:02.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8981654060721064, "accuracy_std_dev": 0.006690047440041041, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.01891549778621126, "avg_duration_per_task_ms": 414943.30676786846, "avg_output_tokens_per_task": 22047.87349778621, "last_run_timestamp": "2026-10-24T09:25:31.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.898148, "accuracy_std_dev": 0.012012972737836369, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03667170117845118, "avg_duration_per_task_ms": 157797.89814814815, "avg_output_tokens_per_task": 27519.53787878788, "last_run_timestamp": "2026-10-03T09:28:24.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.8977871428571428, "accuracy_std_dev": 0.005599042607665047, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.15678494083694083, "avg_duration_per_task_ms": 138524.9898989899, "avg_output_tokens_per_task": 9631.916305916306, "last_run_timestamp": "2026-10-01T19:18:24.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8977654545454545, "accuracy_std_dev": 0.00783374636212555, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.19846787741046834, "avg_duration_per_task_ms": 95325.20982552801, "avg_output_tokens_per_task": 7728.792011019284, "last_run_timestamp": "2026-10-24T08:17:56.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.8975652307692306, "accuracy_std_dev": 0.01420604709353269, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.004205098290598292, "avg_duration_per_task_ms": 359286.8714063714, "avg_output_tokens_per_task": 23062.278943278943, "last_run_timestamp": "2026-10-24T08:41:47.029Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8973063333333333, "accuracy_std_dev": 0.004955863351178606, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03703221212121212, "avg_duration_per_task_ms": 137593.69696969696, "avg_output_tokens_per_task": 26012.949494949495, "last_run_timestamp": "2026-10-02T23:26:10.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.8973062499999999, "accuracy_std_dev": 0.01842900724458137, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.030114116582491583, "avg_duration_per_task_ms": 249874.79587542088, "avg_output_tokens_per_task": 27231.26473063973, "last_run_timestamp": "2026-10-24T08:56:13.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.8972558314475225, "accuracy_std_dev": 0.033281996803894424, "run_count": 13, "total_questions": 2563, "avg_cost_per_task_usd": 0.1455034553257901, "avg_duration_per_task_ms": 811067.1357783847, "avg_output_tokens_per_task": 36442.67655091689, "last_run_timestamp": "2026-10-24T08:25:27.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.8971767692307694, "accuracy_std_dev": 0.016248030982026696, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.019287930069930074, "avg_duration_per_task_ms": 202055.53224553223, "avg_output_tokens_per_task": 25397.89898989899, "last_run_timestamp": "2026-10-27T00:43:30.287Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.8971659166666668, "accuracy_std_dev": 0.011261007344359652, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.19972315109427613, "avg_duration_per_task_ms": 95894.5180976431, "avg_output_tokens_per_task": 7770.697390572391, "last_run_timestamp": "2026-10-24T08:28:56.790Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.8970973821999613, "accuracy_std_dev": 0.12179084265551246, "run_count": 28, "total_questions": 5191, "avg_cost_per_task_usd": 0.1641544681178963, "avg_duration_per_task_ms": 204815.23540743595, "avg_output_tokens_per_task": 10801.00500866885, "last_run_timestamp": "2026-10-29T22:49:50.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.8965847857142858, "accuracy_std_dev": 0.006695321321410832, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.20728678535353537, "avg_duration_per_task_ms": 100509.16955266955, "avg_output_tokens_per_task": 8077.520562770563, "last_run_timestamp": "2026-10-24T08:30:35.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "CoreWeave", "accuracy": 0.8964644999999999, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.027913444444444446, "avg_duration_per_task_ms": 54299.99747474748, "avg_output_tokens_per_task": 13871.598484848484, "last_run_timestamp": "2026-10-24T08:12:49.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.8962877525832377, "accuracy_std_dev": 0.04356588502231736, "run_count": 9, "total_questions": 1742, "avg_cost_per_task_usd": 0.04917616016073478, "avg_duration_per_task_ms": 485084.33180252585, "avg_output_tokens_per_task": 15776.056831228472, "last_run_timestamp": "2026-10-24T16:26:47.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Nebius", "accuracy": 0.8960434999999999, "accuracy_std_dev": 0.007996499999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06795408585858587, "avg_duration_per_task_ms": 278542.00757575757, "avg_output_tokens_per_task": 15808.588383838383, "last_run_timestamp": "2026-10-24T08:15:15.302Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.8959899189640768, "accuracy_std_dev": 0.02896400676270464, "run_count": 8, "total_questions": 1197, "avg_cost_per_task_usd": 0.024112938178780283, "avg_duration_per_task_ms": 783093.3592314118, "avg_output_tokens_per_task": 22333.22807017544, "last_run_timestamp": "2026-10-17T16:03:42.308Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.8958047837837839, "accuracy_std_dev": 0.007786697038158158, "run_count": 37, "total_questions": 7326, "avg_cost_per_task_usd": 0.20536014018564022, "avg_duration_per_task_ms": 70550.94512694512, "avg_output_tokens_per_task": 7473.251979251979, "last_run_timestamp": "2026-10-24T08:43:08.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "OpenInference", "accuracy": 0.895623, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014240813131313132, "avg_duration_per_task_ms": 199874.40656565657, "avg_output_tokens_per_task": 21243.522727272728, "last_run_timestamp": "2026-10-24T08:54:21.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.895623, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03841230555555556, "avg_duration_per_task_ms": 144059.0782828283, "avg_output_tokens_per_task": 27039.489898989897, "last_run_timestamp": "2026-10-01T16:12:34.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Crusoe", "accuracy": 0.8956225, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026159166666666667, "avg_duration_per_task_ms": 177167.38888888888, "avg_output_tokens_per_task": 21649.300505050505, "last_run_timestamp": "2026-10-24T08:22:00.564Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8953421666666667, "accuracy_std_dev": 0.0047035083153133565, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05463887542087542, "avg_duration_per_task_ms": 31241.656565656565, "avg_output_tokens_per_task": 2527.686026936027, "last_run_timestamp": "2026-10-24T08:08:54.167Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.8952485555555553, "accuracy_std_dev": 0.018860205708263727, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02531761784511784, "avg_duration_per_task_ms": 163044.16722783388, "avg_output_tokens_per_task": 28644.837822671157, "last_run_timestamp": "2026-10-27T00:41:39.750Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.8952178253164556, "accuracy_std_dev": 0.009821681674220657, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.01650155105485232, "avg_duration_per_task_ms": 347394.32742616034, "avg_output_tokens_per_task": 28453.445147679326, "last_run_timestamp": "2026-10-24T08:49:27.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.8952138875490746, "accuracy_std_dev": 0.03543317040246446, "run_count": 19, "total_questions": 3566, "avg_cost_per_task_usd": 0.3690802299495232, "avg_duration_per_task_ms": 494589.1385305665, "avg_output_tokens_per_task": 57144.12535053281, "last_run_timestamp": "2026-10-29T22:54:33.936Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.8950615, "accuracy_std_dev": 0.01045325149973282, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008697955387205388, "avg_duration_per_task_ms": 170461.9739057239, "avg_output_tokens_per_task": 30584.97053872054, "last_run_timestamp": "2026-10-24T08:08:54.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.8949211666666667, "accuracy_std_dev": 0.007698231721563628, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.12088044486531986, "avg_duration_per_task_ms": 58308.99410774411, "avg_output_tokens_per_task": 4615.2697811447815, "last_run_timestamp": "2026-10-24T08:16:38.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "BaseTen", "accuracy": 0.8947810000000002, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03846228535353535, "avg_duration_per_task_ms": 31568.290404040403, "avg_output_tokens_per_task": 13846.992424242424, "last_run_timestamp": "2026-10-24T08:14:03.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "StreamLake", "accuracy": 0.894781, "accuracy_std_dev": 0.007575999999999972, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.036287184343434344, "avg_duration_per_task_ms": 737979.1590909091, "avg_output_tokens_per_task": 37423.542929292926, "last_run_timestamp": "2026-10-24T08:21:38.159Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8946608571428573, "accuracy_std_dev": 0.021330086138923062, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.025403829725829725, "avg_duration_per_task_ms": 208889.86363636365, "avg_output_tokens_per_task": 27754.472582972583, "last_run_timestamp": "2026-10-24T08:01:41.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8946406666666666, "accuracy_std_dev": 0.009509966424172523, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.23073020496632995, "avg_duration_per_task_ms": 110250.86616161616, "avg_output_tokens_per_task": 9005.217171717171, "last_run_timestamp": "2026-10-24T08:44:41.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.8945726512842839, "accuracy_std_dev": 0.020487988184645325, "run_count": 12, "total_questions": 2297, "avg_cost_per_task_usd": 0.15815831606443187, "avg_duration_per_task_ms": 350172.93774488464, "avg_output_tokens_per_task": 39679.15150195908, "last_run_timestamp": "2026-10-26T03:00:10.340Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "ModelRun", "accuracy": 0.8943087542213883, "accuracy_std_dev": 0.03942204850926279, "run_count": 3, "total_questions": 533, "avg_cost_per_task_usd": 0.019450917448405252, "avg_duration_per_task_ms": 177927.2420262664, "avg_output_tokens_per_task": 28174.943714821762, "last_run_timestamp": "2026-10-28T05:00:14.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.8940067465753425, "accuracy_std_dev": 0.01621889745889719, "run_count": 15, "total_questions": 2920, "avg_cost_per_task_usd": 0.014883987671232876, "avg_duration_per_task_ms": 203624.06232876712, "avg_output_tokens_per_task": 20864.14212328767, "last_run_timestamp": "2026-10-29T22:27:58.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.8934709999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 194, "avg_cost_per_task_usd": 0.11686864948453608, "avg_duration_per_task_ms": 605687.7474226804, "avg_output_tokens_per_task": 46496.701030927834, "last_run_timestamp": "2026-10-13T03:00:07.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8932483811659193, "accuracy_std_dev": 0.017336749323389345, "run_count": 7, "total_questions": 1338, "avg_cost_per_task_usd": 0.03355175710014948, "avg_duration_per_task_ms": 240320.3542600897, "avg_output_tokens_per_task": 32642.40284005979, "last_run_timestamp": "2026-10-24T08:41:23.835Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8930975, "accuracy_std_dev": 0.010942499999999966, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04334855050505051, "avg_duration_per_task_ms": 40520.42929292929, "avg_output_tokens_per_task": 3151.881313131313, "last_run_timestamp": "2026-10-01T19:11:43.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.8930975, "accuracy_std_dev": 0.009560420192125432, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.11553578787878788, "avg_duration_per_task_ms": 625203.1957070708, "avg_output_tokens_per_task": 46042.48863636364, "last_run_timestamp": "2026-10-24T08:01:41.143Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8928570861678005, "accuracy_std_dev": 0.015340394005370288, "run_count": 5, "total_questions": 882, "avg_cost_per_task_usd": 0.12683532766439912, "avg_duration_per_task_ms": 285532.1031746032, "avg_output_tokens_per_task": 28366.77097505669, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8924427777777778, "accuracy_std_dev": 0.010300002662111597, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.005114002244668912, "avg_duration_per_task_ms": 223535.9657687991, "avg_output_tokens_per_task": 28038.338383838385, "last_run_timestamp": "2026-10-27T01:02:43.701Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.8922560000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0070467222222222224, "avg_duration_per_task_ms": 212650.5101010101, "avg_output_tokens_per_task": 24920.08080808081, "last_run_timestamp": "2026-08-08T10:06:09.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8922560000000002, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14220060606060608, "avg_duration_per_task_ms": 188559.2904040404, "avg_output_tokens_per_task": 47276.156565656565, "last_run_timestamp": "2026-10-01T19:12:30.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.8922557999999999, "accuracy_std_dev": 0.007299435934371887, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.11377392424242423, "avg_duration_per_task_ms": 848578.0848484848, "avg_output_tokens_per_task": 45335.65555555555, "last_run_timestamp": "2026-10-24T08:01:41.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.8920152857142859, "accuracy_std_dev": 0.011608927534005704, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.20755713311688312, "avg_duration_per_task_ms": 99648.18037518038, "avg_output_tokens_per_task": 8074.249278499278, "last_run_timestamp": "2026-10-24T08:05:46.408Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8919751666666667, "accuracy_std_dev": 0.008661714291768222, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024034642255892257, "avg_duration_per_task_ms": 245030.94696969696, "avg_output_tokens_per_task": 30463.32239057239, "last_run_timestamp": "2026-10-24T08:01:41.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8917748571428571, "accuracy_std_dev": 0.009721593908534165, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04607931240981241, "avg_duration_per_task_ms": 43786.40764790765, "avg_output_tokens_per_task": 3453.090909090909, "last_run_timestamp": "2026-10-01T19:13:06.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.8916709261744967, "accuracy_std_dev": 0.019973452544815567, "run_count": 10, "total_questions": 1937, "avg_cost_per_task_usd": 0.13092793236964376, "avg_duration_per_task_ms": 553822.8373773877, "avg_output_tokens_per_task": 42781.386164171396, "last_run_timestamp": "2026-10-24T09:11:22.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.8916244166666666, "accuracy_std_dev": 0.008865984904475566, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.24456361553030298, "avg_duration_per_task_ms": 85715.98253367003, "avg_output_tokens_per_task": 9091.301978114478, "last_run_timestamp": "2026-10-24T08:14:36.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.8915753430656934, "accuracy_std_dev": 0.004887565515332395, "run_count": 6, "total_questions": 1096, "avg_cost_per_task_usd": 0.03103391879562044, "avg_duration_per_task_ms": 795798.2390510949, "avg_output_tokens_per_task": 36089.42244525548, "last_run_timestamp": "2026-10-24T08:07:59.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.8915342857142857, "accuracy_std_dev": 0.010905197538189691, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011785490620490622, "avg_duration_per_task_ms": 284433.1883116883, "avg_output_tokens_per_task": 21375.483405483406, "last_run_timestamp": "2026-10-24T08:45:15.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.8914141, "accuracy_std_dev": 0.016939939187907378, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1456811595959596, "avg_duration_per_task_ms": 619546.0101010101, "avg_output_tokens_per_task": 36709.93131313131, "last_run_timestamp": "2026-10-17T08:18:00.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Novita", "accuracy": 0.891414, "accuracy_std_dev": 0.01094299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007142083333333334, "avg_duration_per_task_ms": 125847.88383838384, "avg_output_tokens_per_task": 11714.222222222223, "last_run_timestamp": "2026-10-24T08:16:38.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "DigitalOcean", "accuracy": 0.891414, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.016427454545454544, "avg_duration_per_task_ms": 96439.17424242424, "avg_output_tokens_per_task": 13618.247474747475, "last_run_timestamp": "2026-10-24T08:14:51.086Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Crusoe", "accuracy": 0.8912280000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 190, "avg_cost_per_task_usd": 0.14632182105263158, "avg_duration_per_task_ms": 676754.2315789474, "avg_output_tokens_per_task": 41671.152631578945, "last_run_timestamp": "2026-10-26T03:00:09.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.8907934361904761, "accuracy_std_dev": 0.020177999596946557, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.18863606476190478, "avg_duration_per_task_ms": 1541820.2831746032, "avg_output_tokens_per_task": 55365.57968253968, "last_run_timestamp": "2026-10-24T16:20:08.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.8906711574504008, "accuracy_std_dev": 0.010298496956136629, "run_count": 12, "total_questions": 2369, "avg_cost_per_task_usd": 0.007867603630223723, "avg_duration_per_task_ms": 337616.2794428029, "avg_output_tokens_per_task": 27764.332207682568, "last_run_timestamp": "2026-10-24T08:50:11.572Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "GMICloud", "accuracy": 0.8905723333333333, "accuracy_std_dev": 0.0076535129334327255, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007299473063973064, "avg_duration_per_task_ms": 131190.53535353535, "avg_output_tokens_per_task": 12389.143097643098, "last_run_timestamp": "2026-10-27T00:37:23.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8901177599344533, "accuracy_std_dev": 0.009279724700032873, "run_count": 37, "total_questions": 7323, "avg_cost_per_task_usd": 0.1335219325413082, "avg_duration_per_task_ms": 59352.6749965861, "avg_output_tokens_per_task": 4769.18080021849, "last_run_timestamp": "2026-10-24T08:16:43.073Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008230473063973063, "avg_duration_per_task_ms": 80128.80303030302, "avg_output_tokens_per_task": 12935.998316498317, "last_run_timestamp": "2026-10-02T23:23:43.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.8899192124789208, "accuracy_std_dev": 0.02293354363329505, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.06400783642495786, "avg_duration_per_task_ms": 535683.2737492974, "avg_output_tokens_per_task": 35982.14896008994, "last_run_timestamp": "2026-10-27T02:49:26.609Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.02853141346198763, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.136280962962963, "avg_duration_per_task_ms": 40331.19781144781, "avg_output_tokens_per_task": 2516.2390572390573, "last_run_timestamp": "2026-10-01T08:24:17.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.009896244159893304, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008765856902356902, "avg_duration_per_task_ms": 59664.589225589225, "avg_output_tokens_per_task": 12901.691919191919, "last_run_timestamp": "2026-10-01T08:29:53.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.8888890000000002, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0431288872053872, "avg_duration_per_task_ms": 52519.67676767677, "avg_output_tokens_per_task": 3463.8030303030305, "last_run_timestamp": "2026-10-02T23:21:06.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.888889, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1979060606060606, "avg_duration_per_task_ms": 95702.44696969698, "avg_output_tokens_per_task": 7691.454545454545, "last_run_timestamp": "2026-10-01T19:11:18.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8887654163757536, "accuracy_std_dev": 0.008192069777191544, "run_count": 16, "total_questions": 3151, "avg_cost_per_task_usd": 0.01559116534433513, "avg_duration_per_task_ms": 197619.02634084417, "avg_output_tokens_per_task": 26956.708981275784, "last_run_timestamp": "2026-10-24T08:24:10.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.8886482857142856, "accuracy_std_dev": 0.013807459233682208, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1021348318903319, "avg_duration_per_task_ms": 444900.012987013, "avg_output_tokens_per_task": 35591.61255411255, "last_run_timestamp": "2026-10-24T08:42:40.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.8885464319465846, "accuracy_std_dev": 0.029017071552622252, "run_count": 10, "total_questions": 1947, "avg_cost_per_task_usd": 0.009370765279917823, "avg_duration_per_task_ms": 312934.7123780175, "avg_output_tokens_per_task": 26808.334874165383, "last_run_timestamp": "2026-10-24T08:13:41.086Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.888468, "accuracy_std_dev": 0.008504715878068274, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.20666023274410775, "avg_duration_per_task_ms": 100029.38846801347, "avg_output_tokens_per_task": 8038.521464646465, "last_run_timestamp": "2026-10-24T08:44:29.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.888467875, "accuracy_std_dev": 0.02891189332021296, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10564124999999999, "avg_duration_per_task_ms": 446348.9172979798, "avg_output_tokens_per_task": 36610.40214646464, "last_run_timestamp": "2026-10-24T16:12:09.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Phala", "accuracy": 0.8880469999999999, "accuracy_std_dev": 0.01094299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02377390909090909, "avg_duration_per_task_ms": 131245.66919191918, "avg_output_tokens_per_task": 19627.237373737375, "last_run_timestamp": "2026-10-24T08:41:24.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8880229138638227, "accuracy_std_dev": 0.007263570665748784, "run_count": 7, "total_questions": 1219, "avg_cost_per_task_usd": 0.01426234372436423, "avg_duration_per_task_ms": 205415.2715340443, "avg_output_tokens_per_task": 24591.009844134536, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.8878528461538461, "accuracy_std_dev": 0.03528588596427789, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.11375538578088577, "avg_duration_per_task_ms": 500007.0135975136, "avg_output_tokens_per_task": 36427.52797202797, "last_run_timestamp": "2026-10-27T01:11:27.731Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.887537924012158, "accuracy_std_dev": 0.020250680116973837, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.12033997061803446, "avg_duration_per_task_ms": 580755.4113475177, "avg_output_tokens_per_task": 34357.5663627153, "last_run_timestamp": "2026-10-03T16:44:56.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8872051666666666, "accuracy_std_dev": 0.024761493125527695, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1434989882154882, "avg_duration_per_task_ms": 42121.643939393936, "avg_output_tokens_per_task": 2660.465488215488, "last_run_timestamp": "2026-10-01T08:24:34.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "AtlasCloud", "accuracy": 0.887205, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11313772222222222, "avg_duration_per_task_ms": 834184.0151515151, "avg_output_tokens_per_task": 45085.207070707074, "last_run_timestamp": "2026-10-27T03:00:05.728Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.887042338150289, "accuracy_std_dev": 0.010371066166434932, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.010374203757225434, "avg_duration_per_task_ms": 78284.89884393064, "avg_output_tokens_per_task": 7947.398121387283, "last_run_timestamp": "2026-10-01T19:12:42.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8869462307692308, "accuracy_std_dev": 0.01086717272801863, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.025130035353535356, "avg_duration_per_task_ms": 337121.5742035742, "avg_output_tokens_per_task": 26419.156565656565, "last_run_timestamp": "2026-10-24T08:11:11.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8869174161290322, "accuracy_std_dev": 0.06211553768895107, "run_count": 20, "total_questions": 3720, "avg_cost_per_task_usd": 0.00878794623655914, "avg_duration_per_task_ms": 150880.02284946237, "avg_output_tokens_per_task": 24534.02741935484, "last_run_timestamp": "2026-10-29T22:36:27.977Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.8867541405460061, "accuracy_std_dev": 0.003543823280018365, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.03347630637007078, "avg_duration_per_task_ms": 767457.7462082912, "avg_output_tokens_per_task": 38389.49544994944, "last_run_timestamp": "2026-10-24T08:26:00.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8867506138022562, "accuracy_std_dev": 0.020605395430793746, "run_count": 8, "total_questions": 1507, "avg_cost_per_task_usd": 0.1588602813536828, "avg_duration_per_task_ms": 705608.4233576643, "avg_output_tokens_per_task": 51029.565361645655, "last_run_timestamp": "2026-10-24T16:27:11.137Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.8867197848605577, "accuracy_std_dev": 0.007952474956944445, "run_count": 13, "total_questions": 2510, "avg_cost_per_task_usd": 0.09700779960159363, "avg_duration_per_task_ms": 461798.00398406375, "avg_output_tokens_per_task": 28989.141832669324, "last_run_timestamp": "2026-10-24T16:06:30.904Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.8866872307692307, "accuracy_std_dev": 0.01902814900716969, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.027383982128982128, "avg_duration_per_task_ms": 210380.49145299144, "avg_output_tokens_per_task": 25687.588966588966, "last_run_timestamp": "2026-10-24T08:19:41.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.8863635000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.22157512626262627, "avg_duration_per_task_ms": 106335.13383838384, "avg_output_tokens_per_task": 8638.217171717171, "last_run_timestamp": "2026-10-01T19:10:42.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.8861492509064539, "accuracy_std_dev": 0.01420488521313591, "run_count": 7, "total_questions": 1379, "avg_cost_per_task_usd": 0.12877551051486583, "avg_duration_per_task_ms": 1724556.7200870195, "avg_output_tokens_per_task": 32019.171863669326, "last_run_timestamp": "2026-10-24T16:28:09.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8859760038834952, "accuracy_std_dev": 0.023393307207604786, "run_count": 8, "total_questions": 1545, "avg_cost_per_task_usd": 0.02020688673139159, "avg_duration_per_task_ms": 264412.80194174754, "avg_output_tokens_per_task": 30725.21488673139, "last_run_timestamp": "2026-10-24T09:10:08.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.885662, "accuracy_std_dev": 0.014374902133000174, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.11714179755892257, "avg_duration_per_task_ms": 572173.5332491583, "avg_output_tokens_per_task": 30582.406986531987, "last_run_timestamp": "2026-10-24T16:30:32.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.8856471629778674, "accuracy_std_dev": 0.009013068317726206, "run_count": 8, "total_questions": 1491, "avg_cost_per_task_usd": 0.010023977867203217, "avg_duration_per_task_ms": 270739.96177062375, "avg_output_tokens_per_task": 28220.264922870556, "last_run_timestamp": "2026-10-24T09:01:43.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-max", "model_slug": "sakana/fugu-max", "model_name": "Fugu Max", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07555469191919191, "avg_duration_per_task_ms": 225091.94949494948, "avg_output_tokens_per_task": 12279.550505050505, "last_run_timestamp": "2026-09-15T01:07:47.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Minimax", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007081343434343435, "avg_duration_per_task_ms": 121564.28282828283, "avg_output_tokens_per_task": 11603.333333333334, "last_run_timestamp": "2026-10-24T08:10:23.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Nebius", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.05586563636363637, "avg_duration_per_task_ms": 471872.54545454547, "avg_output_tokens_per_task": 61805.67676767677, "last_run_timestamp": "2026-10-27T00:00:04.640Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02146120202020202, "avg_duration_per_task_ms": 674351.9393939395, "avg_output_tokens_per_task": 45464.17676767677, "last_run_timestamp": "2026-10-01T08:31:57.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.8850689656218401, "accuracy_std_dev": 0.014411932672615443, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.026149528816986855, "avg_duration_per_task_ms": 355797.47320525785, "avg_output_tokens_per_task": 26995.41152679474, "last_run_timestamp": "2026-10-10T08:22:45.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.884890375, "accuracy_std_dev": 0.021787089743340537, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.22976492803030302, "avg_duration_per_task_ms": 949653.6900252525, "avg_output_tokens_per_task": 57582.06123737374, "last_run_timestamp": "2026-10-24T16:07:10.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Together", "accuracy": 0.8846800000000001, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01685389393939394, "avg_duration_per_task_ms": 60063.59848484849, "avg_output_tokens_per_task": 13866.29292929293, "last_run_timestamp": "2026-10-24T08:11:43.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.88468, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1319599772727273, "avg_duration_per_task_ms": 249775.19949494948, "avg_output_tokens_per_task": 43863.606060606064, "last_run_timestamp": "2026-10-01T19:15:31.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.8845502920892495, "accuracy_std_dev": 0.01776525517520085, "run_count": 5, "total_questions": 986, "avg_cost_per_task_usd": 0.040126028397565926, "avg_duration_per_task_ms": 169902.08620689655, "avg_output_tokens_per_task": 33271.34381338742, "last_run_timestamp": "2026-10-29T23:12:01.788Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8843169242648614, "accuracy_std_dev": 0.01377803394592024, "run_count": 24, "total_questions": 4727, "avg_cost_per_task_usd": 0.19024729342077426, "avg_duration_per_task_ms": 371375.61032367253, "avg_output_tokens_per_task": 31453.42183202877, "last_run_timestamp": "2026-10-24T16:13:46.475Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04134767424242424, "avg_duration_per_task_ms": 54936.305555555555, "avg_output_tokens_per_task": 3316.282828282828, "last_run_timestamp": "2026-10-01T19:13:07.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008301368686868687, "avg_duration_per_task_ms": 72985.27777777778, "avg_output_tokens_per_task": 13098.969696969696, "last_run_timestamp": "2026-10-01T08:24:38.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014282979797979798, "avg_duration_per_task_ms": 55075.11111111111, "avg_output_tokens_per_task": 13355.585858585859, "last_run_timestamp": "2026-10-24T08:02:32.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.8838218675352877, "accuracy_std_dev": 0.0032754605783004037, "run_count": 5, "total_questions": 921, "avg_cost_per_task_usd": 0.10965064820846905, "avg_duration_per_task_ms": 399019.9750271444, "avg_output_tokens_per_task": 36828.9565689468, "last_run_timestamp": "2026-10-24T16:03:25.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.88341725, "accuracy_std_dev": 0.003834202809907171, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03130394823232323, "avg_duration_per_task_ms": 201934.6616161616, "avg_output_tokens_per_task": 25861.328282828283, "last_run_timestamp": "2026-10-24T08:53:53.931Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.8833333472727273, "accuracy_std_dev": 0.012158692807021237, "run_count": 7, "total_questions": 1100, "avg_cost_per_task_usd": 0.007285390000000001, "avg_duration_per_task_ms": 942880.7490909091, "avg_output_tokens_per_task": 28917.175454545453, "last_run_timestamp": "2026-10-17T08:19:36.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.8828761428571429, "accuracy_std_dev": 0.004395219722398795, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.029912125541125538, "avg_duration_per_task_ms": 82923.05122655122, "avg_output_tokens_per_task": 24775.42784992785, "last_run_timestamp": "2026-10-24T08:49:27.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.8828537731629394, "accuracy_std_dev": 0.008622317150279293, "run_count": 5, "total_questions": 939, "avg_cost_per_task_usd": 0.006383692225772098, "avg_duration_per_task_ms": 468696.3525026624, "avg_output_tokens_per_task": 35072.061767838124, "last_run_timestamp": "2026-10-24T08:44:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.882503491800857, "accuracy_std_dev": 0.009500768601346247, "run_count": 35, "total_questions": 6769, "avg_cost_per_task_usd": 0.14009028970305806, "avg_duration_per_task_ms": 63158.250849460775, "avg_output_tokens_per_task": 5048.065888609839, "last_run_timestamp": "2026-10-24T08:22:55.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.882356599609375, "accuracy_std_dev": 0.02350065555877767, "run_count": 13, "total_questions": 2560, "avg_cost_per_task_usd": 0.0909012125, "avg_duration_per_task_ms": 446817.64765625, "avg_output_tokens_per_task": 32851.274609375, "last_run_timestamp": "2026-10-28T04:23:20.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "ModelRun", "accuracy": 0.8821550000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.026309151515151515, "avg_duration_per_task_ms": 202247.18686868687, "avg_output_tokens_per_task": 39119.5505050505, "last_run_timestamp": "2026-10-29T03:00:03.996Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.8821550000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.129083601010101, "avg_duration_per_task_ms": 388751.6590909091, "avg_output_tokens_per_task": 32112.176767676767, "last_run_timestamp": "2026-10-17T16:07:32.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.88215475, "accuracy_std_dev": 0.018821912135527027, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008138106691919191, "avg_duration_per_task_ms": 380823.72411616164, "avg_output_tokens_per_task": 28681.33712121212, "last_run_timestamp": "2026-10-24T16:29:33.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.882034357142857, "accuracy_std_dev": 0.011301592736090304, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.006003468614718615, "avg_duration_per_task_ms": 401285.20815295813, "avg_output_tokens_per_task": 29705.93326118326, "last_run_timestamp": "2026-10-24T08:46:33.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8818485454545456, "accuracy_std_dev": 0.01327175411002725, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.18638605096418728, "avg_duration_per_task_ms": 321116.0505050505, "avg_output_tokens_per_task": 46783.020661157025, "last_run_timestamp": "2026-10-24T08:02:27.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.8817956985294118, "accuracy_std_dev": 0.012032407447571536, "run_count": 8, "total_questions": 1496, "avg_cost_per_task_usd": 0.06691362366310159, "avg_duration_per_task_ms": 318081.7332887701, "avg_output_tokens_per_task": 18489.88435828877, "last_run_timestamp": "2026-10-24T16:08:29.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.8816240000000001, "accuracy_std_dev": 0.004701000000000011, "run_count": 2, "total_questions": 390, "avg_cost_per_task_usd": 0.020361825641025642, "avg_duration_per_task_ms": 90976.65641025642, "avg_output_tokens_per_task": 40519.597435897434, "last_run_timestamp": "2026-10-24T08:02:32.160Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.8815936666666665, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002981575757575758, "avg_duration_per_task_ms": 52993.44276094276, "avg_output_tokens_per_task": 5804.632996632997, "last_run_timestamp": "2026-10-02T23:20:31.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.8814814000000001, "accuracy_std_dev": 0.011789297597397391, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007892147474747474, "avg_duration_per_task_ms": 236163.43737373737, "avg_output_tokens_per_task": 27867.079797979797, "last_run_timestamp": "2026-10-03T16:57:09.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.8814552816901408, "accuracy_std_dev": 0.00965776820360461, "run_count": 13, "total_questions": 2556, "avg_cost_per_task_usd": 0.00679829303599374, "avg_duration_per_task_ms": 375447.8978873239, "avg_output_tokens_per_task": 19618.11854460094, "last_run_timestamp": "2026-10-24T08:46:28.764Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "BaseTen", "accuracy": 0.881313, "accuracy_std_dev": 0.012626000000000026, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.022825318181818183, "avg_duration_per_task_ms": 147723.58080808082, "avg_output_tokens_per_task": 45335.5101010101, "last_run_timestamp": "2026-10-24T08:12:49.570Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.8813128749999999, "accuracy_std_dev": 0.02861962867700026, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.19908026893939393, "avg_duration_per_task_ms": 958354.9299242424, "avg_output_tokens_per_task": 58904.72348484849, "last_run_timestamp": "2026-10-24T16:17:57.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8811927142857143, "accuracy_std_dev": 0.007888829783113527, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015891973304473304, "avg_duration_per_task_ms": 279407.10173160175, "avg_output_tokens_per_task": 24276.554834054834, "last_run_timestamp": "2026-10-08T22:18:51.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.8808292124352332, "accuracy_std_dev": 0.0020954999999999724, "run_count": 2, "total_questions": 386, "avg_cost_per_task_usd": 0.03105037564766839, "avg_duration_per_task_ms": 282019.7538860104, "avg_output_tokens_per_task": 61860.47668393782, "last_run_timestamp": "2026-10-24T08:07:42.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.8807385965665235, "accuracy_std_dev": 0.04941426514861386, "run_count": 13, "total_questions": 2563, "avg_cost_per_task_usd": 0.0748979442060086, "avg_duration_per_task_ms": 557063.2532188841, "avg_output_tokens_per_task": 39438.98985563793, "last_run_timestamp": "2026-10-24T16:40:43.534Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.8805647777777778, "accuracy_std_dev": 0.0331443107562395, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.14604359764309763, "avg_duration_per_task_ms": 370818.19837261504, "avg_output_tokens_per_task": 10967.793209876543, "last_run_timestamp": "2026-10-24T08:22:00.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.880471, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0032475000000000004, "avg_duration_per_task_ms": 58949.292929292926, "avg_output_tokens_per_task": 5746.025252525253, "last_run_timestamp": "2026-09-23T06:09:34.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8801906666666667, "accuracy_std_dev": 0.011226983784119793, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0227881835016835, "avg_duration_per_task_ms": 234700.68434343435, "avg_output_tokens_per_task": 25264.190235690236, "last_run_timestamp": "2026-10-03T09:30:18.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.8800047366136035, "accuracy_std_dev": 0.007207181987717666, "run_count": 7, "total_questions": 1382, "avg_cost_per_task_usd": 0.03244589001447178, "avg_duration_per_task_ms": 148073.17149059335, "avg_output_tokens_per_task": 26851.936324167873, "last_run_timestamp": "2026-10-24T08:26:33.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "StreamLake", "accuracy": 0.879965, "accuracy_std_dev": null, "run_count": 1, "total_questions": 193, "avg_cost_per_task_usd": 0.006318481865284975, "avg_duration_per_task_ms": 107877.19170984456, "avg_output_tokens_per_task": 10411.435233160622, "last_run_timestamp": "2026-10-27T15:00:04.487Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8798644568527919, "accuracy_std_dev": 0.004527248931623837, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.0029938477157360403, "avg_duration_per_task_ms": 316818.90016920475, "avg_output_tokens_per_task": 18321.88663282572, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto", "model_slug": "openrouter/auto", "model_name": "Auto Router", "provider_name": "auto-routing", "accuracy": 0.8795957999999999, "accuracy_std_dev": 0.026351833738091142, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08917041111111111, "avg_duration_per_task_ms": 196788.44343434344, "avg_output_tokens_per_task": 37445.42121212121, "last_run_timestamp": "2026-09-29T14:28:17.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8794892499999999, "accuracy_std_dev": 0.0183834671245887, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.12937238047138044, "avg_duration_per_task_ms": 63385.43771043771, "avg_output_tokens_per_task": 4942.099747474747, "last_run_timestamp": "2026-10-24T08:27:56.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "OpenInference", "accuracy": 0.879386, "accuracy_std_dev": null, "run_count": 1, "total_questions": 76, "avg_cost_per_task_usd": 0.03120878947368421, "avg_duration_per_task_ms": 274583.63157894736, "avg_output_tokens_per_task": 9741.526315789473, "last_run_timestamp": "2026-10-09T00:00:06.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.879370525058275, "accuracy_std_dev": 0.02189493848211963, "run_count": 9, "total_questions": 1716, "avg_cost_per_task_usd": 0.09564452447552448, "avg_duration_per_task_ms": 551582.8269230769, "avg_output_tokens_per_task": 28335.815268065267, "last_run_timestamp": "2026-10-24T08:13:46.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.8792376607706888, "accuracy_std_dev": 0.006337813294770935, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.016976666456096023, "avg_duration_per_task_ms": 306778.0650663297, "avg_output_tokens_per_task": 30476.99052432091, "last_run_timestamp": "2026-10-24T08:43:17.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.8790282857142857, "accuracy_std_dev": 0.019161523885971404, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03542920851370852, "avg_duration_per_task_ms": 140156.7987012987, "avg_output_tokens_per_task": 27224.489898989897, "last_run_timestamp": "2026-10-24T08:28:56.672Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.8788318459807073, "accuracy_std_dev": 0.05618657197923134, "run_count": 16, "total_questions": 3110, "avg_cost_per_task_usd": 0.010130650482315113, "avg_duration_per_task_ms": 433648.34083601285, "avg_output_tokens_per_task": 25902.576527331188, "last_run_timestamp": "2026-10-29T22:45:55.902Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8787880000000001, "accuracy_std_dev": 0.007273551310513108, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.009556249158249158, "avg_duration_per_task_ms": 52070.99494949495, "avg_output_tokens_per_task": 7331.292929292929, "last_run_timestamp": "2026-10-30T00:00:06.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "Deepseek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.010643959595959596, "avg_duration_per_task_ms": 127627.76767676767, "avg_output_tokens_per_task": 14277.949494949495, "last_run_timestamp": "2026-10-08T13:29:57.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Mimo V2.6 Pro", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01249009595959596, "avg_duration_per_task_ms": 315799.5808080808, "avg_output_tokens_per_task": 14226.873737373737, "last_run_timestamp": "2026-10-08T01:55:44.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.043995565656565656, "avg_duration_per_task_ms": 588519.7272727273, "avg_output_tokens_per_task": 34166.82828282828, "last_run_timestamp": "2026-10-01T08:47:14.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.8787668916494136, "accuracy_std_dev": 0.03281417517628624, "run_count": 8, "total_questions": 1449, "avg_cost_per_task_usd": 0.2005868881987578, "avg_duration_per_task_ms": 1332066.3747412008, "avg_output_tokens_per_task": 50003.33195307109, "last_run_timestamp": "2026-10-24T16:26:57.307Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.8786261545293074, "accuracy_std_dev": 0.0068412882173039926, "run_count": 6, "total_questions": 1126, "avg_cost_per_task_usd": 0.11316016607460035, "avg_duration_per_task_ms": 476858.84191829484, "avg_output_tokens_per_task": 32318.380994671403, "last_run_timestamp": "2026-10-17T16:07:16.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.87780575, "accuracy_std_dev": 0.016897481104812654, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1286623606902357, "avg_duration_per_task_ms": 62852.628367003366, "avg_output_tokens_per_task": 4923.856481481482, "last_run_timestamp": "2026-10-24T08:15:51.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.8776513109987357, "accuracy_std_dev": 0.007333672175410435, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.11767066118836916, "avg_duration_per_task_ms": 752365.9431099873, "avg_output_tokens_per_task": 30843.225874420565, "last_run_timestamp": "2026-10-24T16:07:46.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.8775252499999999, "accuracy_std_dev": 0.03188403682128148, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.01210295664983165, "avg_duration_per_task_ms": 348579.569023569, "avg_output_tokens_per_task": 22092.35732323232, "last_run_timestamp": "2026-10-24T08:45:03.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.8774407999999999, "accuracy_std_dev": 0.00954717737134911, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03921924343434343, "avg_duration_per_task_ms": 242470.04242424242, "avg_output_tokens_per_task": 37354.009090909094, "last_run_timestamp": "2026-10-21T11:23:15.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.8773447142857143, "accuracy_std_dev": 0.012707857195050472, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03401366883116883, "avg_duration_per_task_ms": 154330.79004329004, "avg_output_tokens_per_task": 28188.37950937951, "last_run_timestamp": "2026-10-24T09:09:44.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.8772070529695024, "accuracy_std_dev": 0.03893659390197737, "run_count": 4, "total_questions": 623, "avg_cost_per_task_usd": 0.09425666452648475, "avg_duration_per_task_ms": 461375.2568218299, "avg_output_tokens_per_task": 23922.329052969504, "last_run_timestamp": "2026-09-30T11:37:23.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.8771043333333335, "accuracy_std_dev": 0.007896528133018712, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003143374579124579, "avg_duration_per_task_ms": 35450.33585858586, "avg_output_tokens_per_task": 5733.669191919192, "last_run_timestamp": "2026-10-01T08:25:15.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.877104, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07509321212121212, "avg_duration_per_task_ms": 263175.32828282827, "avg_output_tokens_per_task": 20738.459595959597, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.8770437482014388, "accuracy_std_dev": 0.016843957465790968, "run_count": 8, "total_questions": 1529, "avg_cost_per_task_usd": 0.1855886553302812, "avg_duration_per_task_ms": 777107.6200130804, "avg_output_tokens_per_task": 53473.10595160235, "last_run_timestamp": "2026-10-24T16:28:49.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.876999, "accuracy_std_dev": 0.06406231714705611, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1307801066919192, "avg_duration_per_task_ms": 1601351.4103535353, "avg_output_tokens_per_task": 55537.77272727273, "last_run_timestamp": "2026-10-24T16:02:47.376Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.8769172222222222, "accuracy_std_dev": 0.01657946057812884, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02028404994388328, "avg_duration_per_task_ms": 725618.4191919192, "avg_output_tokens_per_task": 35595.71436588103, "last_run_timestamp": "2026-10-24T16:27:17.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.876893965909091, "accuracy_std_dev": 0.02910034944176192, "run_count": 7, "total_questions": 1232, "avg_cost_per_task_usd": 0.09717019561688313, "avg_duration_per_task_ms": 193376.24837662338, "avg_output_tokens_per_task": 26307.616071428572, "last_run_timestamp": "2026-09-26T09:19:13.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.8767675999999999, "accuracy_std_dev": 0.014101374310328764, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.042260786868686874, "avg_duration_per_task_ms": 210147.5181818182, "avg_output_tokens_per_task": 35072.08888888889, "last_run_timestamp": "2026-10-24T08:50:45.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8765966933797908, "accuracy_std_dev": 0.020189124617118887, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.004125730836236933, "avg_duration_per_task_ms": 453615.28397212544, "avg_output_tokens_per_task": 16054.564459930314, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8765431666666668, "accuracy_std_dev": 0.03217293038884639, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11818556060606061, "avg_duration_per_task_ms": 28260.9031986532, "avg_output_tokens_per_task": 2161.995791245791, "last_run_timestamp": "2026-10-01T08:25:56.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.8765007667238421, "accuracy_std_dev": 0.009087011145341243, "run_count": 3, "total_questions": 583, "avg_cost_per_task_usd": 0.09344886277873071, "avg_duration_per_task_ms": 568169.3550600343, "avg_output_tokens_per_task": 27396.61578044597, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.8763533692307693, "accuracy_std_dev": 0.006821595577445394, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.027450420512820508, "avg_duration_per_task_ms": 188376.252991453, "avg_output_tokens_per_task": 30391.492307692308, "last_run_timestamp": "2026-10-23T06:17:23.826Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8762625, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009382047979797978, "avg_duration_per_task_ms": 83118.84595959596, "avg_output_tokens_per_task": 7689.015151515152, "last_run_timestamp": "2026-10-01T19:14:22.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.8762115941845765, "accuracy_std_dev": 0.007231120010715849, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.12366235651074589, "avg_duration_per_task_ms": 903569.9494310999, "avg_output_tokens_per_task": 43400.72819216182, "last_run_timestamp": "2026-10-24T16:19:29.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.8760746752655538, "accuracy_std_dev": 0.011470076619186123, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.009704542235710673, "avg_duration_per_task_ms": 429066.8811330298, "avg_output_tokens_per_task": 22619.601416287303, "last_run_timestamp": "2026-10-24T08:41:28.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8760220000000001, "accuracy_std_dev": 0.014151103773204405, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.11097613275613274, "avg_duration_per_task_ms": 457617.5707070707, "avg_output_tokens_per_task": 31565.57683982684, "last_run_timestamp": "2026-10-24T16:02:47.387Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.8754205, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04156198232323233, "avg_duration_per_task_ms": 309570.8888888889, "avg_output_tokens_per_task": 39159.9494949495, "last_run_timestamp": "2026-10-17T08:26:18.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.8754205, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01960839393939394, "avg_duration_per_task_ms": 110709.8308080808, "avg_output_tokens_per_task": 43264.59848484849, "last_run_timestamp": "2026-10-24T08:12:37.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8752913076923076, "accuracy_std_dev": 0.04445532892411749, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.09291413209013209, "avg_duration_per_task_ms": 618152.5388500389, "avg_output_tokens_per_task": 41235.80536130536, "last_run_timestamp": "2026-10-24T16:04:29.298Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.8749998749999999, "accuracy_std_dev": 0.005171981690742456, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0810850505050505, "avg_duration_per_task_ms": 406962.2190656566, "avg_output_tokens_per_task": 23056.515151515152, "last_run_timestamp": "2026-10-24T16:39:19.451Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.8749675384615386, "accuracy_std_dev": 0.013017813792677657, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.006246133255633255, "avg_duration_per_task_ms": 265528.69425019424, "avg_output_tokens_per_task": 18684.931235431235, "last_run_timestamp": "2026-10-24T08:20:25.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8749014196217493, "accuracy_std_dev": 0.02259480542425625, "run_count": 5, "total_questions": 846, "avg_cost_per_task_usd": 0.12702199408983453, "avg_duration_per_task_ms": 548158.9716312056, "avg_output_tokens_per_task": 48997.07092198582, "last_run_timestamp": "2026-10-03T17:01:24.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8747894999999999, "accuracy_std_dev": 0.02495521020748971, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1433024273989899, "avg_duration_per_task_ms": 926726.0309343435, "avg_output_tokens_per_task": 61552.77146464647, "last_run_timestamp": "2026-10-24T16:07:10.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.8746087667870037, "accuracy_std_dev": 0.007592482877484693, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.2179415407942238, "avg_duration_per_task_ms": 1239989.4288808664, "avg_output_tokens_per_task": 48485.83321299639, "last_run_timestamp": "2026-10-24T16:18:35.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Reka", "accuracy": 0.8745790000000001, "accuracy_std_dev": 0.04079651270022963, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.010708270202020202, "avg_duration_per_task_ms": 125365.15404040404, "avg_output_tokens_per_task": 15791.75, "last_run_timestamp": "2026-10-24T08:20:32.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Together", "accuracy": 0.874579, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018065838383838382, "avg_duration_per_task_ms": 168823.5202020202, "avg_output_tokens_per_task": 29810.62373737374, "last_run_timestamp": "2026-10-24T08:41:24.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.8745317312013828, "accuracy_std_dev": 0.011186512454241005, "run_count": 6, "total_questions": 1157, "avg_cost_per_task_usd": 0.0043274416594641305, "avg_duration_per_task_ms": 185028.52031114954, "avg_output_tokens_per_task": 21870.51080380294, "last_run_timestamp": "2026-09-26T08:53:59.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8740843414084506, "accuracy_std_dev": 0.004005737471597214, "run_count": 9, "total_questions": 1775, "avg_cost_per_task_usd": 0.044288470985915485, "avg_duration_per_task_ms": 203522.5785915493, "avg_output_tokens_per_task": 38212.04394366197, "last_run_timestamp": "2026-10-24T08:44:12.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.8740044607508533, "accuracy_std_dev": 0.007605951280996289, "run_count": 3, "total_questions": 586, "avg_cost_per_task_usd": 0.10401818771331058, "avg_duration_per_task_ms": 459122.319112628, "avg_output_tokens_per_task": 30022.856655290103, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.8739801189873418, "accuracy_std_dev": 0.018911107403381266, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0366216776371308, "avg_duration_per_task_ms": 691274.4362869199, "avg_output_tokens_per_task": 42052.38227848101, "last_run_timestamp": "2026-10-27T01:33:21.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.8739276971507354, "accuracy_std_dev": 0.014730569376258348, "run_count": 11, "total_questions": 2176, "avg_cost_per_task_usd": 0.09131274172794118, "avg_duration_per_task_ms": 205017.0390625, "avg_output_tokens_per_task": 23097.8125, "last_run_timestamp": "2026-10-24T08:48:36.491Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8737371428571428, "accuracy_std_dev": 0.01647031431245381, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.008272517676767677, "avg_duration_per_task_ms": 219776.6417748918, "avg_output_tokens_per_task": 29274.76370851371, "last_run_timestamp": "2026-10-24T09:11:12.594Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "SambaNova", "accuracy": 0.8737370000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03531422474747475, "avg_duration_per_task_ms": 145858.30808080808, "avg_output_tokens_per_task": 29116.540404040403, "last_run_timestamp": "2026-10-24T08:59:28.837Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.8737370000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.023037462121212122, "avg_duration_per_task_ms": 506854.46717171714, "avg_output_tokens_per_task": 48027.71464646464, "last_run_timestamp": "2026-10-24T08:01:41.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0029526111111111116, "avg_duration_per_task_ms": 49722.71717171717, "avg_output_tokens_per_task": 5746.707070707071, "last_run_timestamp": "2026-10-01T08:23:37.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.045554454545454544, "avg_duration_per_task_ms": 605480.6616161616, "avg_output_tokens_per_task": 35385.22727272727, "last_run_timestamp": "2026-10-01T08:40:06.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.873716393364929, "accuracy_std_dev": 0.03410588644533702, "run_count": 9, "total_questions": 1688, "avg_cost_per_task_usd": 0.012903949052132702, "avg_duration_per_task_ms": 360645.17417061614, "avg_output_tokens_per_task": 25970.889218009477, "last_run_timestamp": "2026-10-24T08:43:07.915Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8736468057110863, "accuracy_std_dev": 0.03036521292219109, "run_count": 10, "total_questions": 1786, "avg_cost_per_task_usd": 0.025540696528555434, "avg_duration_per_task_ms": 360273.59238521836, "avg_output_tokens_per_task": 24510.389137737962, "last_run_timestamp": "2026-10-24T16:24:04.274Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.8733163333333334, "accuracy_std_dev": 0.030039827993663497, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.17326605218855218, "avg_duration_per_task_ms": 357944.43055555556, "avg_output_tokens_per_task": 39522.67929292929, "last_run_timestamp": "2026-10-24T16:27:20.826Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.8732174942820837, "accuracy_std_dev": 0.03602354128311638, "run_count": 12, "total_questions": 2361, "avg_cost_per_task_usd": 0.10224943286742906, "avg_duration_per_task_ms": 640310.1185938162, "avg_output_tokens_per_task": 25410.69970351546, "last_run_timestamp": "2026-10-24T16:26:55.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.8722641250000001, "accuracy_std_dev": 0.019820282720218067, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.19987492803030302, "avg_duration_per_task_ms": 567054.6963383838, "avg_output_tokens_per_task": 58481.14962121212, "last_run_timestamp": "2026-10-24T16:09:40.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8721539397539244, "accuracy_std_dev": 0.008970575607073516, "run_count": 12, "total_questions": 2357, "avg_cost_per_task_usd": 0.008344311412812899, "avg_duration_per_task_ms": 339433.18073822657, "avg_output_tokens_per_task": 29494.21552821383, "last_run_timestamp": "2026-10-17T08:23:51.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.872053625, "accuracy_std_dev": 0.02370397511989021, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1190935284090909, "avg_duration_per_task_ms": 606301.1369949495, "avg_output_tokens_per_task": 35231.555555555555, "last_run_timestamp": "2026-10-24T16:26:49.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.8720047896405919, "accuracy_std_dev": 0.013430501280667074, "run_count": 10, "total_questions": 1892, "avg_cost_per_task_usd": 0.00839126691331924, "avg_duration_per_task_ms": 234781.10200845666, "avg_output_tokens_per_task": 29482.704545454544, "last_run_timestamp": "2026-10-28T03:34:12.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8717731666666667, "accuracy_std_dev": 0.004601771648567064, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.00424808627946128, "avg_duration_per_task_ms": 39417.8244949495, "avg_output_tokens_per_task": 7731.690235690236, "last_run_timestamp": "2026-10-24T08:30:49.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.871717, "accuracy_std_dev": 0.007924857828377761, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007889449494949494, "avg_duration_per_task_ms": 615342.6777777778, "avg_output_tokens_per_task": 44673.57070707071, "last_run_timestamp": "2026-10-29T22:53:02.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.8714859264426126, "accuracy_std_dev": 0.00752103815972237, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.06945589727330374, "avg_duration_per_task_ms": 276858.71528218134, "avg_output_tokens_per_task": 11308.629676601142, "last_run_timestamp": "2026-10-10T08:22:14.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.8713054444444444, "accuracy_std_dev": 0.011226478265552096, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.08585465039281706, "avg_duration_per_task_ms": 191685.55331088664, "avg_output_tokens_per_task": 13992.872615039281, "last_run_timestamp": "2026-10-24T08:29:16.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.871212, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017874631313131314, "avg_duration_per_task_ms": 86992.41414141415, "avg_output_tokens_per_task": 29675.651515151516, "last_run_timestamp": "2026-10-24T08:40:56.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.8710915714285715, "accuracy_std_dev": 0.014505946572326232, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.021487838383838383, "avg_duration_per_task_ms": 272335.1349206349, "avg_output_tokens_per_task": 35144.1277056277, "last_run_timestamp": "2026-10-24T08:13:46.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.8710116425661915, "accuracy_std_dev": 0.01025225770257459, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.025059682281059068, "avg_duration_per_task_ms": 328330.6405295316, "avg_output_tokens_per_task": 20642.19348268839, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8706863196512894, "accuracy_std_dev": 0.010805462536728977, "run_count": 14, "total_questions": 2753, "avg_cost_per_task_usd": 0.047070880130766435, "avg_duration_per_task_ms": 208861.94733018524, "avg_output_tokens_per_task": 36299.43697784236, "last_run_timestamp": "2026-10-24T09:06:03.113Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.870484655811342, "accuracy_std_dev": 0.008268565007542176, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.0494731560920831, "avg_duration_per_task_ms": 288367.13307130826, "avg_output_tokens_per_task": 21014.321167883212, "last_run_timestamp": "2026-10-28T03:32:25.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.8703702499999999, "accuracy_std_dev": 0.005050541672682263, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00817759595959596, "avg_duration_per_task_ms": 373579.7626262626, "avg_output_tokens_per_task": 28859.406565656565, "last_run_timestamp": "2026-10-24T16:12:07.782Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.8701915282758622, "accuracy_std_dev": 0.09957007568085358, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.027101170114942532, "avg_duration_per_task_ms": 202993.58068965518, "avg_output_tokens_per_task": 23323.951724137933, "last_run_timestamp": "2026-10-24T16:28:20.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.8699495, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00502325, "avg_duration_per_task_ms": 389434.8611111111, "avg_output_tokens_per_task": 17944.10101010101, "last_run_timestamp": "2026-08-08T10:05:07.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.86994925, "accuracy_std_dev": 0.017018176890827654, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03481110921717172, "avg_duration_per_task_ms": 505625.7146464646, "avg_output_tokens_per_task": 35174.42866161616, "last_run_timestamp": "2026-10-24T16:26:47.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.869809111111111, "accuracy_std_dev": 0.008765689330115383, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.07596271436588103, "avg_duration_per_task_ms": 102049.09147025814, "avg_output_tokens_per_task": 12345.802469135802, "last_run_timestamp": "2026-10-24T08:49:10.928Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.8697483287395923, "accuracy_std_dev": 0.025647694972356895, "run_count": 18, "total_questions": 3483, "avg_cost_per_task_usd": 0.03920849038185472, "avg_duration_per_task_ms": 78569.80017226528, "avg_output_tokens_per_task": 25460.256675279932, "last_run_timestamp": "2026-10-24T08:27:04.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10538721464646464, "avg_duration_per_task_ms": 581981.0101010101, "avg_output_tokens_per_task": 34695.7398989899, "last_run_timestamp": "2026-08-08T09:03:54.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8695284999999999, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2202375707070707, "avg_duration_per_task_ms": 264875.98484848486, "avg_output_tokens_per_task": 21926.78787878788, "last_run_timestamp": "2026-10-01T16:12:49.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.8694541060070672, "accuracy_std_dev": 0.017661177225418892, "run_count": 9, "total_questions": 1698, "avg_cost_per_task_usd": 0.07418639340400471, "avg_duration_per_task_ms": 396957.8833922261, "avg_output_tokens_per_task": 24048.35217903416, "last_run_timestamp": "2026-10-24T08:28:10.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.8692809647058825, "accuracy_std_dev": 0.009104176033557385, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.0047775359477124175, "avg_duration_per_task_ms": 482494.045751634, "avg_output_tokens_per_task": 26353.58431372549, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.8691077499999998, "accuracy_std_dev": 0.005877315899924065, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.003982401515151515, "avg_duration_per_task_ms": 37577.292929292926, "avg_output_tokens_per_task": 7724.75, "last_run_timestamp": "2026-10-24T08:16:38.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8689927272727271, "accuracy_std_dev": 0.03917244891523285, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.17977295500459137, "avg_duration_per_task_ms": 45320.05463728191, "avg_output_tokens_per_task": 3392.5229568411387, "last_run_timestamp": "2026-10-24T08:48:02.790Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.8689616459016394, "accuracy_std_dev": 0.006730575179500894, "run_count": 9, "total_questions": 1525, "avg_cost_per_task_usd": 0.05556747672131148, "avg_duration_per_task_ms": 433024.3068852459, "avg_output_tokens_per_task": 18421.848524590165, "last_run_timestamp": "2026-10-24T16:13:46.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.868687, "accuracy_std_dev": 0.006299080210633947, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004035915404040404, "avg_duration_per_task_ms": 38787.454545454544, "avg_output_tokens_per_task": 7838.214646464647, "last_run_timestamp": "2026-10-24T08:07:26.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.005583575528950976, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.004182061868686869, "avg_duration_per_task_ms": 36016.292929292926, "avg_output_tokens_per_task": 7729.580176767677, "last_run_timestamp": "2026-10-24T08:19:41.880Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.016015464836760137, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.038159512626262625, "avg_duration_per_task_ms": 691231.6849747475, "avg_output_tokens_per_task": 34768.117424242424, "last_run_timestamp": "2026-10-24T16:16:05.009Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.009143975622643609, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.2541581641414141, "avg_duration_per_task_ms": 251149.7840909091, "avg_output_tokens_per_task": 24116.972222222223, "last_run_timestamp": "2026-10-01T16:14:35.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.8686866666666666, "accuracy_std_dev": 0.004955863351178644, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004029786195286195, "avg_duration_per_task_ms": 38652.87037037037, "avg_output_tokens_per_task": 7824.787878787879, "last_run_timestamp": "2026-10-24T08:01:41.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Nebius", "accuracy": 0.8686865, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01774992424242424, "avg_duration_per_task_ms": 402650.36616161617, "avg_output_tokens_per_task": 29383.83585858586, "last_run_timestamp": "2026-10-24T08:28:10.100Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.8686865, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06319552525252525, "avg_duration_per_task_ms": 266969.2752525252, "avg_output_tokens_per_task": 20992.954545454544, "last_run_timestamp": "2026-08-15T15:55:02.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.8686612136752137, "accuracy_std_dev": 0.02320764839922872, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.025878321367521367, "avg_duration_per_task_ms": 786906.9829059829, "avg_output_tokens_per_task": 23783.8, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8682876231003039, "accuracy_std_dev": 0.028807543557374, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.10277313323201621, "avg_duration_per_task_ms": 355978.4974670719, "avg_output_tokens_per_task": 23116.26595744681, "last_run_timestamp": "2026-10-24T08:57:08.885Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8682058571428571, "accuracy_std_dev": 0.007673379706651383, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07688844300144301, "avg_duration_per_task_ms": 83582.13492063493, "avg_output_tokens_per_task": 12574.14357864358, "last_run_timestamp": "2026-10-10T08:21:46.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8676766000000001, "accuracy_std_dev": 0.004205309600968765, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.07033428585858588, "avg_duration_per_task_ms": 232782.28383838382, "avg_output_tokens_per_task": 11454.874242424243, "last_run_timestamp": "2026-10-24T08:34:00.054Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.867634625, "accuracy_std_dev": 0.007431114989311851, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.025694838383838382, "avg_duration_per_task_ms": 248942.36553030304, "avg_output_tokens_per_task": 21167.008207070707, "last_run_timestamp": "2026-10-08T22:14:16.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.8675645, "accuracy_std_dev": 0.008228295808772726, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.004289666245791245, "avg_duration_per_task_ms": 35945.74074074074, "avg_output_tokens_per_task": 7815.2925084175085, "last_run_timestamp": "2026-10-24T08:43:41.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8675643333333334, "accuracy_std_dev": 0.0028614651647170092, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.009448984848484849, "avg_duration_per_task_ms": 79377.02525252526, "avg_output_tokens_per_task": 7744.79797979798, "last_run_timestamp": "2026-10-02T23:19:10.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.8674239999999999, "accuracy_std_dev": 0.03557408983609655, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.18567804966329968, "avg_duration_per_task_ms": 45738.964225589225, "avg_output_tokens_per_task": 3505.928872053872, "last_run_timestamp": "2026-10-24T08:45:03.898Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "auto-routing", "accuracy": 0.867003, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013525050505050505, "avg_duration_per_task_ms": 181228.28787878787, "avg_output_tokens_per_task": 22673.38888888889, "last_run_timestamp": "2026-10-24T08:02:32.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.867003, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.22265377272727274, "avg_duration_per_task_ms": 265427.9646464646, "avg_output_tokens_per_task": 22169.171717171717, "last_run_timestamp": "2026-10-01T16:12:54.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.8666971818181819, "accuracy_std_dev": 0.02628554379335525, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.007686401744719926, "avg_duration_per_task_ms": 446263.08815426996, "avg_output_tokens_per_task": 25752.568411386594, "last_run_timestamp": "2026-10-24T08:15:15.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.8664308674130818, "accuracy_std_dev": 0.027802409922633914, "run_count": 9, "total_questions": 1697, "avg_cost_per_task_usd": 0.011504347672362995, "avg_duration_per_task_ms": 435596.48084855627, "avg_output_tokens_per_task": 25872.670005892753, "last_run_timestamp": "2026-10-24T08:11:43.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8663018333333332, "accuracy_std_dev": 0.04118241672492712, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1777705147306397, "avg_duration_per_task_ms": 43633.450336700334, "avg_output_tokens_per_task": 3347.0820707070707, "last_run_timestamp": "2026-10-24T08:27:26.672Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.8656005, "accuracy_std_dev": 0.00803956708515244, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02525451262626263, "avg_duration_per_task_ms": 233486.3627946128, "avg_output_tokens_per_task": 20811.0765993266, "last_run_timestamp": "2026-10-01T16:08:04.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8654601250000001, "accuracy_std_dev": 0.02784951724493697, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.15895532175925928, "avg_duration_per_task_ms": 153299.00252525252, "avg_output_tokens_per_task": 15366.623316498317, "last_run_timestamp": "2026-10-24T08:20:16.287Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.8653352747021724, "accuracy_std_dev": 0.017080671231962035, "run_count": 8, "total_questions": 1427, "avg_cost_per_task_usd": 0.005114180098107919, "avg_duration_per_task_ms": 702127.0336370007, "avg_output_tokens_per_task": 28117.81079187106, "last_run_timestamp": "2026-10-24T16:15:47.409Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Relace", "accuracy": 0.86532, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01495640909090909, "avg_duration_per_task_ms": 120441.0101010101, "avg_output_tokens_per_task": 29795.70707070707, "last_run_timestamp": "2026-10-24T15:00:02.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8653195, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008587179292929294, "avg_duration_per_task_ms": 240899.30303030304, "avg_output_tokens_per_task": 30387.573232323233, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.8652359694129255, "accuracy_std_dev": 0.07811822893653826, "run_count": 21, "total_questions": 4054, "avg_cost_per_task_usd": 0.22709830858411445, "avg_duration_per_task_ms": 501409.10236803157, "avg_output_tokens_per_task": 63065.26467686236, "last_run_timestamp": "2026-10-29T22:46:44.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8650137272727274, "accuracy_std_dev": 0.051263649171514956, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1117407479338843, "avg_duration_per_task_ms": 477041.2093663912, "avg_output_tokens_per_task": 28184.2681359045, "last_run_timestamp": "2026-10-27T01:15:58.793Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.864899, "accuracy_std_dev": 0.017388735477314038, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07374217613636364, "avg_duration_per_task_ms": 286692.6893939394, "avg_output_tokens_per_task": 20985.801767676767, "last_run_timestamp": "2026-10-24T16:08:41.781Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cohere", "accuracy": 0.8644780000000001, "accuracy_std_dev": 0.01094299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.05228046464646464, "avg_duration_per_task_ms": 195135.57323232322, "avg_output_tokens_per_task": 43383.04797979798, "last_run_timestamp": "2026-10-24T08:15:15.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.86433775, "accuracy_std_dev": 0.014059447115285145, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.7497476914983165, "avg_duration_per_task_ms": 499048.3042929293, "avg_output_tokens_per_task": 29826.423821548822, "last_run_timestamp": "2026-10-24T08:11:56.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8638838235294116, "accuracy_std_dev": 0.02774133966979138, "run_count": 34, "total_questions": 6732, "avg_cost_per_task_usd": 0.17359224925727867, "avg_duration_per_task_ms": 159152.2174688057, "avg_output_tokens_per_task": 16169.014557338087, "last_run_timestamp": "2026-10-24T08:48:23.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.8636430722945668, "accuracy_std_dev": 0.030671339204155882, "run_count": 12, "total_questions": 2227, "avg_cost_per_task_usd": 0.02386835383924562, "avg_duration_per_task_ms": 460447.3304894477, "avg_output_tokens_per_task": 22718.12303547373, "last_run_timestamp": "2026-10-17T08:18:04.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0938164393939394, "avg_duration_per_task_ms": 667365.3939393939, "avg_output_tokens_per_task": 23446.50505050505, "last_run_timestamp": "2026-09-10T01:44:54.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.8634960000000002, "accuracy_std_dev": 0.01008821268444846, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8421511931818182, "avg_duration_per_task_ms": 412180.98316498316, "avg_output_tokens_per_task": 33494.958333333336, "last_run_timestamp": "2026-10-24T08:26:00.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.8634435114503817, "accuracy_std_dev": 0.01366835982067709, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.13531842366412217, "avg_duration_per_task_ms": 289599.17048346053, "avg_output_tokens_per_task": 28034.88358778626, "last_run_timestamp": "2026-10-17T08:23:25.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "ModelRun", "accuracy": 0.8630749999999998, "accuracy_std_dev": 0.0034590473640392213, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.018059974747474748, "avg_duration_per_task_ms": 144168.21043771043, "avg_output_tokens_per_task": 29911.44107744108, "last_run_timestamp": "2026-10-28T04:56:05.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.862963, "accuracy_std_dev": 0.009134508372101892, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.06279948181818182, "avg_duration_per_task_ms": 441139.7515151515, "avg_output_tokens_per_task": 56049.472727272725, "last_run_timestamp": "2026-10-24T16:14:38.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8629348333333335, "accuracy_std_dev": 0.008541693195080755, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8005278598484848, "avg_duration_per_task_ms": 459239.8367003367, "avg_output_tokens_per_task": 31848.97138047138, "last_run_timestamp": "2026-10-24T08:27:05.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.8626261, "accuracy_std_dev": 0.011472416183611877, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.07024428484848486, "avg_duration_per_task_ms": 231811.16010101011, "avg_output_tokens_per_task": 11437.808585858586, "last_run_timestamp": "2026-10-24T08:31:24.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8625841249999999, "accuracy_std_dev": 0.006783236993454885, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.004199931818181818, "avg_duration_per_task_ms": 250234.78724747474, "avg_output_tokens_per_task": 29101.075757575756, "last_run_timestamp": "2026-10-24T16:02:06.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.8625343605313093, "accuracy_std_dev": 0.061262981528279976, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.2509589614168248, "avg_duration_per_task_ms": 1179158.6533839342, "avg_output_tokens_per_task": 55775.307400379505, "last_run_timestamp": "2026-10-24T16:03:06.207Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.8619528181818182, "accuracy_std_dev": 0.017198547374273336, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.09933105968778695, "avg_duration_per_task_ms": 470423.74885215797, "avg_output_tokens_per_task": 26261.255280073463, "last_run_timestamp": "2026-10-24T16:12:09.459Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.86195275, "accuracy_std_dev": 0.04035143977217541, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.181786273989899, "avg_duration_per_task_ms": 45601.15025252525, "avg_output_tokens_per_task": 3426.050925925926, "last_run_timestamp": "2026-10-24T08:08:40.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8619527499999999, "accuracy_std_dev": 0.02717181928832702, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.13016119191919193, "avg_duration_per_task_ms": 482692.7720959596, "avg_output_tokens_per_task": 34118.55934343435, "last_run_timestamp": "2026-10-24T16:05:51.240Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.861837910657307, "accuracy_std_dev": 0.01299998475419779, "run_count": 8, "total_questions": 1567, "avg_cost_per_task_usd": 0.07543853095086152, "avg_duration_per_task_ms": 279737.1965539247, "avg_output_tokens_per_task": 20830.50095724314, "last_run_timestamp": "2026-10-24T16:12:49.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.861708258064516, "accuracy_std_dev": 0.012137624210372552, "run_count": 6, "total_questions": 1116, "avg_cost_per_task_usd": 0.02452295071684588, "avg_duration_per_task_ms": 145394.43727598566, "avg_output_tokens_per_task": 37540.51523297491, "last_run_timestamp": "2026-10-24T08:12:02.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8614937272727271, "accuracy_std_dev": 0.007701093359102963, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.06505497428833792, "avg_duration_per_task_ms": 215627.76170798898, "avg_output_tokens_per_task": 10574.208907254362, "last_run_timestamp": "2026-10-28T03:28:22.479Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.04603966955336814, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004525512626262627, "avg_duration_per_task_ms": 565963.2584175084, "avg_output_tokens_per_task": 18978.082491582492, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.8613914444444443, "accuracy_std_dev": 0.017477429375633225, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.09228736588103255, "avg_duration_per_task_ms": 186627.3569023569, "avg_output_tokens_per_task": 25504.406285072953, "last_run_timestamp": "2026-10-24T16:20:59.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.8613514285714288, "accuracy_std_dev": 0.011656715536635543, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.842712674963925, "avg_duration_per_task_ms": 456401.95995670994, "avg_output_tokens_per_task": 33529.13383838384, "last_run_timestamp": "2026-10-24T08:15:51.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.8612792, "accuracy_std_dev": 0.03136197207702347, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005120404040404041, "avg_duration_per_task_ms": 530889.794949495, "avg_output_tokens_per_task": 17542.636363636364, "last_run_timestamp": "2026-09-05T09:22:54.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Novita", "accuracy": 0.861111, "accuracy_std_dev": 0.04797999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.021780800505050504, "avg_duration_per_task_ms": 138470.81818181818, "avg_output_tokens_per_task": 20189.02525252525, "last_run_timestamp": "2026-10-24T08:12:37.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8609707916666667, "accuracy_std_dev": 0.007924941871391254, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.8830173162878786, "avg_duration_per_task_ms": 414862.5126262626, "avg_output_tokens_per_task": 33448.23737373737, "last_run_timestamp": "2026-10-24T08:44:41.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.8608923543307087, "accuracy_std_dev": 0.017439499999999997, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.003837814960629921, "avg_duration_per_task_ms": 298144.1771653543, "avg_output_tokens_per_task": 11523.807086614173, "last_run_timestamp": "2026-10-03T17:04:09.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.8605965053212455, "accuracy_std_dev": 0.03960090537740342, "run_count": 13, "total_questions": 2537, "avg_cost_per_task_usd": 0.01657229877808435, "avg_duration_per_task_ms": 463843.5068979109, "avg_output_tokens_per_task": 41792.180134016555, "last_run_timestamp": "2026-10-24T08:58:59.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.8602692499999999, "accuracy_std_dev": 0.07328871721500406, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0069893973063973065, "avg_duration_per_task_ms": 603844.6805555555, "avg_output_tokens_per_task": 28814.18181818182, "last_run_timestamp": "2026-10-24T08:23:31.990Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Io Net", "accuracy": 0.860269, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00987439393939394, "avg_duration_per_task_ms": 74831.28787878787, "avg_output_tokens_per_task": 9997.058080808081, "last_run_timestamp": "2026-10-24T08:14:36.264Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.8601809488313328, "accuracy_std_dev": 0.03964118133468496, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.16196978711307647, "avg_duration_per_task_ms": 990576.0770688566, "avg_output_tokens_per_task": 55234.00947567909, "last_run_timestamp": "2026-10-24T16:06:09.255Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.8599885833333334, "accuracy_std_dev": 0.009617771098842076, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8533179797979797, "avg_duration_per_task_ms": 482512.8122895623, "avg_output_tokens_per_task": 33941.89983164983, "last_run_timestamp": "2026-10-24T08:14:51.084Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.8598806153846154, "accuracy_std_dev": 0.030036815288878832, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.15524810489510485, "avg_duration_per_task_ms": 152128.8811188811, "avg_output_tokens_per_task": 15307.565656565657, "last_run_timestamp": "2026-10-24T08:12:37.564Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.8597419723629663, "accuracy_std_dev": 0.016505636989518208, "run_count": 11, "total_questions": 2171, "avg_cost_per_task_usd": 0.08549684339014277, "avg_duration_per_task_ms": 179079.73422385997, "avg_output_tokens_per_task": 24385.910640257945, "last_run_timestamp": "2026-10-24T08:41:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Io Net", "accuracy": 0.8596746149425287, "accuracy_std_dev": 0.000831499999999985, "run_count": 2, "total_questions": 348, "avg_cost_per_task_usd": 0.0899417959770115, "avg_duration_per_task_ms": 1024745.4281609196, "avg_output_tokens_per_task": 35825.26724137931, "last_run_timestamp": "2026-10-27T13:57:40.548Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.8596613057736721, "accuracy_std_dev": 0.0311431999096363, "run_count": 11, "total_questions": 2165, "avg_cost_per_task_usd": 0.1269020115473441, "avg_duration_per_task_ms": 528998.0508083141, "avg_output_tokens_per_task": 43596.739953810626, "last_run_timestamp": "2026-10-24T16:12:49.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.8594274166666667, "accuracy_std_dev": 0.018601260147538098, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.7585617697811449, "avg_duration_per_task_ms": 508230.9494949495, "avg_output_tokens_per_task": 30213.497474747473, "last_run_timestamp": "2026-10-24T08:10:23.014Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.8590258723037101, "accuracy_std_dev": 0.009219054418533492, "run_count": 21, "total_questions": 3477, "avg_cost_per_task_usd": 0.1720266922634455, "avg_duration_per_task_ms": 256422.9841817659, "avg_output_tokens_per_task": 31690.788323267185, "last_run_timestamp": "2026-09-26T08:54:22.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8586340031847132, "accuracy_std_dev": 0.05344060455646062, "run_count": 10, "total_questions": 1884, "avg_cost_per_task_usd": 0.0115320483014862, "avg_duration_per_task_ms": 468303.98089171975, "avg_output_tokens_per_task": 23774.996815286624, "last_run_timestamp": "2026-10-24T08:32:58.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08651329797979797, "avg_duration_per_task_ms": 499241.5707070707, "avg_output_tokens_per_task": 35858.23737373737, "last_run_timestamp": "2026-10-04T03:06:10.317Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.85858575, "accuracy_std_dev": 0.021526546714173627, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1476876622474748, "avg_duration_per_task_ms": 1253876.004419192, "avg_output_tokens_per_task": 52218.26641414141, "last_run_timestamp": "2026-10-24T16:26:15.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8585855, "accuracy_std_dev": 0.012710146271778291, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.014605150252525251, "avg_duration_per_task_ms": 477497.9494949495, "avg_output_tokens_per_task": 26944.89898989899, "last_run_timestamp": "2026-10-24T08:15:32.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.8585232040276647, "accuracy_std_dev": 0.03867598457396728, "run_count": 29, "total_questions": 4916, "avg_cost_per_task_usd": 0.01779470951993491, "avg_duration_per_task_ms": 399750.05593978846, "avg_output_tokens_per_task": 18974.740846216435, "last_run_timestamp": "2026-10-24T08:49:56.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.858164875, "accuracy_std_dev": 0.016185868044667075, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1437290511363636, "avg_duration_per_task_ms": 908116.1098484849, "avg_output_tokens_per_task": 45706.13320707071, "last_run_timestamp": "2026-10-24T16:14:06.777Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.8579636833065611, "accuracy_std_dev": 0.020319706431789875, "run_count": 41, "total_questions": 8093, "avg_cost_per_task_usd": 0.28457569974051644, "avg_duration_per_task_ms": 335733.34103546274, "avg_output_tokens_per_task": 51175.235635734585, "last_run_timestamp": "2026-10-24T16:37:01.548Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.8577439999999998, "accuracy_std_dev": 0.019433210349810964, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02084834090909091, "avg_duration_per_task_ms": 460409.3712121212, "avg_output_tokens_per_task": 41456.85101010101, "last_run_timestamp": "2026-10-29T23:33:29.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.8577439285714286, "accuracy_std_dev": 0.016868073763619186, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.7558037463924961, "avg_duration_per_task_ms": 505406.40187590185, "avg_output_tokens_per_task": 30123.084054834057, "last_run_timestamp": "2026-10-24T08:30:49.044Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8576738333333334, "accuracy_std_dev": 0.01763649745004817, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.8048634583333334, "avg_duration_per_task_ms": 484140.92781986535, "avg_output_tokens_per_task": 30507.85164141414, "last_run_timestamp": "2026-10-24T08:15:15.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.857663467955239, "accuracy_std_dev": 0.05434631352575813, "run_count": 10, "total_questions": 1966, "avg_cost_per_task_usd": 0.0781413972533062, "avg_duration_per_task_ms": 509131.62817904376, "avg_output_tokens_per_task": 21110.80518819939, "last_run_timestamp": "2026-10-24T08:57:31.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.857418, "accuracy_std_dev": null, "run_count": 1, "total_questions": 173, "avg_cost_per_task_usd": 0.08778269942196533, "avg_duration_per_task_ms": 317630.5202312139, "avg_output_tokens_per_task": 36575.57225433526, "last_run_timestamp": "2026-08-04T03:12:38.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8573231875, "accuracy_std_dev": 0.007907506704223743, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.06346724873737374, "avg_duration_per_task_ms": 591294.5839646464, "avg_output_tokens_per_task": 35019.75441919192, "last_run_timestamp": "2026-10-24T16:03:59.167Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8573231666666665, "accuracy_std_dev": 0.0170736054571246, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8399580702861953, "avg_duration_per_task_ms": 458034.78324915824, "avg_output_tokens_per_task": 33422.68813131313, "last_run_timestamp": "2026-10-24T08:13:46.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.8571427108013937, "accuracy_std_dev": 0.01011902539224449, "run_count": 3, "total_questions": 574, "avg_cost_per_task_usd": 0.11599589721254357, "avg_duration_per_task_ms": 363022.99128919863, "avg_output_tokens_per_task": 29391.742160278747, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8570552727272727, "accuracy_std_dev": 0.02678680556845074, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8339512213039484, "avg_duration_per_task_ms": 436211.7764003673, "avg_output_tokens_per_task": 33251.8347107438, "last_run_timestamp": "2026-10-24T08:08:52.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8569032243034418, "accuracy_std_dev": 0.036552330331727854, "run_count": 22, "total_questions": 4271, "avg_cost_per_task_usd": 0.0041591236244439235, "avg_duration_per_task_ms": 82317.1749004917, "avg_output_tokens_per_task": 15615.828377429174, "last_run_timestamp": "2026-10-24T08:41:46.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Tenstorrent", "accuracy": 0.8569025, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017741116161616163, "avg_duration_per_task_ms": 206976.69949494948, "avg_output_tokens_per_task": 29405.780303030304, "last_run_timestamp": "2026-10-30T18:32:25.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Wafer", "accuracy": 0.8569025, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018898722222222223, "avg_duration_per_task_ms": 164757.63383838383, "avg_output_tokens_per_task": 31337.035353535353, "last_run_timestamp": "2026-10-24T08:17:56.393Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8569022500000001, "accuracy_std_dev": 0.013202173483464743, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.007818540404040404, "avg_duration_per_task_ms": 506894.8308080808, "avg_output_tokens_per_task": 28009.079545454544, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.8569019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20989276262626264, "avg_duration_per_task_ms": 263850.72222222225, "avg_output_tokens_per_task": 20893.070707070707, "last_run_timestamp": "2026-10-01T16:31:36.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.8568129407236336, "accuracy_std_dev": 0.01596270457778066, "run_count": 8, "total_questions": 1299, "avg_cost_per_task_usd": 0.004350760585065434, "avg_duration_per_task_ms": 410967.9938414165, "avg_output_tokens_per_task": 9402.565819861431, "last_run_timestamp": "2026-10-24T08:24:10.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.8563635600000002, "accuracy_std_dev": 0.050198181182254005, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.18820920787878787, "avg_duration_per_task_ms": 41314.649292929294, "avg_output_tokens_per_task": 3389.456767676768, "last_run_timestamp": "2026-10-24T08:44:43.918Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.856164284931507, "accuracy_std_dev": 0.05295168533540149, "run_count": 8, "total_questions": 1460, "avg_cost_per_task_usd": 0.18168324041095887, "avg_duration_per_task_ms": 1626314.2294520547, "avg_output_tokens_per_task": 52899.55684931507, "last_run_timestamp": "2026-10-24T16:16:05.013Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.8558499999999999, "accuracy_std_dev": 0.03281670770735542, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.015880464015151512, "avg_duration_per_task_ms": 344363.2039141414, "avg_output_tokens_per_task": 28437.20138888889, "last_run_timestamp": "2026-10-24T16:21:33.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8558169770378875, "accuracy_std_dev": 0.018638102580532955, "run_count": 9, "total_questions": 1742, "avg_cost_per_task_usd": 0.08209278185993112, "avg_duration_per_task_ms": 337904.9672789897, "avg_output_tokens_per_task": 28132.247990815154, "last_run_timestamp": "2026-10-24T08:19:59.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Fireworks", "accuracy": 0.855721, "accuracy_std_dev": null, "run_count": 1, "total_questions": 67, "avg_cost_per_task_usd": 0.015361298507462687, "avg_duration_per_task_ms": 180200.07462686568, "avg_output_tokens_per_task": 25482.86567164179, "last_run_timestamp": "2026-10-28T00:00:04.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8554993333333334, "accuracy_std_dev": 0.009394238245979377, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.005048751683501684, "avg_duration_per_task_ms": 532489.6868686868, "avg_output_tokens_per_task": 30189.478956228955, "last_run_timestamp": "2026-10-24T16:02:08.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.85530295, "accuracy_std_dev": 0.01065364351513132, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.14301361767676768, "avg_duration_per_task_ms": 429371.0404040404, "avg_output_tokens_per_task": 43957.30757575758, "last_run_timestamp": "2026-10-24T16:26:47.544Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.8552187499999999, "accuracy_std_dev": 0.022460588392504314, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04356151957070707, "avg_duration_per_task_ms": 653045.7601010101, "avg_output_tokens_per_task": 36314.97159090909, "last_run_timestamp": "2026-10-24T16:04:33.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.8550657272727274, "accuracy_std_dev": 0.012107062717806176, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.02919473599632691, "avg_duration_per_task_ms": 757966.051882461, "avg_output_tokens_per_task": 33302.242883379244, "last_run_timestamp": "2026-10-17T08:19:33.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.8549618244274809, "accuracy_std_dev": 0.005347999999999964, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.22707390839694658, "avg_duration_per_task_ms": 373958.2061068702, "avg_output_tokens_per_task": 50795.681933842236, "last_run_timestamp": "2026-08-15T08:57:45.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.85479775, "accuracy_std_dev": 0.009017138608644082, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.09609784722222224, "avg_duration_per_task_ms": 435031.48232323234, "avg_output_tokens_per_task": 30609.608585858587, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.8541006015180266, "accuracy_std_dev": 0.015925939750529645, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.037091079696394684, "avg_duration_per_task_ms": 591845.9070208728, "avg_output_tokens_per_task": 34537.16951296648, "last_run_timestamp": "2026-10-24T16:36:57.539Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.853535, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.028549383838383836, "avg_duration_per_task_ms": 582750.7070707071, "avg_output_tokens_per_task": 33383.742424242424, "last_run_timestamp": "2026-08-08T09:24:32.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.8533329920000001, "accuracy_std_dev": 0.013089000000000017, "run_count": 2, "total_questions": 250, "avg_cost_per_task_usd": 0.005868832, "avg_duration_per_task_ms": 205553.388, "avg_output_tokens_per_task": 21941.544, "last_run_timestamp": "2026-10-25T21:00:03.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.8532693421052633, "accuracy_std_dev": 0.035890582042967266, "run_count": 38, "total_questions": 7524, "avg_cost_per_task_usd": 0.1652858625730994, "avg_duration_per_task_ms": 151312.44696969696, "avg_output_tokens_per_task": 15290.7336523126, "last_run_timestamp": "2026-10-24T08:32:13.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.8530642646239555, "accuracy_std_dev": 0.022329477337983097, "run_count": 4, "total_questions": 718, "avg_cost_per_task_usd": 0.02860370194986072, "avg_duration_per_task_ms": 460751.2409470752, "avg_output_tokens_per_task": 34791.85933147632, "last_run_timestamp": "2026-10-10T08:22:41.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.8530043052097741, "accuracy_std_dev": 0.0200463394003836, "run_count": 11, "total_questions": 2169, "avg_cost_per_task_usd": 0.09074984416781928, "avg_duration_per_task_ms": 309963.8911940987, "avg_output_tokens_per_task": 26287.04610419548, "last_run_timestamp": "2026-10-27T01:51:15.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8528355426944971, "accuracy_std_dev": 0.013503895836017096, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.0925430898165718, "avg_duration_per_task_ms": 296081.5515496521, "avg_output_tokens_per_task": 29809.33523086654, "last_run_timestamp": "2026-10-24T08:21:56.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.852693625, "accuracy_std_dev": 0.006627993756362076, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.05533727904040404, "avg_duration_per_task_ms": 425236.9147727273, "avg_output_tokens_per_task": 28304.02904040404, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.8524375810276679, "accuracy_std_dev": 0.011217700923094702, "run_count": 4, "total_questions": 759, "avg_cost_per_task_usd": 0.07750415546772069, "avg_duration_per_task_ms": 263910.441370224, "avg_output_tokens_per_task": 26225.246376811596, "last_run_timestamp": "2026-10-24T08:40:56.292Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8524129999999999, "accuracy_std_dev": 0.012672927610549278, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1049078827160494, "avg_duration_per_task_ms": 476715.49887766555, "avg_output_tokens_per_task": 30743.156565656565, "last_run_timestamp": "2026-10-24T08:15:15.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.8521062280219781, "accuracy_std_dev": 0.006055500000000047, "run_count": 2, "total_questions": 364, "avg_cost_per_task_usd": 0.005501717032967033, "avg_duration_per_task_ms": 539158.467032967, "avg_output_tokens_per_task": 15491.851648351649, "last_run_timestamp": "2026-10-17T08:17:58.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Parasail", "accuracy": 0.851852, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01747031565656566, "avg_duration_per_task_ms": 122982.43434343435, "avg_output_tokens_per_task": 28903.234848484848, "last_run_timestamp": "2026-10-24T08:15:51.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.16585097979797978, "avg_duration_per_task_ms": 389107.60101010103, "avg_output_tokens_per_task": 37536.631313131315, "last_run_timestamp": "2026-08-04T03:12:10.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8518518181818182, "accuracy_std_dev": 0.03385326940520644, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.7072226216712579, "avg_duration_per_task_ms": 481789.99357208447, "avg_output_tokens_per_task": 28288.861799816346, "last_run_timestamp": "2026-10-24T08:43:42.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8518517499999999, "accuracy_std_dev": 0.0020620621201845155, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.21063902146464644, "avg_duration_per_task_ms": 251655.29545454544, "avg_output_tokens_per_task": 19971.159090909092, "last_run_timestamp": "2026-10-01T16:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.8516414999999999, "accuracy_std_dev": 0.028797790015034144, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07571042234848485, "avg_duration_per_task_ms": 512797.04924242425, "avg_output_tokens_per_task": 22901.14393939394, "last_run_timestamp": "2026-10-24T16:15:14.403Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.8515965972461275, "accuracy_std_dev": 0.006985364039578179, "run_count": 9, "total_questions": 1743, "avg_cost_per_task_usd": 0.09721534480780264, "avg_duration_per_task_ms": 672514.3052208836, "avg_output_tokens_per_task": 40113.75157773953, "last_run_timestamp": "2026-10-24T16:04:55.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.8510099999999999, "accuracy_std_dev": 0.037427264881366905, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.12598769823232323, "avg_duration_per_task_ms": 640494.7992424242, "avg_output_tokens_per_task": 42382.61111111111, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.8509267650397275, "accuracy_std_dev": 0.029137382315888, "run_count": 9, "total_questions": 1762, "avg_cost_per_task_usd": 0.14751040351872874, "avg_duration_per_task_ms": 406196.22701475595, "avg_output_tokens_per_task": 33481.99659477866, "last_run_timestamp": "2026-10-24T08:12:02.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.8508249284795883, "accuracy_std_dev": 0.01638111560444773, "run_count": 33, "total_questions": 6222, "avg_cost_per_task_usd": 0.1628150337512054, "avg_duration_per_task_ms": 188242.73111539698, "avg_output_tokens_per_task": 29107.008357441337, "last_run_timestamp": "2026-10-24T09:28:03.268Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.850799625, "accuracy_std_dev": 0.01687570508851038, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.013269784090909089, "avg_duration_per_task_ms": 808248.1130050505, "avg_output_tokens_per_task": 28025.626893939392, "last_run_timestamp": "2026-10-24T16:04:55.172Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.8507279006896552, "accuracy_std_dev": 0.012037642028375229, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.03107361793103448, "avg_duration_per_task_ms": 729282.3756321839, "avg_output_tokens_per_task": 32410.26482758621, "last_run_timestamp": "2026-10-17T08:19:39.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.8504104417831005, "accuracy_std_dev": 0.011757939662675208, "run_count": 9, "total_questions": 1503, "avg_cost_per_task_usd": 0.006495053226879575, "avg_duration_per_task_ms": 421761.22222222225, "avg_output_tokens_per_task": 16002.619427811045, "last_run_timestamp": "2026-10-24T16:11:33.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8504087142857142, "accuracy_std_dev": 0.010549160084565759, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.012687737373737374, "avg_duration_per_task_ms": 687002.3088023087, "avg_output_tokens_per_task": 18864.385281385283, "last_run_timestamp": "2026-10-17T16:05:11.655Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.8502384166666668, "accuracy_std_dev": 0.009974367911955898, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.09800930387205385, "avg_duration_per_task_ms": 567891.4086700337, "avg_output_tokens_per_task": 30514.193602693602, "last_run_timestamp": "2026-10-24T16:13:46.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.8501685000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20853091414141414, "avg_duration_per_task_ms": 280266.345959596, "avg_output_tokens_per_task": 20755.358585858587, "last_run_timestamp": "2026-10-01T16:20:44.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8501683333333333, "accuracy_std_dev": 0.009523284669109108, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03896970159932659, "avg_duration_per_task_ms": 390855.180976431, "avg_output_tokens_per_task": 39310.545454545456, "last_run_timestamp": "2026-10-24T16:23:26.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8501682222222223, "accuracy_std_dev": 0.0192438426734243, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03105917845117845, "avg_duration_per_task_ms": 604384.6245791246, "avg_output_tokens_per_task": 33888.867003367006, "last_run_timestamp": "2026-10-27T03:42:35.337Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8501303479061206, "accuracy_std_dev": 0.03129572966618359, "run_count": 11, "total_questions": 2173, "avg_cost_per_task_usd": 0.008647949838932353, "avg_duration_per_task_ms": 492602.7482742752, "avg_output_tokens_per_task": 30274.529682466637, "last_run_timestamp": "2026-10-27T01:57:27.248Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.8499550449236298, "accuracy_std_dev": 0.04367521121615224, "run_count": 7, "total_questions": 1113, "avg_cost_per_task_usd": 0.023873053908355795, "avg_duration_per_task_ms": 467262.93441150046, "avg_output_tokens_per_task": 37562.04132973944, "last_run_timestamp": "2026-10-24T08:33:50.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.8497263442622951, "accuracy_std_dev": 0.01704, "run_count": 2, "total_questions": 366, "avg_cost_per_task_usd": 0.06129340163934426, "avg_duration_per_task_ms": 530265.6393442623, "avg_output_tokens_per_task": 25778.054644808744, "last_run_timestamp": "2026-08-15T08:47:51.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.8497037279483036, "accuracy_std_dev": 0.02211279185187559, "run_count": 16, "total_questions": 3095, "avg_cost_per_task_usd": 0.2026981760904685, "avg_duration_per_task_ms": 259750.08206785138, "avg_output_tokens_per_task": 43394.09111470113, "last_run_timestamp": "2026-10-24T16:29:51.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.8492062857142857, "accuracy_std_dev": 0.009075999999999973, "run_count": 2, "total_questions": 336, "avg_cost_per_task_usd": 0.1515307232142857, "avg_duration_per_task_ms": 1515335.5773809524, "avg_output_tokens_per_task": 50423.354166666664, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.8489659064484613, "accuracy_std_dev": 0.09543669529109948, "run_count": 21, "total_questions": 4094, "avg_cost_per_task_usd": 0.00791240718124084, "avg_duration_per_task_ms": 580031.3243771372, "avg_output_tokens_per_task": 27584.65388373229, "last_run_timestamp": "2026-10-24T08:58:59.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.8486952499999999, "accuracy_std_dev": 0.02356047809229474, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04730151515151515, "avg_duration_per_task_ms": 620661.3125, "avg_output_tokens_per_task": 38343.770202020205, "last_run_timestamp": "2026-10-24T16:27:04.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0012582575757575758, "avg_duration_per_task_ms": 88400.0101010101, "avg_output_tokens_per_task": 17501.79797979798, "last_run_timestamp": "2026-10-01T08:22:54.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.8482398753606462, "accuracy_std_dev": 0.025739734220236125, "run_count": 9, "total_questions": 1733, "avg_cost_per_task_usd": 0.10720182342758222, "avg_duration_per_task_ms": 497095.3819965378, "avg_output_tokens_per_task": 34429.2602423543, "last_run_timestamp": "2026-10-24T08:11:11.432Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8480638333333334, "accuracy_std_dev": 0.01645355840354568, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.052020993265993264, "avg_duration_per_task_ms": 270683.26851851854, "avg_output_tokens_per_task": 45678.64351851852, "last_run_timestamp": "2026-10-24T16:18:16.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.8480328171206225, "accuracy_std_dev": 0.009316525307514355, "run_count": 12, "total_questions": 2313, "avg_cost_per_task_usd": 0.06691969217466492, "avg_duration_per_task_ms": 195295.10635538262, "avg_output_tokens_per_task": 20814.787721573713, "last_run_timestamp": "2026-10-24T16:04:55.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.8479167041666666, "accuracy_std_dev": 0.00828816902819917, "run_count": 5, "total_questions": 960, "avg_cost_per_task_usd": 0.16138468541666665, "avg_duration_per_task_ms": 277988.090625, "avg_output_tokens_per_task": 24274.81875, "last_run_timestamp": "2026-10-28T05:08:57.249Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.847853375, "accuracy_std_dev": 0.033941681060819205, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.010883301767676767, "avg_duration_per_task_ms": 412406.98232323234, "avg_output_tokens_per_task": 38550.31060606061, "last_run_timestamp": "2026-10-24T16:02:06.909Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.84771325, "accuracy_std_dev": 0.017907204896004843, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.12780570496632998, "avg_duration_per_task_ms": 482349.20328282827, "avg_output_tokens_per_task": 47672.79882154882, "last_run_timestamp": "2026-10-24T16:02:47.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8476430833333334, "accuracy_std_dev": 0.017202829546610125, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.12729099452861953, "avg_duration_per_task_ms": 386377.8813131313, "avg_output_tokens_per_task": 38419.52356902357, "last_run_timestamp": "2026-10-24T16:18:17.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.84722225, "accuracy_std_dev": 0.04063783312613384, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.09673938762626262, "avg_duration_per_task_ms": 1384478.7525252525, "avg_output_tokens_per_task": 42908.3345959596, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.8466609166666665, "accuracy_std_dev": 0.04565389387456148, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.16722773358585863, "avg_duration_per_task_ms": 165303.9515993266, "avg_output_tokens_per_task": 16503.33207070707, "last_run_timestamp": "2026-10-24T08:13:41.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.8464799745547075, "accuracy_std_dev": 0.010187806019943643, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.007149598600508905, "avg_duration_per_task_ms": 580512.5, "avg_output_tokens_per_task": 17687.171119592876, "last_run_timestamp": "2026-10-24T16:26:58.608Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8463488417849898, "accuracy_std_dev": 0.018032907906380492, "run_count": 10, "total_questions": 1972, "avg_cost_per_task_usd": 0.010114639452332656, "avg_duration_per_task_ms": 449682.4756592292, "avg_output_tokens_per_task": 32532.80730223124, "last_run_timestamp": "2026-10-24T08:45:13.862Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8462835489864864, "accuracy_std_dev": 0.018064668045116323, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.06724871058558558, "avg_duration_per_task_ms": 247566.1204954955, "avg_output_tokens_per_task": 19807.05518018018, "last_run_timestamp": "2026-10-24T08:41:57.620Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.84624, "accuracy_std_dev": 0.02002973311521316, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.043265813131313134, "avg_duration_per_task_ms": 121558.87037037036, "avg_output_tokens_per_task": 34452.29797979798, "last_run_timestamp": "2026-10-24T08:13:28.067Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.8458658888888888, "accuracy_std_dev": 0.043929875719895144, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.003920089225589225, "avg_duration_per_task_ms": 538039.7317620651, "avg_output_tokens_per_task": 39775.50785634119, "last_run_timestamp": "2026-10-24T16:14:40.610Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.8456602006079027, "accuracy_std_dev": 0.016336342615469363, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.020697631205673757, "avg_duration_per_task_ms": 324636.22441742657, "avg_output_tokens_per_task": 41178.20871327254, "last_run_timestamp": "2026-10-24T08:56:29.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.8455742847457628, "accuracy_std_dev": 0.019722621271080296, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.10396145480225989, "avg_duration_per_task_ms": 199057.26101694914, "avg_output_tokens_per_task": 30600.066666666666, "last_run_timestamp": "2026-10-24T08:23:31.990Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8455386666666667, "accuracy_std_dev": 0.01527369648193331, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.020861409090909087, "avg_duration_per_task_ms": 420940.80723905726, "avg_output_tokens_per_task": 41492.801346801345, "last_run_timestamp": "2026-10-24T08:13:28.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.8453048148148149, "accuracy_std_dev": 0.010350814475153139, "run_count": 27, "total_questions": 5346, "avg_cost_per_task_usd": 0.1469017957351291, "avg_duration_per_task_ms": 178191.52618780397, "avg_output_tokens_per_task": 32277.341937897494, "last_run_timestamp": "2026-10-30T16:28:33.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.845217870886076, "accuracy_std_dev": 0.011235203598016757, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.018473258227848097, "avg_duration_per_task_ms": 138420.12109704642, "avg_output_tokens_per_task": 18520.817721518986, "last_run_timestamp": "2026-10-28T04:49:31.929Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8449648181818183, "accuracy_std_dev": 0.05078720577757961, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.16265579063360883, "avg_duration_per_task_ms": 162118.34848484848, "avg_output_tokens_per_task": 16092.63269054178, "last_run_timestamp": "2026-10-24T08:18:32.507Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.844913105281815, "accuracy_std_dev": 0.019516578349245098, "run_count": 15, "total_questions": 2821, "avg_cost_per_task_usd": 0.13749859695143568, "avg_duration_per_task_ms": 278207.19319390284, "avg_output_tokens_per_task": 28367.055654023396, "last_run_timestamp": "2026-10-24T08:44:43.305Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.8448274942528734, "accuracy_std_dev": 0.009144082018821893, "run_count": 11, "total_questions": 1914, "avg_cost_per_task_usd": 0.12633844566353186, "avg_duration_per_task_ms": 154602.84848484848, "avg_output_tokens_per_task": 31467.47910135841, "last_run_timestamp": "2026-10-29T23:21:54.785Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8448157507598784, "accuracy_std_dev": 0.02762329063670728, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.013124052684903749, "avg_duration_per_task_ms": 664201.0739614995, "avg_output_tokens_per_task": 40826.44782168186, "last_run_timestamp": "2026-10-24T08:41:23.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.8445204193548387, "accuracy_std_dev": 0.018404855556345665, "run_count": 31, "total_questions": 6138, "avg_cost_per_task_usd": 0.2421819868035191, "avg_duration_per_task_ms": 500404.2896709026, "avg_output_tokens_per_task": 47667.435158031934, "last_run_timestamp": "2026-10-28T04:01:19.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.8443693333333332, "accuracy_std_dev": 0.01866431008338881, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1411051307519641, "avg_duration_per_task_ms": 894934.3092031425, "avg_output_tokens_per_task": 55451.94893378227, "last_run_timestamp": "2026-10-24T16:03:25.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.844276, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.036391522727272724, "avg_duration_per_task_ms": 53072.40404040404, "avg_output_tokens_per_task": 14468.015151515152, "last_run_timestamp": "2026-10-01T19:12:26.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8441825555555555, "accuracy_std_dev": 0.02302992819070732, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.014299253647586982, "avg_duration_per_task_ms": 527298.7811447812, "avg_output_tokens_per_task": 38122.5847362514, "last_run_timestamp": "2026-10-24T08:42:05.088Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.8440474857142857, "accuracy_std_dev": 0.0037822104436426945, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.11554378775510205, "avg_duration_per_task_ms": 243142.18265306123, "avg_output_tokens_per_task": 31758.279591836734, "last_run_timestamp": "2026-10-24T08:42:05.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8440043168316831, "accuracy_std_dev": 0.006860781750965057, "run_count": 8, "total_questions": 1515, "avg_cost_per_task_usd": 0.05332531353135314, "avg_duration_per_task_ms": 266581.80264026404, "avg_output_tokens_per_task": 22128.052145214522, "last_run_timestamp": "2026-09-26T09:50:39.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.843855125, "accuracy_std_dev": 0.007974456116838023, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1439108863636364, "avg_duration_per_task_ms": 148669.27525252526, "avg_output_tokens_per_task": 32529.41856060606, "last_run_timestamp": "2026-10-24T08:10:23.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.8438239857868021, "accuracy_std_dev": 0.012941643289783599, "run_count": 5, "total_questions": 985, "avg_cost_per_task_usd": 0.272832538071066, "avg_duration_per_task_ms": 511204.8578680203, "avg_output_tokens_per_task": 61801.45989847716, "last_run_timestamp": "2026-10-24T16:14:04.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8435453974862529, "accuracy_std_dev": 0.029302991838613988, "run_count": 7, "total_questions": 1273, "avg_cost_per_task_usd": 0.09004028358208956, "avg_duration_per_task_ms": 1451205.2136684996, "avg_output_tokens_per_task": 41372.553024351924, "last_run_timestamp": "2026-10-24T16:07:10.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.8434345000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.020032184343434345, "avg_duration_per_task_ms": 751978.8939393939, "avg_output_tokens_per_task": 23503.446969696968, "last_run_timestamp": "2026-08-15T14:57:08.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.8434342499999999, "accuracy_std_dev": 0.01044584279450443, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.08852940214646465, "avg_duration_per_task_ms": 1063581.5290404041, "avg_output_tokens_per_task": 44778.66351010101, "last_run_timestamp": "2026-10-24T16:33:50.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8434174840336134, "accuracy_std_dev": 0.0058964576706039785, "run_count": 7, "total_questions": 1190, "avg_cost_per_task_usd": 0.07875789159663865, "avg_duration_per_task_ms": 236163.5705882353, "avg_output_tokens_per_task": 19324.579831932773, "last_run_timestamp": "2026-10-08T22:17:54.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.8427474515050167, "accuracy_std_dev": 0.014940270952109593, "run_count": 14, "total_questions": 2691, "avg_cost_per_task_usd": 0.021943730955035302, "avg_duration_per_task_ms": 258715.1181716834, "avg_output_tokens_per_task": 37901.94314381271, "last_run_timestamp": "2026-10-24T08:49:23.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8425437868421053, "accuracy_std_dev": 0.00573439306339873, "run_count": 6, "total_questions": 1140, "avg_cost_per_task_usd": 0.08756267807017544, "avg_duration_per_task_ms": 282086.36140350875, "avg_output_tokens_per_task": 21480.616666666665, "last_run_timestamp": "2026-10-01T16:06:12.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8424523333333334, "accuracy_std_dev": 0.03339142982446578, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.05276241708754208, "avg_duration_per_task_ms": 208964.68855218854, "avg_output_tokens_per_task": 21882.910353535353, "last_run_timestamp": "2026-10-24T16:02:47.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.8423346063291139, "accuracy_std_dev": 0.054296645253950386, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.07867724514767932, "avg_duration_per_task_ms": 564357.3624472574, "avg_output_tokens_per_task": 43556.894092827, "last_run_timestamp": "2026-10-24T16:27:11.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.842279922077922, "accuracy_std_dev": 0.020944075957997636, "run_count": 8, "total_questions": 1155, "avg_cost_per_task_usd": 0.017810632034632037, "avg_duration_per_task_ms": 95543.98354978354, "avg_output_tokens_per_task": 15283.030303030304, "last_run_timestamp": "2026-10-24T16:27:17.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.8419939695817491, "accuracy_std_dev": 0.05773343437894112, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.08744185044359949, "avg_duration_per_task_ms": 697734.2648922687, "avg_output_tokens_per_task": 26121.35234474018, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.841751, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06159810101010101, "avg_duration_per_task_ms": 79946.62794612795, "avg_output_tokens_per_task": 13559.319865319865, "last_run_timestamp": "2026-10-02T23:24:30.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.8417508571428571, "accuracy_std_dev": 0.04451412312458842, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09816640909090911, "avg_duration_per_task_ms": 1173545.2987012987, "avg_output_tokens_per_task": 39422.10028860029, "last_run_timestamp": "2026-10-24T16:02:28.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8415046100309946, "accuracy_std_dev": 0.012823272459877284, "run_count": 18, "total_questions": 3549, "avg_cost_per_task_usd": 0.11274710115525499, "avg_duration_per_task_ms": 322682.1735700197, "avg_output_tokens_per_task": 34551.41730064807, "last_run_timestamp": "2026-10-24T08:41:42.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.84141817433047, "accuracy_std_dev": 0.011968232237469333, "run_count": 10, "total_questions": 1979, "avg_cost_per_task_usd": 0.14348533804951996, "avg_duration_per_task_ms": 205361.71248105104, "avg_output_tokens_per_task": 32378.852956038405, "last_run_timestamp": "2026-10-24T08:12:37.564Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.8411750606060605, "accuracy_std_dev": 0.008569864104844968, "run_count": 7, "total_questions": 1254, "avg_cost_per_task_usd": 0.01576657097288676, "avg_duration_per_task_ms": 669466.4673046252, "avg_output_tokens_per_task": 16726.208931419456, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8409089999999999, "accuracy_std_dev": 0.010943000000000036, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03708839646464646, "avg_duration_per_task_ms": 47515.843434343435, "avg_output_tokens_per_task": 14746.045454545454, "last_run_timestamp": "2026-10-01T19:13:29.958Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.840488, "accuracy_std_dev": 0.00819350402758187, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03994661426767677, "avg_duration_per_task_ms": 52435.7619949495, "avg_output_tokens_per_task": 14777.814393939394, "last_run_timestamp": "2026-10-01T19:13:57.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.8398882325043376, "accuracy_std_dev": 0.015516849481773015, "run_count": 10, "total_questions": 1729, "avg_cost_per_task_usd": 0.14290184152689414, "avg_duration_per_task_ms": 273124.987854251, "avg_output_tokens_per_task": 33153.25390399075, "last_run_timestamp": "2026-10-24T08:53:46.038Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.839837363066202, "accuracy_std_dev": 0.014816504049501316, "run_count": 15, "total_questions": 2870, "avg_cost_per_task_usd": 0.07680571289198605, "avg_duration_per_task_ms": 389476.49477351917, "avg_output_tokens_per_task": 22208.951219512193, "last_run_timestamp": "2026-10-24T08:10:23.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.8398090817995909, "accuracy_std_dev": 0.059804520823169324, "run_count": 3, "total_questions": 489, "avg_cost_per_task_usd": 0.019161192229038854, "avg_duration_per_task_ms": 228340.47034764825, "avg_output_tokens_per_task": 18072.97955010225, "last_run_timestamp": "2026-10-17T16:05:00.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.8397870000000001, "accuracy_std_dev": 0.06305207943174174, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013501392255892255, "avg_duration_per_task_ms": 269245.1767676768, "avg_output_tokens_per_task": 24293.242424242424, "last_run_timestamp": "2026-10-01T17:44:47.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.839646375, "accuracy_std_dev": 0.01029199634348825, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006103154040404041, "avg_duration_per_task_ms": 431518.17424242425, "avg_output_tokens_per_task": 25837.31818181818, "last_run_timestamp": "2026-10-24T16:03:25.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8396166498586238, "accuracy_std_dev": 0.03493831082667691, "run_count": 11, "total_questions": 2122, "avg_cost_per_task_usd": 0.029913213949104622, "avg_duration_per_task_ms": 288209.38784165884, "avg_output_tokens_per_task": 9239.83930254477, "last_run_timestamp": "2026-10-24T08:08:54.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.8391320000000001, "accuracy_std_dev": 0.025876072714210524, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.15949684006734002, "avg_duration_per_task_ms": 973112.1127946128, "avg_output_tokens_per_task": 52412.06341189674, "last_run_timestamp": "2026-10-24T16:05:13.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.8389520859671303, "accuracy_std_dev": 0.014418680686632029, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.22912267003792672, "avg_duration_per_task_ms": 279595.97260851244, "avg_output_tokens_per_task": 51927.226717235564, "last_run_timestamp": "2026-10-24T16:04:55.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.8389448333333334, "accuracy_std_dev": 0.03599784248820413, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004737409090909091, "avg_duration_per_task_ms": 1034495.3484848485, "avg_output_tokens_per_task": 27462.325757575756, "last_run_timestamp": "2026-10-24T16:19:29.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.8385887456445994, "accuracy_std_dev": 0.01177027910963325, "run_count": 29, "total_questions": 5740, "avg_cost_per_task_usd": 0.2625816120209059, "avg_duration_per_task_ms": 276081.5918118467, "avg_output_tokens_per_task": 57083.71358885017, "last_run_timestamp": "2026-10-30T16:10:59.813Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.838522452928095, "accuracy_std_dev": 0.010501612316687394, "run_count": 8, "total_questions": 1349, "avg_cost_per_task_usd": 0.006855791697553743, "avg_duration_per_task_ms": 622066.7916975537, "avg_output_tokens_per_task": 16928.25722757598, "last_run_timestamp": "2026-10-24T16:21:33.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Modal", "accuracy": 0.8383968759493671, "accuracy_std_dev": 0.006763999999999992, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.033454222784810125, "avg_duration_per_task_ms": 81357.47848101266, "avg_output_tokens_per_task": 14534.177215189873, "last_run_timestamp": "2026-10-24T09:12:44.274Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.8378226666666666, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.15828378956228956, "avg_duration_per_task_ms": 446608.7861952862, "avg_output_tokens_per_task": 56298.93097643097, "last_run_timestamp": "2026-10-10T08:30:28.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.8377524999999999, "accuracy_std_dev": 0.05290935711667643, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09474105176767675, "avg_duration_per_task_ms": 401286.8162878788, "avg_output_tokens_per_task": 21287.625, "last_run_timestamp": "2026-10-24T16:06:30.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.8377103, "accuracy_std_dev": 0.023183391999662194, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.10627496717171718, "avg_duration_per_task_ms": 386419.75505050505, "avg_output_tokens_per_task": 35856.45909090909, "last_run_timestamp": "2026-10-24T08:25:29.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.837692958038147, "accuracy_std_dev": 0.020571358270420545, "run_count": 10, "total_questions": 1835, "avg_cost_per_task_usd": 0.07314428610354223, "avg_duration_per_task_ms": 380364.9722070845, "avg_output_tokens_per_task": 24128.798910081743, "last_run_timestamp": "2026-10-24T16:08:29.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.8376859309031556, "accuracy_std_dev": 0.027367300407603226, "run_count": 10, "total_questions": 1838, "avg_cost_per_task_usd": 0.011182803590859632, "avg_duration_per_task_ms": 390045.14526659413, "avg_output_tokens_per_task": 22148.7992383025, "last_run_timestamp": "2026-10-24T08:39:40.816Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.8375420000000001, "accuracy_std_dev": 0.005646742468007568, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02810060606060606, "avg_duration_per_task_ms": 530216.4090909091, "avg_output_tokens_per_task": 23231.272727272728, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.837542, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06186689898989899, "avg_duration_per_task_ms": 75089.61363636363, "avg_output_tokens_per_task": 13620.507575757576, "last_run_timestamp": "2026-10-01T19:12:44.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.8375419999999999, "accuracy_std_dev": 0.012922002863333565, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.15197074579124578, "avg_duration_per_task_ms": 226641.9739057239, "avg_output_tokens_per_task": 34291.69781144781, "last_run_timestamp": "2026-10-24T08:43:12.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.8372897388724037, "accuracy_std_dev": 0.028847852416095617, "run_count": 12, "total_questions": 2359, "avg_cost_per_task_usd": 0.06617999915218314, "avg_duration_per_task_ms": 1053779.2886816447, "avg_output_tokens_per_task": 33011.97880457821, "last_run_timestamp": "2026-10-24T16:02:06.932Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.8371210833333332, "accuracy_std_dev": 0.012487611784340083, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0845222638888889, "avg_duration_per_task_ms": 727103.742003367, "avg_output_tokens_per_task": 48945.89772727273, "last_run_timestamp": "2026-10-24T16:11:13.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.8368686, "accuracy_std_dev": 0.01362896250783603, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1397023909090909, "avg_duration_per_task_ms": 182914.46161616163, "avg_output_tokens_per_task": 31522.08787878788, "last_run_timestamp": "2026-10-24T08:24:39.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Amazon Bedrock", "accuracy": 0.8367000000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.18573769191919193, "avg_duration_per_task_ms": 281534.297979798, "avg_output_tokens_per_task": 34913.7095959596, "last_run_timestamp": "2026-10-24T08:56:15.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.8366667090476191, "accuracy_std_dev": 0.01978861195373632, "run_count": 12, "total_questions": 2100, "avg_cost_per_task_usd": 0.09574838047619047, "avg_duration_per_task_ms": 613293.1757142857, "avg_output_tokens_per_task": 29845.292857142857, "last_run_timestamp": "2026-10-24T16:10:16.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.836653454183267, "accuracy_std_dev": 0.007833999999999952, "run_count": 2, "total_questions": 251, "avg_cost_per_task_usd": 0.06977503187250997, "avg_duration_per_task_ms": 237256.35059760956, "avg_output_tokens_per_task": 17511.450199203187, "last_run_timestamp": "2026-08-15T08:47:41.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.836139, "accuracy_std_dev": 0.05678065343641852, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005011018518518518, "avg_duration_per_task_ms": 214818.47643097644, "avg_output_tokens_per_task": 17529.722222222223, "last_run_timestamp": "2026-09-26T11:06:41.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.836139, "accuracy_std_dev": 0.0007933738084913017, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05002690909090909, "avg_duration_per_task_ms": 537584.335016835, "avg_output_tokens_per_task": 49932.06734006734, "last_run_timestamp": "2026-09-30T11:44:08.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8360894826189674, "accuracy_std_dev": 0.027487401181389745, "run_count": 15, "total_questions": 2963, "avg_cost_per_task_usd": 0.02392725244684441, "avg_duration_per_task_ms": 312624.6992912589, "avg_output_tokens_per_task": 17783.592642591968, "last_run_timestamp": "2026-10-24T08:22:23.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.83599875, "accuracy_std_dev": 0.06733100699544627, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.20365952861952863, "avg_duration_per_task_ms": 537295.401936027, "avg_output_tokens_per_task": 46280.556818181816, "last_run_timestamp": "2026-10-24T16:48:27.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.8358585, "accuracy_std_dev": 0.01430950000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0011451691919191918, "avg_duration_per_task_ms": 116299.33333333333, "avg_output_tokens_per_task": 18355.219696969696, "last_run_timestamp": "2026-10-01T08:23:48.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.8354376249999999, "accuracy_std_dev": 0.020682897348639886, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09533192992424241, "avg_duration_per_task_ms": 1095682.7007575757, "avg_output_tokens_per_task": 43690.37752525252, "last_run_timestamp": "2026-10-24T16:12:09.457Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.835384616923077, "accuracy_std_dev": 0.032642666915557006, "run_count": 5, "total_questions": 650, "avg_cost_per_task_usd": 0.09615046153846153, "avg_duration_per_task_ms": 334242.9476923077, "avg_output_tokens_per_task": 39819.60307692308, "last_run_timestamp": "2026-10-24T16:40:24.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.8353175277777779, "accuracy_std_dev": 0.01836981616741691, "run_count": 9, "total_questions": 1764, "avg_cost_per_task_usd": 0.014329623015873016, "avg_duration_per_task_ms": 445894.7193877551, "avg_output_tokens_per_task": 39122.69671201814, "last_run_timestamp": "2026-10-24T08:17:53.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.8351849999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 180, "avg_cost_per_task_usd": 0.0030483999999999997, "avg_duration_per_task_ms": 288802.22222222225, "avg_output_tokens_per_task": 16703.5, "last_run_timestamp": "2026-10-01T08:43:11.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8349678663276426, "accuracy_std_dev": 0.012269176788711802, "run_count": 17, "total_questions": 2753, "avg_cost_per_task_usd": 0.11747415800944423, "avg_duration_per_task_ms": 293233.16164184525, "avg_output_tokens_per_task": 27740.3185615692, "last_run_timestamp": "2026-10-28T12:31:58.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.834789191724138, "accuracy_std_dev": 0.03295145245372225, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.10863842022988504, "avg_duration_per_task_ms": 1052937.216091954, "avg_output_tokens_per_task": 33937.42436781609, "last_run_timestamp": "2026-10-24T16:17:57.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.8344555333333333, "accuracy_std_dev": 0.011296009695856743, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.24910045925925925, "avg_duration_per_task_ms": 387517.9037037037, "avg_output_tokens_per_task": 40180.79898989899, "last_run_timestamp": "2026-10-28T04:11:19.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.8343080263157894, "accuracy_std_dev": 0.018459495220003813, "run_count": 16, "total_questions": 3078, "avg_cost_per_task_usd": 0.260129997725796, "avg_duration_per_task_ms": 393015.818388564, "avg_output_tokens_per_task": 45642.69135802469, "last_run_timestamp": "2026-10-24T16:03:06.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8341984152249134, "accuracy_std_dev": 0.029002188381032377, "run_count": 3, "total_questions": 578, "avg_cost_per_task_usd": 0.005492470588235294, "avg_duration_per_task_ms": 956260.6712802768, "avg_output_tokens_per_task": 15404.778546712803, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.8340433944004868, "accuracy_std_dev": 0.008747099804372552, "run_count": 9, "total_questions": 1643, "avg_cost_per_task_usd": 0.04404786731588557, "avg_duration_per_task_ms": 94664.89896530737, "avg_output_tokens_per_task": 15175.068167985393, "last_run_timestamp": "2026-10-24T08:10:23.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.8340364430379746, "accuracy_std_dev": 0.01069546220626094, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0954248565400844, "avg_duration_per_task_ms": 293770.05400843883, "avg_output_tokens_per_task": 23403.932489451476, "last_run_timestamp": "2026-10-01T16:11:05.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8336574603112842, "accuracy_std_dev": 0.021426026057470033, "run_count": 13, "total_questions": 2570, "avg_cost_per_task_usd": 0.024801823735408558, "avg_duration_per_task_ms": 393313.6081712062, "avg_output_tokens_per_task": 46122.12840466926, "last_run_timestamp": "2026-10-24T08:48:13.246Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.8334735000000002, "accuracy_std_dev": 0.03349024054482143, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.05413035648148148, "avg_duration_per_task_ms": 251545.94865319866, "avg_output_tokens_per_task": 23002.68813131313, "last_run_timestamp": "2026-10-24T16:06:30.902Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.833193, "accuracy_std_dev": 0.011323617332813778, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03877796464646465, "avg_duration_per_task_ms": 229512.37962962964, "avg_output_tokens_per_task": 14499.016414141413, "last_run_timestamp": "2026-10-28T03:33:49.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8330725023474179, "accuracy_std_dev": 0.010168177287347969, "run_count": 7, "total_questions": 1278, "avg_cost_per_task_usd": 0.029710525039123633, "avg_duration_per_task_ms": 655555.5469483568, "avg_output_tokens_per_task": 42323.18544600939, "last_run_timestamp": "2026-10-03T09:40:53.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.8327628225898459, "accuracy_std_dev": 0.014159242972544667, "run_count": 9, "total_questions": 1753, "avg_cost_per_task_usd": 0.15039315516257845, "avg_duration_per_task_ms": 277652.9520821449, "avg_output_tokens_per_task": 33995.16029663434, "last_run_timestamp": "2026-10-24T08:12:02.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.832178375487224, "accuracy_std_dev": 0.011051436981414083, "run_count": 12, "total_questions": 2309, "avg_cost_per_task_usd": 0.2290666569943699, "avg_duration_per_task_ms": 296861.6738847986, "avg_output_tokens_per_task": 51892.169770463406, "last_run_timestamp": "2026-10-24T16:31:42.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8320383846153845, "accuracy_std_dev": 0.008064443483478893, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.07931912276612277, "avg_duration_per_task_ms": 701901.0648795649, "avg_output_tokens_per_task": 57457.6013986014, "last_run_timestamp": "2026-10-24T16:24:24.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.8319864, "accuracy_std_dev": 0.01402072559605957, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.06441977171717173, "avg_duration_per_task_ms": 117114.99292929293, "avg_output_tokens_per_task": 13401.907070707071, "last_run_timestamp": "2026-10-01T19:14:05.661Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8318702952978058, "accuracy_std_dev": 0.01302850195570758, "run_count": 9, "total_questions": 1595, "avg_cost_per_task_usd": 0.21279191097178685, "avg_duration_per_task_ms": 458880.45830721, "avg_output_tokens_per_task": 48250.53855799373, "last_run_timestamp": "2026-10-28T11:00:59.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.83176317025572, "accuracy_std_dev": 0.014297695836545667, "run_count": 8, "total_questions": 1486, "avg_cost_per_task_usd": 0.023221829744279946, "avg_duration_per_task_ms": 299260.8001345895, "avg_output_tokens_per_task": 46221.84320323015, "last_run_timestamp": "2026-10-24T08:22:23.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.8316682557442558, "accuracy_std_dev": 0.01526659275642514, "run_count": 13, "total_questions": 2002, "avg_cost_per_task_usd": 0.039858755744255736, "avg_duration_per_task_ms": 286266.92507492506, "avg_output_tokens_per_task": 31778.31018981019, "last_run_timestamp": "2026-10-24T16:05:40.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.83143925, "accuracy_std_dev": 0.015980774330034836, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1378152821969697, "avg_duration_per_task_ms": 293606.83648989897, "avg_output_tokens_per_task": 56806.145833333336, "last_run_timestamp": "2026-10-24T16:04:33.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.8313810512445096, "accuracy_std_dev": 0.012942947508356888, "run_count": 7, "total_questions": 1366, "avg_cost_per_task_usd": 0.027591674963396776, "avg_duration_per_task_ms": 601596.977306003, "avg_output_tokens_per_task": 14650.56149341142, "last_run_timestamp": "2026-10-24T08:48:04.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.8312629347826086, "accuracy_std_dev": 0.044536486139877485, "run_count": 4, "total_questions": 644, "avg_cost_per_task_usd": 0.005694647515527951, "avg_duration_per_task_ms": 270388.4906832298, "avg_output_tokens_per_task": 13521.194099378881, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.83122875, "accuracy_std_dev": 0.007839878231675557, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022249369949494953, "avg_duration_per_task_ms": 11972.136363636364, "avg_output_tokens_per_task": 14384.287878787878, "last_run_timestamp": "2026-10-24T08:21:05.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.8311627944186045, "accuracy_std_dev": 0.006786821620057476, "run_count": 11, "total_questions": 2150, "avg_cost_per_task_usd": 0.03491511023255814, "avg_duration_per_task_ms": 201279.6446511628, "avg_output_tokens_per_task": 14261.299069767441, "last_run_timestamp": "2026-10-24T08:41:23.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8311506707700367, "accuracy_std_dev": 0.02069707210422527, "run_count": 20, "total_questions": 3818, "avg_cost_per_task_usd": 0.04166823703509692, "avg_duration_per_task_ms": 443524.7218438973, "avg_output_tokens_per_task": 45446.65243583028, "last_run_timestamp": "2026-10-30T00:41:07.469Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8310956896816264, "accuracy_std_dev": 0.010867420886961873, "run_count": 16, "total_questions": 2607, "avg_cost_per_task_usd": 0.020108517453011125, "avg_duration_per_task_ms": 239467.13693901035, "avg_output_tokens_per_task": 40022.99923283468, "last_run_timestamp": "2026-10-24T08:13:46.749Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.8309013333333333, "accuracy_std_dev": 0.004566576227814942, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.048388571829405154, "avg_duration_per_task_ms": 267954.62850729516, "avg_output_tokens_per_task": 15225.407968574635, "last_run_timestamp": "2026-10-29T22:38:23.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.830874032193159, "accuracy_std_dev": 0.029241810305408605, "run_count": 8, "total_questions": 1491, "avg_cost_per_task_usd": 0.00674413145539906, "avg_duration_per_task_ms": 717174.3158953722, "avg_output_tokens_per_task": 18114.10865191147, "last_run_timestamp": "2026-10-24T16:05:36.255Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8306676666666667, "accuracy_std_dev": 0.02403995285052688, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.13645198905723904, "avg_duration_per_task_ms": 551462.0761784512, "avg_output_tokens_per_task": 43445.10816498317, "last_run_timestamp": "2026-10-24T16:02:47.376Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.8302375046439628, "accuracy_std_dev": 0.02221590376784662, "run_count": 3, "total_questions": 323, "avg_cost_per_task_usd": 0.10860701238390094, "avg_duration_per_task_ms": 289034.8173374613, "avg_output_tokens_per_task": 24728.442724458204, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.8301606153846154, "accuracy_std_dev": 0.025057780433164603, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.038417987567987574, "avg_duration_per_task_ms": 349233.05205905205, "avg_output_tokens_per_task": 39432.61577311577, "last_run_timestamp": "2026-10-24T16:01:48.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8299001138748543, "accuracy_std_dev": 0.01880026569622147, "run_count": 13, "total_questions": 2573, "avg_cost_per_task_usd": 0.2475731441896619, "avg_duration_per_task_ms": 394929.43801010493, "avg_output_tokens_per_task": 56101.092499028375, "last_run_timestamp": "2026-10-28T11:16:13.363Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8295452499999999, "accuracy_std_dev": 0.02345987124192076, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07824036237373737, "avg_duration_per_task_ms": 700469.6212121212, "avg_output_tokens_per_task": 42273.732323232325, "last_run_timestamp": "2026-10-04T00:07:08.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8293709188811189, "accuracy_std_dev": 0.009670510440897125, "run_count": 4, "total_questions": 715, "avg_cost_per_task_usd": 0.025556813986013987, "avg_duration_per_task_ms": 118281.40559440559, "avg_output_tokens_per_task": 20315.50909090909, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.8288160468661773, "accuracy_std_dev": 0.05013602361575955, "run_count": 9, "total_questions": 1771, "avg_cost_per_task_usd": 0.1908068712591756, "avg_duration_per_task_ms": 608860.6832298137, "avg_output_tokens_per_task": 45412.08865047996, "last_run_timestamp": "2026-10-28T04:00:18.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.828497570984456, "accuracy_std_dev": 0.015230029187102692, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.11257541554404144, "avg_duration_per_task_ms": 438124.0186528497, "avg_output_tokens_per_task": 31261.672538860104, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8284027318022578, "accuracy_std_dev": 0.02855983228366381, "run_count": 13, "total_questions": 2569, "avg_cost_per_task_usd": 0.08278847567146748, "avg_duration_per_task_ms": 354761.10003892565, "avg_output_tokens_per_task": 20569.06267029973, "last_run_timestamp": "2026-10-28T03:29:22.918Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.82828275, "accuracy_std_dev": 0.004610363346581274, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0223316047979798, "avg_duration_per_task_ms": 615131.3409090909, "avg_output_tokens_per_task": 56159.15404040404, "last_run_timestamp": "2026-10-17T08:18:07.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8279138147973608, "accuracy_std_dev": 0.02296828124369033, "run_count": 11, "total_questions": 2122, "avg_cost_per_task_usd": 0.021867254476908574, "avg_duration_per_task_ms": 225918.76531573987, "avg_output_tokens_per_task": 43510.79641847314, "last_run_timestamp": "2026-10-28T10:43:17.083Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8276692169459963, "accuracy_std_dev": 0.017541679518523267, "run_count": 11, "total_questions": 2148, "avg_cost_per_task_usd": 0.14409020577281192, "avg_duration_per_task_ms": 292895.9767225326, "avg_output_tokens_per_task": 32531.135009310987, "last_run_timestamp": "2026-10-24T08:12:02.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.827651625, "accuracy_std_dev": 0.00789359865551672, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04827323547979799, "avg_duration_per_task_ms": 460754.56123737374, "avg_output_tokens_per_task": 26739.357954545456, "last_run_timestamp": "2026-10-24T16:31:34.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.827440875, "accuracy_std_dev": 0.011355911736156446, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008075719065656566, "avg_duration_per_task_ms": 508001.31565656565, "avg_output_tokens_per_task": 20787.5827020202, "last_run_timestamp": "2026-10-24T16:12:11.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.8273709166666667, "accuracy_std_dev": 0.056014389335030584, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.051560792508417504, "avg_duration_per_task_ms": 531412.6561447812, "avg_output_tokens_per_task": 24685.848905723906, "last_run_timestamp": "2026-10-24T16:11:29.497Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.8273257868177137, "accuracy_std_dev": 0.013461365335284555, "run_count": 10, "total_questions": 1942, "avg_cost_per_task_usd": 0.0351326035015448, "avg_duration_per_task_ms": 424285.9001029866, "avg_output_tokens_per_task": 48592.330072090626, "last_run_timestamp": "2026-10-17T08:18:01.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.8272187207100591, "accuracy_std_dev": 0.0496186451996, "run_count": 13, "total_questions": 2535, "avg_cost_per_task_usd": 0.10137412702169626, "avg_duration_per_task_ms": 538485.309270217, "avg_output_tokens_per_task": 29237.781459566075, "last_run_timestamp": "2026-10-24T08:19:59.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.8269358, "accuracy_std_dev": 0.006767606146932625, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.04578402121212121, "avg_duration_per_task_ms": 60291.652525252524, "avg_output_tokens_per_task": 11285.754545454545, "last_run_timestamp": "2026-09-26T09:19:03.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8268798333333333, "accuracy_std_dev": 0.014166079426064071, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.06329198569023568, "avg_duration_per_task_ms": 528541.8636363636, "avg_output_tokens_per_task": 49829.92003367003, "last_run_timestamp": "2026-10-24T16:07:10.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.8268397142857142, "accuracy_std_dev": 0.013924120672049498, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09914931746031745, "avg_duration_per_task_ms": 361736.2164502164, "avg_output_tokens_per_task": 45492.343434343435, "last_run_timestamp": "2026-10-17T16:06:16.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8267677, "accuracy_std_dev": 0.010999569410208776, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.03662119747474748, "avg_duration_per_task_ms": 290125.5409090909, "avg_output_tokens_per_task": 15502.71515151515, "last_run_timestamp": "2026-10-24T08:13:28.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.82659925, "accuracy_std_dev": 0.019009280711996968, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02540684343434343, "avg_duration_per_task_ms": 237042.44823232322, "avg_output_tokens_per_task": 21223.366161616163, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8264543613323679, "accuracy_std_dev": 0.018188360323715465, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.20616153874004342, "avg_duration_per_task_ms": 315117.1817523534, "avg_output_tokens_per_task": 46875.9225199131, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8264309000000002, "accuracy_std_dev": 0.009654922303674958, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04312889949494949, "avg_duration_per_task_ms": 229184.7, "avg_output_tokens_per_task": 14260.256565656566, "last_run_timestamp": "2026-10-24T08:44:44.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.8262931818181818, "accuracy_std_dev": 0.009811627071241754, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.14266403719008267, "avg_duration_per_task_ms": 193077.25711662075, "avg_output_tokens_per_task": 35369.469696969696, "last_run_timestamp": "2026-10-28T11:04:39.923Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.8259678749999999, "accuracy_std_dev": 0.013440054505446589, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.19472516477272722, "avg_duration_per_task_ms": 963909.0416666666, "avg_output_tokens_per_task": 51831.05113636364, "last_run_timestamp": "2026-10-24T16:12:49.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8258736540920717, "accuracy_std_dev": 0.01098913364419598, "run_count": 8, "total_questions": 1564, "avg_cost_per_task_usd": 0.00797793094629156, "avg_duration_per_task_ms": 338576.25767263427, "avg_output_tokens_per_task": 19235.87084398977, "last_run_timestamp": "2026-10-24T16:01:08.902Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cohere", "accuracy": 0.8257575, "accuracy_std_dev": 0.017676499999999984, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02807809090909091, "avg_duration_per_task_ms": 230919.77777777778, "avg_output_tokens_per_task": 55916.378787878784, "last_run_timestamp": "2026-10-24T08:15:10.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.8257574999999999, "accuracy_std_dev": 0.015993499999999994, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2609003181818182, "avg_duration_per_task_ms": 231557.84848484848, "avg_output_tokens_per_task": 25997.717171717173, "last_run_timestamp": "2026-10-01T19:12:42.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Darkbloom", "accuracy": 0.8257141371428571, "accuracy_std_dev": 0.01176326794163367, "run_count": 3, "total_questions": 525, "avg_cost_per_task_usd": 0.010625175238095239, "avg_duration_per_task_ms": 1617186.0876190476, "avg_output_tokens_per_task": 30239.152380952382, "last_run_timestamp": "2026-10-30T05:26:04.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.8253999912733042, "accuracy_std_dev": 0.022060427402287506, "run_count": 13, "total_questions": 2521, "avg_cost_per_task_usd": 0.027058457754859183, "avg_duration_per_task_ms": 873701.1269337564, "avg_output_tokens_per_task": 48650.64220547402, "last_run_timestamp": "2026-10-24T08:26:00.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.8253968696428573, "accuracy_std_dev": 0.014772865576229071, "run_count": 9, "total_questions": 1680, "avg_cost_per_task_usd": 0.024977942261904765, "avg_duration_per_task_ms": 272190.5416666667, "avg_output_tokens_per_task": 49745.9630952381, "last_run_timestamp": "2026-10-24T08:27:40.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.8252562970922881, "accuracy_std_dev": 0.043641090936944585, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.08621131184155077, "avg_duration_per_task_ms": 759710.0198061526, "avg_output_tokens_per_task": 31352.24863042562, "last_run_timestamp": "2026-10-24T16:13:03.130Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.8246474711042312, "accuracy_std_dev": 0.04610697461436829, "run_count": 10, "total_questions": 1938, "avg_cost_per_task_usd": 0.019926908668730647, "avg_duration_per_task_ms": 787644.0030959753, "avg_output_tokens_per_task": 45616.08255933953, "last_run_timestamp": "2026-10-24T08:22:23.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8244948000000002, "accuracy_std_dev": 0.0601779527199123, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.028505976262626263, "avg_duration_per_task_ms": 700158.9757575757, "avg_output_tokens_per_task": 29700.529292929292, "last_run_timestamp": "2026-10-17T08:18:04.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.8243546666666667, "accuracy_std_dev": 0.03111308863734931, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.25956171212121215, "avg_duration_per_task_ms": 233681.0909090909, "avg_output_tokens_per_task": 25863.887205387204, "last_run_timestamp": "2026-10-01T19:17:25.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.8240257172774869, "accuracy_std_dev": 0.000479814547507715, "run_count": 3, "total_questions": 573, "avg_cost_per_task_usd": 0.1285148184991274, "avg_duration_per_task_ms": 295843.96684118675, "avg_output_tokens_per_task": 31965.242582897034, "last_run_timestamp": "2026-10-24T08:50:12.983Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8239975454545455, "accuracy_std_dev": 0.022862830634275574, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.010785916437098255, "avg_duration_per_task_ms": 717852.0280073462, "avg_output_tokens_per_task": 38282.08126721763, "last_run_timestamp": "2026-10-17T08:02:36.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.8239540000000001, "accuracy_std_dev": 0.009474601205327877, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.14372687085137084, "avg_duration_per_task_ms": 420021.29581529583, "avg_output_tokens_per_task": 28768.17316017316, "last_run_timestamp": "2026-10-24T08:48:18.601Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.8236529999999999, "accuracy_std_dev": 0.0034448118235979193, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.018703443181818182, "avg_duration_per_task_ms": 518705.38636363635, "avg_output_tokens_per_task": 18559.679292929293, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.823635741091314, "accuracy_std_dev": 0.01667549518358386, "run_count": 19, "total_questions": 3592, "avg_cost_per_task_usd": 0.03049394097995545, "avg_duration_per_task_ms": 266457.14532293985, "avg_output_tokens_per_task": 14043.167873051225, "last_run_timestamp": "2026-10-30T00:29:53.146Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.823232125, "accuracy_std_dev": 0.02934086116850314, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09752844696969695, "avg_duration_per_task_ms": 1429250.3825757576, "avg_output_tokens_per_task": 44988.86111111111, "last_run_timestamp": "2026-10-24T16:04:55.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.823232, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025912994949494946, "avg_duration_per_task_ms": 162312.898989899, "avg_output_tokens_per_task": 9036.257575757576, "last_run_timestamp": "2026-10-01T23:50:45.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.82281125, "accuracy_std_dev": 0.014500513393928511, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.024772287878787876, "avg_duration_per_task_ms": 511120.71338383836, "avg_output_tokens_per_task": 23142.588383838385, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.8225588000000001, "accuracy_std_dev": 0.009908448060115158, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02341788282828283, "avg_duration_per_task_ms": 349498.4303030303, "avg_output_tokens_per_task": 20884.227272727272, "last_run_timestamp": "2026-10-04T00:17:30.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.0244105, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2415952904040404, "avg_duration_per_task_ms": 181953.0101010101, "avg_output_tokens_per_task": 24066.49494949495, "last_run_timestamp": "2026-10-01T19:12:31.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8223551976047904, "accuracy_std_dev": 0.029631373343505716, "run_count": 6, "total_questions": 1169, "avg_cost_per_task_usd": 0.06724194183062446, "avg_duration_per_task_ms": 236148.74337040205, "avg_output_tokens_per_task": 20314.131736526946, "last_run_timestamp": "2026-10-17T08:26:17.960Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.8223140000000001, "accuracy_std_dev": 0.020822165248512558, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.012249293847566574, "avg_duration_per_task_ms": 720758.7061524334, "avg_output_tokens_per_task": 36235.862258953166, "last_run_timestamp": "2026-10-17T08:04:21.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.8223139999999999, "accuracy_std_dev": 0.02011718801965576, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03311863039485767, "avg_duration_per_task_ms": 262977.6432506887, "avg_output_tokens_per_task": 14545.10927456382, "last_run_timestamp": "2026-10-24T08:29:59.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8221660379266751, "accuracy_std_dev": 0.02808252331410946, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.20924120101137803, "avg_duration_per_task_ms": 851157.8482932997, "avg_output_tokens_per_task": 47928.662452591656, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.8217830000000002, "accuracy_std_dev": 0.015536946332318115, "run_count": 9, "total_questions": 1746, "avg_cost_per_task_usd": 0.018000982817869417, "avg_duration_per_task_ms": 602038.2628865979, "avg_output_tokens_per_task": 20185.35166093929, "last_run_timestamp": "2026-10-24T16:01:29.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.8214768773857721, "accuracy_std_dev": 0.011242622925586103, "run_count": 9, "total_questions": 1729, "avg_cost_per_task_usd": 0.024795357432041643, "avg_duration_per_task_ms": 445258.44707923656, "avg_output_tokens_per_task": 24754.15095430885, "last_run_timestamp": "2026-10-24T16:11:33.501Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.8214198161925602, "accuracy_std_dev": 0.04954945097072217, "run_count": 8, "total_questions": 1371, "avg_cost_per_task_usd": 0.04645164551422319, "avg_duration_per_task_ms": 608936.6484318016, "avg_output_tokens_per_task": 28638.388767323122, "last_run_timestamp": "2026-09-26T09:56:00.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.8212682499999999, "accuracy_std_dev": 0.0185418798135689, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.04273052441077441, "avg_duration_per_task_ms": 239740.20075757575, "avg_output_tokens_per_task": 14238.065656565657, "last_run_timestamp": "2026-10-24T08:50:39.623Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.8212120999999999, "accuracy_std_dev": 0.028508221310527243, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.028071248484848484, "avg_duration_per_task_ms": 579831.6848484848, "avg_output_tokens_per_task": 50141.56717171717, "last_run_timestamp": "2026-10-30T14:53:57.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.8210525603238865, "accuracy_std_dev": 0.07599126782917703, "run_count": 7, "total_questions": 1235, "avg_cost_per_task_usd": 0.007044106882591094, "avg_duration_per_task_ms": 525854.9595141701, "avg_output_tokens_per_task": 14201.662348178137, "last_run_timestamp": "2026-10-03T10:22:02.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.8207548040638606, "accuracy_std_dev": 0.12376641907798643, "run_count": 7, "total_questions": 1378, "avg_cost_per_task_usd": 0.1265669557329463, "avg_duration_per_task_ms": 747147.1386066764, "avg_output_tokens_per_task": 38728.26777939042, "last_run_timestamp": "2026-10-24T16:03:25.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8207070000000001, "accuracy_std_dev": 0.024411000000000016, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04063828030303031, "avg_duration_per_task_ms": 83296.42424242424, "avg_output_tokens_per_task": 26969.131313131315, "last_run_timestamp": "2026-10-01T19:23:01.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.8202743410372041, "accuracy_std_dev": 0.011484132116496031, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.09063781341600902, "avg_duration_per_task_ms": 168767.6234498309, "avg_output_tokens_per_task": 22236.227170236754, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.8192149207920791, "accuracy_std_dev": 0.018823182524040232, "run_count": 6, "total_questions": 909, "avg_cost_per_task_usd": 0.11224348184818483, "avg_duration_per_task_ms": 401079.6094609461, "avg_output_tokens_per_task": 46532.80198019802, "last_run_timestamp": "2026-10-17T16:06:46.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.8191437142857143, "accuracy_std_dev": 0.03986796895724072, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04288789971139971, "avg_duration_per_task_ms": 992008.3816738817, "avg_output_tokens_per_task": 26956.146464646463, "last_run_timestamp": "2026-10-17T08:20:47.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.8190235000000001, "accuracy_std_dev": 0.020635683523934946, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006401880681818182, "avg_duration_per_task_ms": 463424.9640151515, "avg_output_tokens_per_task": 18789.763257575756, "last_run_timestamp": "2026-10-24T16:31:34.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.8187429166666668, "accuracy_std_dev": 0.012566775670117427, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.31407284974747474, "avg_duration_per_task_ms": 289933.9659090909, "avg_output_tokens_per_task": 20747.42213804714, "last_run_timestamp": "2026-10-24T08:12:02.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.8183921250000001, "accuracy_std_dev": 0.03063364984146315, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.08543528156565658, "avg_duration_per_task_ms": 987445.6388888889, "avg_output_tokens_per_task": 42416.07765151515, "last_run_timestamp": "2026-10-24T16:15:14.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.8183501, "accuracy_std_dev": 0.014134440253862193, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.033925831313131316, "avg_duration_per_task_ms": 215707.2409090909, "avg_output_tokens_per_task": 13837.719191919192, "last_run_timestamp": "2026-10-24T08:16:59.511Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.81818175, "accuracy_std_dev": 0.008828586845441324, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06985739772727273, "avg_duration_per_task_ms": 843652.7916666666, "avg_output_tokens_per_task": 40039.878787878784, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.8181817499999999, "accuracy_std_dev": 0.021788216475597543, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02338400883838384, "avg_duration_per_task_ms": 497693.29671717173, "avg_output_tokens_per_task": 22657.377525252527, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.818143387341772, "accuracy_std_dev": 0.008051034079545293, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.03837104746835443, "avg_duration_per_task_ms": 216584.66329113924, "avg_output_tokens_per_task": 14732.209493670885, "last_run_timestamp": "2026-10-24T08:26:33.259Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.8178564226152875, "accuracy_std_dev": 0.005702516062011583, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.025747458622867977, "avg_duration_per_task_ms": 332030.03348073276, "avg_output_tokens_per_task": 25676.125710675933, "last_run_timestamp": "2026-10-24T16:05:36.229Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.8178077777777777, "accuracy_std_dev": 0.009999885885768644, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.024910318742985414, "avg_duration_per_task_ms": 190389.1571268238, "avg_output_tokens_per_task": 16514.1734006734, "last_run_timestamp": "2026-10-03T10:10:35.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.8177608749999999, "accuracy_std_dev": 0.007516885898387367, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.027347258838383834, "avg_duration_per_task_ms": 177583.7866161616, "avg_output_tokens_per_task": 23473.940656565657, "last_run_timestamp": "2026-10-24T16:20:59.116Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.8177376144306652, "accuracy_std_dev": 0.03151916867911054, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.013423886696730551, "avg_duration_per_task_ms": 855848.4718151072, "avg_output_tokens_per_task": 39679.556933483655, "last_run_timestamp": "2026-10-17T08:19:03.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8174469182990923, "accuracy_std_dev": 0.06189112957986377, "run_count": 22, "total_questions": 4186, "avg_cost_per_task_usd": 0.0075936266125179175, "avg_duration_per_task_ms": 503995.6545628285, "avg_output_tokens_per_task": 17382.02126134735, "last_run_timestamp": "2026-10-24T16:15:47.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.8173077377622378, "accuracy_std_dev": 0.04183386114142466, "run_count": 3, "total_questions": 572, "avg_cost_per_task_usd": 0.00918055944055944, "avg_duration_per_task_ms": 556054.2482517483, "avg_output_tokens_per_task": 20397.216783216783, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.816967559819413, "accuracy_std_dev": 0.022532849378432182, "run_count": 9, "total_questions": 1772, "avg_cost_per_task_usd": 0.021389000564334085, "avg_duration_per_task_ms": 297135.54796839727, "avg_output_tokens_per_task": 17946.336343115123, "last_run_timestamp": "2026-10-03T09:25:02.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8169191785714285, "accuracy_std_dev": 0.010832356972296234, "run_count": 28, "total_questions": 5544, "avg_cost_per_task_usd": 0.3328294617604617, "avg_duration_per_task_ms": 295647.11093073594, "avg_output_tokens_per_task": 21018.825216450216, "last_run_timestamp": "2026-10-24T08:56:14.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.8167087499999999, "accuracy_std_dev": 0.004328069568237081, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.010634625000000002, "avg_duration_per_task_ms": 790260.4886363636, "avg_output_tokens_per_task": 18570.635101010103, "last_run_timestamp": "2026-10-24T16:12:04.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.8167087499999999, "accuracy_std_dev": 0.01709464425448801, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007263619318181818, "avg_duration_per_task_ms": 512210.79671717173, "avg_output_tokens_per_task": 18196.01452020202, "last_run_timestamp": "2026-10-24T16:06:30.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.8164054780192015, "accuracy_std_dev": 0.008737950672783622, "run_count": 10, "total_questions": 1979, "avg_cost_per_task_usd": 0.03418838352703386, "avg_duration_per_task_ms": 578940.7988883274, "avg_output_tokens_per_task": 15946.312784234462, "last_run_timestamp": "2026-10-30T01:10:29.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.8161616, "accuracy_std_dev": 0.060936149095918435, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.09838498181818181, "avg_duration_per_task_ms": 774603.698989899, "avg_output_tokens_per_task": 39333.56565656565, "last_run_timestamp": "2026-10-03T17:58:22.995Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.81607725, "accuracy_std_dev": 0.011564661524986357, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.018877026515151513, "avg_duration_per_task_ms": 531080.7954545454, "avg_output_tokens_per_task": 18678.97095959596, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "PrimeIntellect", "accuracy": 0.8157141657142857, "accuracy_std_dev": 0.010726999999999987, "run_count": 2, "total_questions": 350, "avg_cost_per_task_usd": 0.00801732, "avg_duration_per_task_ms": 285410.90857142856, "avg_output_tokens_per_task": 31549.22857142857, "last_run_timestamp": "2026-10-29T23:13:31.837Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.815286480254777, "accuracy_std_dev": 0.006665997576319691, "run_count": 4, "total_questions": 785, "avg_cost_per_task_usd": 0.08723442165605096, "avg_duration_per_task_ms": 185900.4152866242, "avg_output_tokens_per_task": 24915.47388535032, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.81481475, "accuracy_std_dev": 0.005455009183081147, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006102087121212121, "avg_duration_per_task_ms": 532828.8005050505, "avg_output_tokens_per_task": 18519.08712121212, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8146800399999999, "accuracy_std_dev": 0.010528738528351803, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.36333395474747476, "avg_duration_per_task_ms": 358588.514949495, "avg_output_tokens_per_task": 22927.937777777777, "last_run_timestamp": "2026-10-24T08:44:41.412Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8143338571428571, "accuracy_std_dev": 0.014248444761532711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026212479076479073, "avg_duration_per_task_ms": 329590.34992784995, "avg_output_tokens_per_task": 17511.52380952381, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.813973, "accuracy_std_dev": 0.01262599999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04186610353535354, "avg_duration_per_task_ms": 76135.87878787878, "avg_output_tokens_per_task": 27786.689393939392, "last_run_timestamp": "2026-10-01T19:16:58.595Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.8138046, "accuracy_std_dev": 0.01492933679169976, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010332651515151515, "avg_duration_per_task_ms": 579885.2626262626, "avg_output_tokens_per_task": 12515.30505050505, "last_run_timestamp": "2026-10-24T08:57:37.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.8136272825651302, "accuracy_std_dev": 0.008773473238499465, "run_count": 3, "total_questions": 499, "avg_cost_per_task_usd": 0.0068851422845691386, "avg_duration_per_task_ms": 1090136, "avg_output_tokens_per_task": 17404.21643286573, "last_run_timestamp": "2026-08-15T10:05:32.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.8134042249778565, "accuracy_std_dev": 0.008662954935945464, "run_count": 6, "total_questions": 1129, "avg_cost_per_task_usd": 0.04263479716563331, "avg_duration_per_task_ms": 176521.81842338352, "avg_output_tokens_per_task": 16879.856510186004, "last_run_timestamp": "2026-10-30T18:58:50.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.8130656097285068, "accuracy_std_dev": 0.09667022957112152, "run_count": 9, "total_questions": 1768, "avg_cost_per_task_usd": 0.061109888574660626, "avg_duration_per_task_ms": 1704990.4128959277, "avg_output_tokens_per_task": 19426.679864253394, "last_run_timestamp": "2026-10-24T16:10:43.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.813051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.00766168253968254, "avg_duration_per_task_ms": 356938.3703703704, "avg_output_tokens_per_task": 33904.37566137566, "last_run_timestamp": "2026-10-14T18:00:03.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.812725848879248, "accuracy_std_dev": 0.09609347186495672, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.09072004772234273, "avg_duration_per_task_ms": 711729.140274765, "avg_output_tokens_per_task": 26485.660882140273, "last_run_timestamp": "2026-10-24T16:10:43.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.8127104999999999, "accuracy_std_dev": 0.020528377888912718, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.10173348611111112, "avg_duration_per_task_ms": 574816.9229797979, "avg_output_tokens_per_task": 37826.392676767675, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.8122894999999999, "accuracy_std_dev": 0.02853281357051911, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.026932902777777778, "avg_duration_per_task_ms": 507868.57196969696, "avg_output_tokens_per_task": 24489.684343434343, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.8122604367816092, "accuracy_std_dev": 0.03327589059946405, "run_count": 4, "total_questions": 783, "avg_cost_per_task_usd": 0.024626680715197958, "avg_duration_per_task_ms": 153695.50446998724, "avg_output_tokens_per_task": 14302.206896551725, "last_run_timestamp": "2026-10-24T08:18:45.292Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.812079, "accuracy_std_dev": 0.1279844797709863, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.15244413194444442, "avg_duration_per_task_ms": 1980931.956439394, "avg_output_tokens_per_task": 55523.080808080806, "last_run_timestamp": "2026-10-24T16:14:24.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.812016, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.14135672674418603, "avg_duration_per_task_ms": 305120.88372093026, "avg_output_tokens_per_task": 39843.74418604651, "last_run_timestamp": "2026-08-04T03:11:59.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.8120089333333333, "accuracy_std_dev": 0.016976505295914775, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.3404598653198653, "avg_duration_per_task_ms": 353948.01683501684, "avg_output_tokens_per_task": 22525.30909090909, "last_run_timestamp": "2026-10-24T08:55:49.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.8118685, "accuracy_std_dev": 0.0618254994258033, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.042041156565656566, "avg_duration_per_task_ms": 1127766.4457070706, "avg_output_tokens_per_task": 24710.378787878788, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.8111930073800738, "accuracy_std_dev": 0.03224095604623893, "run_count": 7, "total_questions": 813, "avg_cost_per_task_usd": 0.05198734809348094, "avg_duration_per_task_ms": 309460.76998769987, "avg_output_tokens_per_task": 20725.47355473555, "last_run_timestamp": "2026-10-24T16:02:08.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.810966857142857, "accuracy_std_dev": 0.041886993615231564, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.14839432323232324, "avg_duration_per_task_ms": 1654857.906926407, "avg_output_tokens_per_task": 48197.77489177489, "last_run_timestamp": "2026-10-24T16:00:53.339Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.8108163749999999, "accuracy_std_dev": 0.048001397940418106, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.22005288573232326, "avg_duration_per_task_ms": 1162217.8257575757, "avg_output_tokens_per_task": 50375.75252525252, "last_run_timestamp": "2026-10-24T16:06:30.902Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.8099512222222223, "accuracy_std_dev": 0.024479317441363883, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.11572864197530866, "avg_duration_per_task_ms": 766947.6907968575, "avg_output_tokens_per_task": 60198.64141414141, "last_run_timestamp": "2026-10-24T16:03:25.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.8097645, "accuracy_std_dev": 0.021885500000000002, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.044968992424242425, "avg_duration_per_task_ms": 208495.61616161617, "avg_output_tokens_per_task": 22388.290404040403, "last_run_timestamp": "2026-10-01T19:19:28.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.8094875524100994, "accuracy_std_dev": 0.03366403756751318, "run_count": 7, "total_questions": 1307, "avg_cost_per_task_usd": 0.028843539403213467, "avg_duration_per_task_ms": 492278.9755164499, "avg_output_tokens_per_task": 17132.139250191278, "last_run_timestamp": "2026-10-17T16:03:44.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8094485, "accuracy_std_dev": 0.017925139427630677, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.22312939835858592, "avg_duration_per_task_ms": 1023977.4273989899, "avg_output_tokens_per_task": 60760.55808080808, "last_run_timestamp": "2026-10-24T16:04:12.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.8093435, "accuracy_std_dev": 0.02388992446095214, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.2666720176767677, "avg_duration_per_task_ms": 969823.9217171717, "avg_output_tokens_per_task": 60642.0404040404, "last_run_timestamp": "2026-10-24T16:00:53.339Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.8091328750000001, "accuracy_std_dev": 0.024870082244724787, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.009033124368686868, "avg_duration_per_task_ms": 569112.759469697, "avg_output_tokens_per_task": 19095.31818181818, "last_run_timestamp": "2026-10-24T16:00:55.675Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.8091166153846153, "accuracy_std_dev": 0.03183613126622014, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.19830280854700855, "avg_duration_per_task_ms": 1060541.5008547008, "avg_output_tokens_per_task": 55592.3264957265, "last_run_timestamp": "2026-10-03T16:58:49.489Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8089225, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.15927496717171716, "avg_duration_per_task_ms": 42973.2904040404, "avg_output_tokens_per_task": 2992.459595959596, "last_run_timestamp": "2026-08-15T09:15:35.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.8089225, "accuracy_std_dev": 0.07491550000000002, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.023557068181818182, "avg_duration_per_task_ms": 701295.6338383838, "avg_output_tokens_per_task": 58706.843434343435, "last_run_timestamp": "2026-10-24T08:01:41.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.8083212857142857, "accuracy_std_dev": 0.009239648782970742, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.045195821067821065, "avg_duration_per_task_ms": 59529.89033189033, "avg_output_tokens_per_task": 15222.556277056277, "last_run_timestamp": "2026-10-28T04:47:40.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.8082103076923077, "accuracy_std_dev": 0.02014208867935159, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.3422723123543123, "avg_duration_per_task_ms": 356747.24864024867, "avg_output_tokens_per_task": 22654.46153846154, "last_run_timestamp": "2026-10-24T08:14:36.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.8082103076923076, "accuracy_std_dev": 0.09500960411646216, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.09292749378399379, "avg_duration_per_task_ms": 514800.35275835276, "avg_output_tokens_per_task": 34280.08508158508, "last_run_timestamp": "2026-10-28T03:53:08.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8080809999999999, "accuracy_std_dev": 0.032382769067823704, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.026336486111111113, "avg_duration_per_task_ms": 464443.5934343434, "avg_output_tokens_per_task": 21920.722222222223, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.8080704302325582, "accuracy_std_dev": 0.10379216096569623, "run_count": 13, "total_questions": 2408, "avg_cost_per_task_usd": 0.041565338870431894, "avg_duration_per_task_ms": 155784.30357142858, "avg_output_tokens_per_task": 19890.849667774088, "last_run_timestamp": "2026-10-28T03:27:17.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.8068179999999999, "accuracy_std_dev": 0.027363478050131, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.046258910353535354, "avg_duration_per_task_ms": 196125.59974747474, "avg_output_tokens_per_task": 21946.10606060606, "last_run_timestamp": "2026-10-01T19:34:33.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.806397, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04417995707070707, "avg_duration_per_task_ms": 210130.46464646465, "avg_output_tokens_per_task": 21993.772727272728, "last_run_timestamp": "2026-10-01T19:26:25.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.8059763749999999, "accuracy_std_dev": 0.01367028747811745, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.022531703914141412, "avg_duration_per_task_ms": 251092.985479798, "avg_output_tokens_per_task": 17458.415404040403, "last_run_timestamp": "2026-10-03T09:13:36.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.805345125, "accuracy_std_dev": 0.036272944774437245, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.19822186489898988, "avg_duration_per_task_ms": 832606.8560606061, "avg_output_tokens_per_task": 49537.894570707074, "last_run_timestamp": "2026-10-24T16:31:00.679Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8048667272727271, "accuracy_std_dev": 0.020423284626269588, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.33759371303948577, "avg_duration_per_task_ms": 354437.65059687785, "avg_output_tokens_per_task": 22371.59963269054, "last_run_timestamp": "2026-10-24T08:45:54.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Particle AI", "accuracy": 0.8047135000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.028701856060606063, "avg_duration_per_task_ms": 283393.26767676766, "avg_output_tokens_per_task": 57180.469696969696, "last_run_timestamp": "2026-10-28T23:02:42.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.8043401184631804, "accuracy_std_dev": 0.024791902944308233, "run_count": 5, "total_questions": 937, "avg_cost_per_task_usd": 0.03799079615795091, "avg_duration_per_task_ms": 223986.25720384205, "avg_output_tokens_per_task": 37938.2166488794, "last_run_timestamp": "2026-10-24T16:04:55.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.8042929999999999, "accuracy_std_dev": 0.028405059672530177, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04403506628787878, "avg_duration_per_task_ms": 84875.49242424243, "avg_output_tokens_per_task": 27183.9375, "last_run_timestamp": "2026-10-01T19:25:58.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.8042647569692637, "accuracy_std_dev": 0.11128768555499524, "run_count": 8, "total_questions": 1399, "avg_cost_per_task_usd": 0.04140569907076483, "avg_duration_per_task_ms": 141306.68977841316, "avg_output_tokens_per_task": 29614.428877769835, "last_run_timestamp": "2026-10-24T16:27:33.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.8040404, "accuracy_std_dev": 0.050668612985160726, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.19756500707070707, "avg_duration_per_task_ms": 1564421.486868687, "avg_output_tokens_per_task": 47084.214141414144, "last_run_timestamp": "2026-10-03T17:00:45.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.8036315714285714, "accuracy_std_dev": 0.0246877462424299, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0294750468975469, "avg_duration_per_task_ms": 317654.27489177487, "avg_output_tokens_per_task": 19508.803030303032, "last_run_timestamp": "2026-10-17T16:09:25.569Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.8028197499999999, "accuracy_std_dev": 0.04280068601304774, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.026668318181818182, "avg_duration_per_task_ms": 335428.4646464646, "avg_output_tokens_per_task": 25945.37941919192, "last_run_timestamp": "2026-10-24T16:03:25.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.8026092499999999, "accuracy_std_dev": 0.024724444537491615, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03972177714646465, "avg_duration_per_task_ms": 187520.72222222222, "avg_output_tokens_per_task": 21966.539772727272, "last_run_timestamp": "2026-10-24T16:05:36.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.8011755539419088, "accuracy_std_dev": 0.08103010676902754, "run_count": 8, "total_questions": 1446, "avg_cost_per_task_usd": 0.018679138312586448, "avg_duration_per_task_ms": 300257.92392807745, "avg_output_tokens_per_task": 39516.01521438451, "last_run_timestamp": "2026-10-24T08:32:39.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.8, "accuracy_std_dev": 0.004689918080308032, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.013948154545454547, "avg_duration_per_task_ms": 321232.3868686869, "avg_output_tokens_per_task": 14312.276767676767, "last_run_timestamp": "2026-10-04T00:00:36.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.7993280088709678, "accuracy_std_dev": 0.03778798369706964, "run_count": 10, "total_questions": 1240, "avg_cost_per_task_usd": 0.006506485483870966, "avg_duration_per_task_ms": 293916.95967741933, "avg_output_tokens_per_task": 25872.287096774195, "last_run_timestamp": "2026-10-24T16:40:33.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.798727888888889, "accuracy_std_dev": 0.10517670417967454, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.05109221941638608, "avg_duration_per_task_ms": 1276720.367003367, "avg_output_tokens_per_task": 27966.93265993266, "last_run_timestamp": "2026-10-24T16:17:39.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.7986353029239767, "accuracy_std_dev": 0.024634052579147848, "run_count": 6, "total_questions": 855, "avg_cost_per_task_usd": 0.06986718011695907, "avg_duration_per_task_ms": 1584713.8748538012, "avg_output_tokens_per_task": 39897.390643274855, "last_run_timestamp": "2026-10-24T16:26:52.183Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Io Net", "accuracy": 0.7976190000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 98, "avg_cost_per_task_usd": 0.012265704081632654, "avg_duration_per_task_ms": 285854.5612244898, "avg_output_tokens_per_task": 10545.989795918367, "last_run_timestamp": "2026-10-03T17:16:45.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7975339581632652, "accuracy_std_dev": 0.14314198310659945, "run_count": 10, "total_questions": 1960, "avg_cost_per_task_usd": 0.01657647346938776, "avg_duration_per_task_ms": 547902.756122449, "avg_output_tokens_per_task": 26479.933673469386, "last_run_timestamp": "2026-10-17T08:10:43.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.7971601298174443, "accuracy_std_dev": 0.023536053284742144, "run_count": 3, "total_questions": 493, "avg_cost_per_task_usd": 0.04863367951318459, "avg_duration_per_task_ms": 714404.5334685599, "avg_output_tokens_per_task": 21075.50304259635, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.79629625, "accuracy_std_dev": 0.01565761636353056, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.17376329545454547, "avg_duration_per_task_ms": 832021.8737373737, "avg_output_tokens_per_task": 55481.57828282828, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.7962405172932331, "accuracy_std_dev": 0.005792281993642719, "run_count": 7, "total_questions": 1330, "avg_cost_per_task_usd": 0.050177010526315784, "avg_duration_per_task_ms": 1098597.4812030075, "avg_output_tokens_per_task": 22847.669172932332, "last_run_timestamp": "2026-10-17T16:06:10.469Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.7958755, "accuracy_std_dev": 0.00853230152127785, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.014628411616161617, "avg_duration_per_task_ms": 291373.6123737374, "avg_output_tokens_per_task": 13989.849747474747, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.795735, "accuracy_std_dev": 0.01152797383180008, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0539772760942761, "avg_duration_per_task_ms": 405752.09764309763, "avg_output_tokens_per_task": 23428.74074074074, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7955037897026831, "accuracy_std_dev": 0.07014310843946839, "run_count": 7, "total_questions": 1379, "avg_cost_per_task_usd": 0.19247783321247278, "avg_duration_per_task_ms": 1684826.8375634518, "avg_output_tokens_per_task": 45136.36040609137, "last_run_timestamp": "2026-10-24T16:01:08.930Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Mistral", "accuracy": 0.7953587113924051, "accuracy_std_dev": 0.017304818120613137, "run_count": 4, "total_questions": 790, "avg_cost_per_task_usd": 0.027054707594936712, "avg_duration_per_task_ms": 235650.0012658228, "avg_output_tokens_per_task": 53875.47088607595, "last_run_timestamp": "2026-10-30T16:25:09.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7944022499999999, "accuracy_std_dev": 0.03928997983185408, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.04325068055555555, "avg_duration_per_task_ms": 495166.12626262626, "avg_output_tokens_per_task": 42549.15993265993, "last_run_timestamp": "2026-10-24T16:02:47.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "PrimeIntellect", "accuracy": 0.7939496780821917, "accuracy_std_dev": 0.012044500000000014, "run_count": 2, "total_questions": 292, "avg_cost_per_task_usd": 0.01254563698630137, "avg_duration_per_task_ms": 564085.2363013698, "avg_output_tokens_per_task": 36747.568493150684, "last_run_timestamp": "2026-10-29T23:46:30.580Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.7933882727272729, "accuracy_std_dev": 0.01128387116915042, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0045943420569329665, "avg_duration_per_task_ms": 250886.0945821855, "avg_output_tokens_per_task": 30080.480257116622, "last_run_timestamp": "2026-10-17T16:03:34.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.79335, "accuracy_std_dev": 0.008448975529613064, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013479858585858586, "avg_duration_per_task_ms": 277514.2765151515, "avg_output_tokens_per_task": 14053.487373737375, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7929315464581417, "accuracy_std_dev": 0.02021084770060026, "run_count": 11, "total_questions": 2174, "avg_cost_per_task_usd": 0.06395397148114075, "avg_duration_per_task_ms": 189643.35372585096, "avg_output_tokens_per_task": 20031.46780128795, "last_run_timestamp": "2026-10-24T08:53:01.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7920900579268293, "accuracy_std_dev": 0.011512971285033247, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.01824498475609756, "avg_duration_per_task_ms": 181543.2469512195, "avg_output_tokens_per_task": 15051.83943089431, "last_run_timestamp": "2026-10-08T22:29:11.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.7916666329787234, "accuracy_std_dev": 0.015009500000000009, "run_count": 2, "total_questions": 188, "avg_cost_per_task_usd": 0.0925983085106383, "avg_duration_per_task_ms": 133806.4414893617, "avg_output_tokens_per_task": 38480.17021276596, "last_run_timestamp": "2026-08-15T14:54:40.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.7916664999999999, "accuracy_std_dev": 0.0223775773990394, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006407165404040405, "avg_duration_per_task_ms": 353276.53661616164, "avg_output_tokens_per_task": 15555.522727272728, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7914561249999998, "accuracy_std_dev": 0.012094065181293467, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.017830758207070706, "avg_duration_per_task_ms": 61119.01325757576, "avg_output_tokens_per_task": 5789.232323232323, "last_run_timestamp": "2026-10-24T16:02:28.570Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7914427791218638, "accuracy_std_dev": 0.014961655425666921, "run_count": 12, "total_questions": 2232, "avg_cost_per_task_usd": 0.004660359318996416, "avg_duration_per_task_ms": 1335489.0049283155, "avg_output_tokens_per_task": 30608.047491039426, "last_run_timestamp": "2026-10-24T16:07:10.479Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.7911994699286443, "accuracy_std_dev": 0.008413367458990474, "run_count": 5, "total_questions": 981, "avg_cost_per_task_usd": 0.01172353007135576, "avg_duration_per_task_ms": 806571.2609582059, "avg_output_tokens_per_task": 39025.64220183486, "last_run_timestamp": "2026-10-27T15:15:33.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.7902097692307692, "accuracy_std_dev": 0.03221779293198532, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.10744422882672883, "avg_duration_per_task_ms": 389672.32750582753, "avg_output_tokens_per_task": 29212.660062160063, "last_run_timestamp": "2026-10-27T01:36:12.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.789983125, "accuracy_std_dev": 0.044698004036638775, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.17659961553030304, "avg_duration_per_task_ms": 551265.0208333334, "avg_output_tokens_per_task": 54985.17803030303, "last_run_timestamp": "2026-10-24T16:04:55.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7893355839160839, "accuracy_std_dev": 0.10239971274711347, "run_count": 9, "total_questions": 1716, "avg_cost_per_task_usd": 0.07681546794871794, "avg_duration_per_task_ms": 733179.0168997669, "avg_output_tokens_per_task": 27895.818764568765, "last_run_timestamp": "2026-10-10T16:29:40.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7891638401060072, "accuracy_std_dev": 0.014821929173009913, "run_count": 12, "total_questions": 2264, "avg_cost_per_task_usd": 0.008643914752650177, "avg_duration_per_task_ms": 377329.0070671378, "avg_output_tokens_per_task": 34148.09540636042, "last_run_timestamp": "2026-10-24T16:13:05.629Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.7876364502435629, "accuracy_std_dev": 0.019910444432115912, "run_count": 8, "total_questions": 1437, "avg_cost_per_task_usd": 0.04065875156576201, "avg_duration_per_task_ms": 745848.8225469729, "avg_output_tokens_per_task": 18787.501043841337, "last_run_timestamp": "2026-10-17T16:06:15.029Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.787037125, "accuracy_std_dev": 0.02095938087371319, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.3012645997474747, "avg_duration_per_task_ms": 971517.2935606061, "avg_output_tokens_per_task": 75923.07512626263, "last_run_timestamp": "2026-10-24T16:02:08.814Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.787037, "accuracy_std_dev": 0.014506239226622442, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.028991700757575758, "avg_duration_per_task_ms": 504207.9595959596, "avg_output_tokens_per_task": 24268.680555555555, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.7861952499999999, "accuracy_std_dev": 0.010513514181162274, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.021063606691919187, "avg_duration_per_task_ms": 256646.1351010101, "avg_output_tokens_per_task": 12520.557449494949, "last_run_timestamp": "2026-10-24T16:18:15.010Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.786195, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2643429696969697, "avg_duration_per_task_ms": 180778.88888888888, "avg_output_tokens_per_task": 17428.555555555555, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7854672791234141, "accuracy_std_dev": 0.12208478471357521, "run_count": 5, "total_questions": 867, "avg_cost_per_task_usd": 0.023352209919261825, "avg_duration_per_task_ms": 590708.8500576701, "avg_output_tokens_per_task": 19635.643598615916, "last_run_timestamp": "2026-10-17T08:18:07.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.7853535, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01384800505050505, "avg_duration_per_task_ms": 544619.0151515151, "avg_output_tokens_per_task": 14426.747474747475, "last_run_timestamp": "2026-08-15T14:56:55.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.7853107987288136, "accuracy_std_dev": 0.021945448081549836, "run_count": 4, "total_questions": 472, "avg_cost_per_task_usd": 0.03487327542372881, "avg_duration_per_task_ms": 120089.44067796611, "avg_output_tokens_per_task": 36221.95338983051, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.7834595000000001, "accuracy_std_dev": 0.025125138984690213, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0537799494949495, "avg_duration_per_task_ms": 455539.1212121212, "avg_output_tokens_per_task": 23376.137626262625, "last_run_timestamp": "2026-10-17T16:03:34.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7832240326797386, "accuracy_std_dev": 0.07801749999999996, "run_count": 2, "total_questions": 153, "avg_cost_per_task_usd": 0.07358333333333333, "avg_duration_per_task_ms": 500220.6274509804, "avg_output_tokens_per_task": 27733.385620915033, "last_run_timestamp": "2026-08-15T15:24:07.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.7816675629587804, "accuracy_std_dev": 0.02991313237433426, "run_count": 9, "total_questions": 1771, "avg_cost_per_task_usd": 0.011191632411067194, "avg_duration_per_task_ms": 582007.9429700734, "avg_output_tokens_per_task": 37062.36589497459, "last_run_timestamp": "2026-10-24T16:18:35.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7814084898477156, "accuracy_std_dev": 0.060666856050972155, "run_count": 8, "total_questions": 1576, "avg_cost_per_task_usd": 0.007266644670050761, "avg_duration_per_task_ms": 867252.3737309645, "avg_output_tokens_per_task": 18458.295050761422, "last_run_timestamp": "2026-10-24T16:21:54.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Nebius", "accuracy": 0.7805836666666667, "accuracy_std_dev": 0.009152126540986119, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03482366666666667, "avg_duration_per_task_ms": 531300.7171717172, "avg_output_tokens_per_task": 69410.62962962964, "last_run_timestamp": "2026-10-24T08:41:52.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.780302875, "accuracy_std_dev": 0.044334001808537145, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.012861900883838383, "avg_duration_per_task_ms": 776646.4172979798, "avg_output_tokens_per_task": 38634.71654040404, "last_run_timestamp": "2026-10-17T08:18:06.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7786195454545455, "accuracy_std_dev": 0.04555204390657037, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.21966554178145087, "avg_duration_per_task_ms": 867448.9150596878, "avg_output_tokens_per_task": 73058.02662993572, "last_run_timestamp": "2026-10-17T16:06:46.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.7785086368421053, "accuracy_std_dev": 0.010062970594585871, "run_count": 4, "total_questions": 760, "avg_cost_per_task_usd": 0.002249036842105263, "avg_duration_per_task_ms": 780019.0210526315, "avg_output_tokens_per_task": 14812.598684210527, "last_run_timestamp": "2026-10-24T16:06:32.425Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.7784476091370558, "accuracy_std_dev": 0.018688411382040342, "run_count": 8, "total_questions": 1576, "avg_cost_per_task_usd": 0.022516517131979694, "avg_duration_per_task_ms": 871719.5628172589, "avg_output_tokens_per_task": 38655.122461928935, "last_run_timestamp": "2026-10-24T16:37:21.981Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.776936, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013052893939393939, "avg_duration_per_task_ms": 64108.87373737374, "avg_output_tokens_per_task": 10319.823232323231, "last_run_timestamp": "2026-10-01T19:12:45.924Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.7769359999999998, "accuracy_std_dev": 0.005389920933248147, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00513745202020202, "avg_duration_per_task_ms": 836559.9427609427, "avg_output_tokens_per_task": 28379.09006734007, "last_run_timestamp": "2026-10-24T16:31:50.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7768158778004073, "accuracy_std_dev": 0.02614019988234382, "run_count": 6, "total_questions": 982, "avg_cost_per_task_usd": 0.010268270875763747, "avg_duration_per_task_ms": 1330121.382892057, "avg_output_tokens_per_task": 39680.64867617108, "last_run_timestamp": "2026-10-24T08:12:02.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7767955833333333, "accuracy_std_dev": 0.0196863031719109, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.005944730218855219, "avg_duration_per_task_ms": 663975.4595959596, "avg_output_tokens_per_task": 36414.121632996634, "last_run_timestamp": "2026-10-24T16:20:08.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7765211365054602, "accuracy_std_dev": 0.014331339795043158, "run_count": 9, "total_questions": 1282, "avg_cost_per_task_usd": 0.04696385179407177, "avg_duration_per_task_ms": 968660.7449297972, "avg_output_tokens_per_task": 23871.57488299532, "last_run_timestamp": "2026-10-24T16:01:45.820Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.7760940000000002, "accuracy_std_dev": 0.012598160421267844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013424281144781143, "avg_duration_per_task_ms": 64995.4595959596, "avg_output_tokens_per_task": 10615.195286195287, "last_run_timestamp": "2026-10-02T23:20:38.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.7753727142857143, "accuracy_std_dev": 0.018106406953769444, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02145574458874459, "avg_duration_per_task_ms": 495034.65367965365, "avg_output_tokens_per_task": 28526.727272727272, "last_run_timestamp": "2026-10-30T01:54:18.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7751615438596491, "accuracy_std_dev": 0.0257925287798497, "run_count": 8, "total_questions": 1083, "avg_cost_per_task_usd": 0.013254867959372115, "avg_duration_per_task_ms": 53661.341643582644, "avg_output_tokens_per_task": 17570.362880886427, "last_run_timestamp": "2026-10-24T16:14:06.770Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7746153472485767, "accuracy_std_dev": 0.05521980939561002, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.15729624731182795, "avg_duration_per_task_ms": 963160.6476913346, "avg_output_tokens_per_task": 65807.53447185326, "last_run_timestamp": "2026-10-24T16:03:59.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7741301666666666, "accuracy_std_dev": 0.007235617536802828, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.019828150673400675, "avg_duration_per_task_ms": 721854.4562289562, "avg_output_tokens_per_task": 49359.81902356903, "last_run_timestamp": "2026-10-10T16:25:17.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7741139703903096, "accuracy_std_dev": 0.09718478875928833, "run_count": 4, "total_questions": 743, "avg_cost_per_task_usd": 0.004475604306864065, "avg_duration_per_task_ms": 312870.95827725437, "avg_output_tokens_per_task": 22926.07267833109, "last_run_timestamp": "2026-10-08T22:27:01.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.7734006, "accuracy_std_dev": 0.005280843894681953, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.014113778787878787, "avg_duration_per_task_ms": 123713.42222222222, "avg_output_tokens_per_task": 10542.545454545454, "last_run_timestamp": "2026-10-01T19:20:04.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.7713244166666666, "accuracy_std_dev": 0.0099082406313998, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.014599125841750842, "avg_duration_per_task_ms": 134726.7032828283, "avg_output_tokens_per_task": 29087.89814814815, "last_run_timestamp": "2026-10-24T16:26:32.305Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.7710211897522523, "accuracy_std_dev": 0.007222635018972211, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.011324844594594596, "avg_duration_per_task_ms": 964384.0050675676, "avg_output_tokens_per_task": 11774.962274774774, "last_run_timestamp": "2026-10-24T16:05:13.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.7704826666666666, "accuracy_std_dev": 0.023421454841424542, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0568129494949495, "avg_duration_per_task_ms": 614174.2895622896, "avg_output_tokens_per_task": 22780.829966329966, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7703704, "accuracy_std_dev": 0.025566915297704586, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.011996698989898991, "avg_duration_per_task_ms": 914513.3373737374, "avg_output_tokens_per_task": 45163.47373737374, "last_run_timestamp": "2026-10-27T02:58:53.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "CoreWeave", "accuracy": 0.769679306122449, "accuracy_std_dev": 0.017686462624278497, "run_count": 5, "total_questions": 686, "avg_cost_per_task_usd": 0.009246491253644315, "avg_duration_per_task_ms": 700873.1428571428, "avg_output_tokens_per_task": 30693.99271137026, "last_run_timestamp": "2026-10-24T16:14:24.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7691307272727272, "accuracy_std_dev": 0.12910889092269848, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.016461994031221304, "avg_duration_per_task_ms": 677663.191919192, "avg_output_tokens_per_task": 35541.38842975206, "last_run_timestamp": "2026-10-24T08:57:59.951Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.768994582345191, "accuracy_std_dev": 0.016269118990725185, "run_count": 8, "total_questions": 1518, "avg_cost_per_task_usd": 0.021888522397891963, "avg_duration_per_task_ms": 236418.71277997366, "avg_output_tokens_per_task": 29098.49209486166, "last_run_timestamp": "2026-10-24T16:34:51.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7689713726812815, "accuracy_std_dev": 0.07621573205272868, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.018058317875210794, "avg_duration_per_task_ms": 1377820.0623946036, "avg_output_tokens_per_task": 17446.093591905566, "last_run_timestamp": "2026-10-24T16:01:29.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.7686387142857142, "accuracy_std_dev": 0.0359817938793428, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.05187446248196248, "avg_duration_per_task_ms": 873111.9227994228, "avg_output_tokens_per_task": 23166.116883116883, "last_run_timestamp": "2026-10-17T16:03:24.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.7685711884534847, "accuracy_std_dev": 0.015738499677663245, "run_count": 17, "total_questions": 3343, "avg_cost_per_task_usd": 0.006987174394256656, "avg_duration_per_task_ms": 444537.46365539933, "avg_output_tokens_per_task": 28098.178881244392, "last_run_timestamp": "2026-10-30T00:43:52.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.7685185, "accuracy_std_dev": 0.018461144134370444, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01486243308080808, "avg_duration_per_task_ms": 1091396.6527777778, "avg_output_tokens_per_task": 52782.86363636364, "last_run_timestamp": "2026-10-24T08:54:50.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7677873143148243, "accuracy_std_dev": 0.10954529486385985, "run_count": 10, "total_questions": 1963, "avg_cost_per_task_usd": 0.15105290779419256, "avg_duration_per_task_ms": 869942.4396332145, "avg_output_tokens_per_task": 39571.91288843607, "last_run_timestamp": "2026-10-24T16:19:11.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.7671445564924115, "accuracy_std_dev": 0.01261567820104104, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.019929450252951095, "avg_duration_per_task_ms": 1212404.510961214, "avg_output_tokens_per_task": 60253.559021922425, "last_run_timestamp": "2026-10-24T16:08:41.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.767059119032047, "accuracy_std_dev": 0.008480289053888434, "run_count": 8, "total_questions": 1529, "avg_cost_per_task_usd": 0.02492288554610857, "avg_duration_per_task_ms": 72092.31916285153, "avg_output_tokens_per_task": 26100.194244604318, "last_run_timestamp": "2026-10-24T16:10:43.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7669488305084746, "accuracy_std_dev": 0.016365499999999977, "run_count": 2, "total_questions": 354, "avg_cost_per_task_usd": 0.21194037853107345, "avg_duration_per_task_ms": 159764.62146892655, "avg_output_tokens_per_task": 13982.706214689266, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.057140755050505046, "avg_duration_per_task_ms": 101024.92676767676, "avg_output_tokens_per_task": 22766.989898989897, "last_run_timestamp": "2026-10-01T19:11:42.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7659344612403101, "accuracy_std_dev": 0.01074604027072297, "run_count": 4, "total_questions": 774, "avg_cost_per_task_usd": 0.004193177002583979, "avg_duration_per_task_ms": 4103.266149870801, "avg_output_tokens_per_task": 2323.702842377261, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Sail Research", "accuracy": 0.7657, "accuracy_std_dev": null, "run_count": 1, "total_questions": 138, "avg_cost_per_task_usd": 0.012955934782608695, "avg_duration_per_task_ms": 216746.15942028986, "avg_output_tokens_per_task": 32299.463768115944, "last_run_timestamp": "2026-10-26T18:00:03.751Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7651274895635675, "accuracy_std_dev": 0.012168813200753945, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.01713221790006325, "avg_duration_per_task_ms": 104832.72896900696, "avg_output_tokens_per_task": 5987.342504743833, "last_run_timestamp": "2026-10-24T09:05:10.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7647332578268877, "accuracy_std_dev": 0.03506906370263671, "run_count": 7, "total_questions": 1086, "avg_cost_per_task_usd": 0.06476073848987109, "avg_duration_per_task_ms": 751502.6731123389, "avg_output_tokens_per_task": 52466.43278084714, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7645579698795182, "accuracy_std_dev": 0.052301214506824685, "run_count": 3, "total_questions": 332, "avg_cost_per_task_usd": 0.11985862349397591, "avg_duration_per_task_ms": 1173733.4337349397, "avg_output_tokens_per_task": 34138.882530120485, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.7636309629005059, "accuracy_std_dev": 0.012408560333719428, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.02061922428330523, "avg_duration_per_task_ms": 132670.10623946038, "avg_output_tokens_per_task": 11913.527824620573, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7632574999999999, "accuracy_std_dev": 0.011447194448859535, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.020826034090909093, "avg_duration_per_task_ms": 1150567.0700757576, "avg_output_tokens_per_task": 51976.968434343435, "last_run_timestamp": "2026-10-24T16:20:24.814Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.7630004444444446, "accuracy_std_dev": 0.01096142269873061, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0185216531986532, "avg_duration_per_task_ms": 131653.27160493826, "avg_output_tokens_per_task": 30636.1088664422, "last_run_timestamp": "2026-10-24T16:05:36.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7629069999999999, "accuracy_std_dev": 0.00819966182229484, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2718667558922559, "avg_duration_per_task_ms": 264601.48148148146, "avg_output_tokens_per_task": 17934.003367003366, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.7628913204419889, "accuracy_std_dev": 0.046742499999999965, "run_count": 2, "total_questions": 362, "avg_cost_per_task_usd": 0.015507096685082874, "avg_duration_per_task_ms": 476532.59392265195, "avg_output_tokens_per_task": 53949.68508287293, "last_run_timestamp": "2026-10-24T08:33:50.635Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.7627229007633587, "accuracy_std_dev": 0.011904999999999999, "run_count": 2, "total_questions": 262, "avg_cost_per_task_usd": 0.003081797709923664, "avg_duration_per_task_ms": 72385.01526717558, "avg_output_tokens_per_task": 3744.9809160305344, "last_run_timestamp": "2026-08-15T14:54:48.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.7626265, "accuracy_std_dev": 0.007714766182458135, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0075155000000000005, "avg_duration_per_task_ms": 129316.82575757576, "avg_output_tokens_per_task": 29867.227272727272, "last_run_timestamp": "2026-10-24T16:12:49.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.7626260000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022027883838383836, "avg_duration_per_task_ms": 90904.42929292929, "avg_output_tokens_per_task": 14522.767676767677, "last_run_timestamp": "2026-10-04T03:02:47.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7626259999999999, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01717197474747475, "avg_duration_per_task_ms": 622307.4974747475, "avg_output_tokens_per_task": 61038.34090909091, "last_run_timestamp": "2026-10-03T10:05:09.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.7624391111111111, "accuracy_std_dev": 0.12809223425194174, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0067465028058361414, "avg_duration_per_task_ms": 341598.29517396184, "avg_output_tokens_per_task": 21125.109427609426, "last_run_timestamp": "2026-10-27T02:22:11.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.762346, "accuracy_std_dev": 0.0014306672103136615, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004426390572390572, "avg_duration_per_task_ms": 502309.91414141416, "avg_output_tokens_per_task": 29382.90404040404, "last_run_timestamp": "2026-10-24T16:30:53.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7619949375000001, "accuracy_std_dev": 0.014121240413242524, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.018249934027777776, "avg_duration_per_task_ms": 152465.32638888888, "avg_output_tokens_per_task": 30190.491161616163, "last_run_timestamp": "2026-10-24T16:26:58.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7616160933333332, "accuracy_std_dev": 0.0330626227537687, "run_count": 9, "total_questions": 1650, "avg_cost_per_task_usd": 0.0073882921212121205, "avg_duration_per_task_ms": 943448.4084848484, "avg_output_tokens_per_task": 19044.426666666666, "last_run_timestamp": "2026-10-24T16:00:43.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.76094275, "accuracy_std_dev": 0.01636554188493312, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05637325126262625, "avg_duration_per_task_ms": 580765.970959596, "avg_output_tokens_per_task": 26018.467171717173, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7605754720812183, "accuracy_std_dev": 0.010447569992586773, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.0025078527918781725, "avg_duration_per_task_ms": 100160.06345177664, "avg_output_tokens_per_task": 31388.34771573604, "last_run_timestamp": "2026-10-01T16:04:09.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.760101, "accuracy_std_dev": 0.012052078264764132, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.016351531565656566, "avg_duration_per_task_ms": 481999.97601010103, "avg_output_tokens_per_task": 16387.108585858587, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.7588383750000002, "accuracy_std_dev": 0.006285020782334392, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01747037941919192, "avg_duration_per_task_ms": 73518.15088383839, "avg_output_tokens_per_task": 28877.368055555555, "last_run_timestamp": "2026-10-24T16:15:45.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7582697022900764, "accuracy_std_dev": 0.01511744199591981, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.016854382951653944, "avg_duration_per_task_ms": 249305.42239185752, "avg_output_tokens_per_task": 28356.20165394402, "last_run_timestamp": "2026-10-24T16:36:35.542Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.7572390000000001, "accuracy_std_dev": 0.007014128456194694, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.022558349494949493, "avg_duration_per_task_ms": 96018.14343434344, "avg_output_tokens_per_task": 14878.855555555556, "last_run_timestamp": "2026-10-03T16:56:57.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.7570765817490493, "accuracy_std_dev": 0.03212040139824491, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.004716984157160963, "avg_duration_per_task_ms": 687613.1001267427, "avg_output_tokens_per_task": 28235.273764258556, "last_run_timestamp": "2026-10-24T16:26:55.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7570145555555555, "accuracy_std_dev": 0.13531913577753976, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.029267514029180697, "avg_duration_per_task_ms": 135665.3411896745, "avg_output_tokens_per_task": 22416.14478114478, "last_run_timestamp": "2026-10-28T03:24:15.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7564585415688118, "accuracy_std_dev": 0.10021722464025354, "run_count": 11, "total_questions": 2129, "avg_cost_per_task_usd": 0.006281726632221701, "avg_duration_per_task_ms": 548745.0742132457, "avg_output_tokens_per_task": 26811.618130577735, "last_run_timestamp": "2026-10-27T02:37:18.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.755892, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018171858585858586, "avg_duration_per_task_ms": 285760.0353535354, "avg_output_tokens_per_task": 24027.954545454544, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7547698333333334, "accuracy_std_dev": 0.0766049855914889, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004288400673400673, "avg_duration_per_task_ms": 447675.2971380471, "avg_output_tokens_per_task": 15291.021885521886, "last_run_timestamp": "2026-10-17T08:26:03.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7544054846967168, "accuracy_std_dev": 0.03260817390833722, "run_count": 11, "total_questions": 1797, "avg_cost_per_task_usd": 0.045709284919309964, "avg_duration_per_task_ms": 262995.0717863105, "avg_output_tokens_per_task": 17224.77796327212, "last_run_timestamp": "2026-10-24T08:41:25.770Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.7525250000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008515843434343435, "avg_duration_per_task_ms": 110884.25757575757, "avg_output_tokens_per_task": 8312.732323232323, "last_run_timestamp": "2026-10-04T03:06:11.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.752104375, "accuracy_std_dev": 0.041320927025350916, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.05484566098484848, "avg_duration_per_task_ms": 1035357.8099747475, "avg_output_tokens_per_task": 28498.281565656565, "last_run_timestamp": "2026-10-24T16:02:06.908Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7516834000000001, "accuracy_std_dev": 0.0580270664952141, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.01393037222222222, "avg_duration_per_task_ms": 97444.4191919192, "avg_output_tokens_per_task": 5974.968181818182, "last_run_timestamp": "2026-10-24T08:48:00.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.7515440142932769, "accuracy_std_dev": 0.14128088782365433, "run_count": 10, "total_questions": 1889, "avg_cost_per_task_usd": 0.1006153043938592, "avg_duration_per_task_ms": 662503.1037586024, "avg_output_tokens_per_task": 29255.94706193753, "last_run_timestamp": "2026-10-24T16:11:49.038Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.751402888888889, "accuracy_std_dev": 0.03418981154101141, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.31025038888888895, "avg_duration_per_task_ms": 925905.71661055, "avg_output_tokens_per_task": 79989.34511784512, "last_run_timestamp": "2026-10-27T01:41:24.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7513468000000002, "accuracy_std_dev": 0.014443665156739118, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.01692796111111111, "avg_duration_per_task_ms": 1325396.3136363637, "avg_output_tokens_per_task": 54802.47171717172, "last_run_timestamp": "2026-10-30T15:58:48.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7508415000000002, "accuracy_std_dev": 0.0622895, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06288509848484848, "avg_duration_per_task_ms": 493754.3712121212, "avg_output_tokens_per_task": 26228.825757575756, "last_run_timestamp": "2026-08-15T10:07:37.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.7502806666666667, "accuracy_std_dev": 0.02200747420511694, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.07882791077441077, "avg_duration_per_task_ms": 825090.7154882154, "avg_output_tokens_per_task": 22562.79292929293, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.75, "accuracy_std_dev": 0.00985231011827516, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013619438552188553, "avg_duration_per_task_ms": 95832.97558922559, "avg_output_tokens_per_task": 14970.221380471381, "last_run_timestamp": "2026-10-03T16:54:39.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7495791249999999, "accuracy_std_dev": 0.028467424081911153, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.18216496464646464, "avg_duration_per_task_ms": 1070460.8421717172, "avg_output_tokens_per_task": 82670.29924242424, "last_run_timestamp": "2026-10-24T16:01:48.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.7487311624365482, "accuracy_std_dev": 0.019042500000000018, "run_count": 2, "total_questions": 394, "avg_cost_per_task_usd": 0.00260698730964467, "avg_duration_per_task_ms": 166191.14467005077, "avg_output_tokens_per_task": 30980.81725888325, "last_run_timestamp": "2026-10-01T16:04:24.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.7481013645569621, "accuracy_std_dev": 0.004003499999999993, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.010130237974683544, "avg_duration_per_task_ms": 298779.6734177215, "avg_output_tokens_per_task": 60721.07341772152, "last_run_timestamp": "2026-10-10T08:39:12.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7479075236336781, "accuracy_std_dev": 0.023052021760906493, "run_count": 7, "total_questions": 1354, "avg_cost_per_task_usd": 0.01579615952732644, "avg_duration_per_task_ms": 361173.26514032495, "avg_output_tokens_per_task": 51714.83751846381, "last_run_timestamp": "2026-10-27T15:23:44.931Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7468014, "accuracy_std_dev": 0.009196272649285712, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015463989898989897, "avg_duration_per_task_ms": 342566.1424242424, "avg_output_tokens_per_task": 63071.326262626266, "last_run_timestamp": "2026-10-24T16:16:25.413Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7465941264367815, "accuracy_std_dev": 0.10532480903970348, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.04816395977011494, "avg_duration_per_task_ms": 468882.4540229885, "avg_output_tokens_per_task": 36470.09642401022, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7465043071718538, "accuracy_std_dev": 0.027768881506803414, "run_count": 6, "total_questions": 739, "avg_cost_per_task_usd": 0.053211952638700946, "avg_duration_per_task_ms": 424838.5182679296, "avg_output_tokens_per_task": 15554.012178619756, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.7464647999999999, "accuracy_std_dev": 0.010410614859843804, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.014634281818181818, "avg_duration_per_task_ms": 108043.59494949495, "avg_output_tokens_per_task": 14531.531313131312, "last_run_timestamp": "2026-10-03T16:43:33.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7462993119730186, "accuracy_std_dev": 0.0800254867207247, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.00489628442945475, "avg_duration_per_task_ms": 596052.9263631253, "avg_output_tokens_per_task": 32164.360314783586, "last_run_timestamp": "2026-10-17T16:03:21.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7459349573170732, "accuracy_std_dev": 0.02985061357091343, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.014618331300813008, "avg_duration_per_task_ms": 1032360.968495935, "avg_output_tokens_per_task": 51870.760162601626, "last_run_timestamp": "2026-10-24T08:20:32.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.7457911666666666, "accuracy_std_dev": 0.008528929132012367, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006685603535353536, "avg_duration_per_task_ms": 166915.6186868687, "avg_output_tokens_per_task": 7768.992424242424, "last_run_timestamp": "2026-10-03T16:58:32.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.7443482857142857, "accuracy_std_dev": 0.006065167898142083, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005301684704184704, "avg_duration_per_task_ms": 153567.91919191918, "avg_output_tokens_per_task": 8565.327561327562, "last_run_timestamp": "2026-10-17T16:06:15.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7438972500000001, "accuracy_std_dev": 0.045129307669046964, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.3025249255050505, "avg_duration_per_task_ms": 1004822.8832070708, "avg_output_tokens_per_task": 88113.10290404041, "last_run_timestamp": "2026-10-24T16:08:29.474Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.7430977999999999, "accuracy_std_dev": 0.008356521127837845, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0232193797979798, "avg_duration_per_task_ms": 98352.36666666667, "avg_output_tokens_per_task": 15371.260606060607, "last_run_timestamp": "2026-10-03T16:58:08.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.74242425, "accuracy_std_dev": 0.013572888443050747, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.11500917424242424, "avg_duration_per_task_ms": 764443.2228535353, "avg_output_tokens_per_task": 44885.719696969696, "last_run_timestamp": "2026-10-24T16:39:40.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.742214, "accuracy_std_dev": 0.015132597149861616, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01671199116161616, "avg_duration_per_task_ms": 766495.9797979798, "avg_output_tokens_per_task": 59349.530303030304, "last_run_timestamp": "2026-10-24T08:26:00.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.7420804924406049, "accuracy_std_dev": 0.0038769124828915004, "run_count": 5, "total_questions": 926, "avg_cost_per_task_usd": 0.008837730021598272, "avg_duration_per_task_ms": 138969.186825054, "avg_output_tokens_per_task": 11241.735421166306, "last_run_timestamp": "2026-10-03T17:03:40.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7401795555555556, "accuracy_std_dev": 0.09837566628221198, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.13554839506172842, "avg_duration_per_task_ms": 959035.4539842873, "avg_output_tokens_per_task": 71622.32940516275, "last_run_timestamp": "2026-10-27T01:16:10.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.7384959999999999, "accuracy_std_dev": 0.008170584087648423, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06071549915824916, "avg_duration_per_task_ms": 125902.70959595959, "avg_output_tokens_per_task": 24199.07239057239, "last_run_timestamp": "2026-10-01T19:15:31.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.7373190000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 184, "avg_cost_per_task_usd": 0.008365875, "avg_duration_per_task_ms": 223563.88043478262, "avg_output_tokens_per_task": 10361.597826086956, "last_run_timestamp": "2026-08-08T16:46:11.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.737284008888889, "accuracy_std_dev": 0.05166831288604067, "run_count": 7, "total_questions": 1350, "avg_cost_per_task_usd": 0.017425666666666666, "avg_duration_per_task_ms": 990522.262962963, "avg_output_tokens_per_task": 44050.16814814815, "last_run_timestamp": "2026-10-24T16:13:05.630Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.736027, "accuracy_std_dev": 0.010289969057290673, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004868150505050505, "avg_duration_per_task_ms": 107346.51515151515, "avg_output_tokens_per_task": 7938.19797979798, "last_run_timestamp": "2026-10-03T16:59:36.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.7360049885496184, "accuracy_std_dev": 0.012867937149534676, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.006530778625954198, "avg_duration_per_task_ms": 339780.16603053437, "avg_output_tokens_per_task": 25930.9058524173, "last_run_timestamp": "2026-10-24T16:16:05.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.7353067940797942, "accuracy_std_dev": 0.007845382539271094, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.006840141570141572, "avg_duration_per_task_ms": 249346.28893178893, "avg_output_tokens_per_task": 8460.478764478765, "last_run_timestamp": "2026-10-24T16:02:47.278Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.7349895701863354, "accuracy_std_dev": 0.017162011317797845, "run_count": 6, "total_questions": 805, "avg_cost_per_task_usd": 0.015730058385093167, "avg_duration_per_task_ms": 68792.66708074535, "avg_output_tokens_per_task": 9043.119254658384, "last_run_timestamp": "2026-10-17T16:03:44.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.7343291640304627, "accuracy_std_dev": 0.0944960545028733, "run_count": 10, "total_questions": 1707, "avg_cost_per_task_usd": 0.0041295178676039835, "avg_duration_per_task_ms": 550309.4973637961, "avg_output_tokens_per_task": 25314.345049794963, "last_run_timestamp": "2026-10-27T15:54:59.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.7325637142857143, "accuracy_std_dev": 0.0065623172238881, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006008777777777779, "avg_duration_per_task_ms": 423083.5331890332, "avg_output_tokens_per_task": 7933.917027417027, "last_run_timestamp": "2026-10-24T16:07:46.992Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.732043837671728, "accuracy_std_dev": 0.018715369477343178, "run_count": 14, "total_questions": 2766, "avg_cost_per_task_usd": 0.006619636659436008, "avg_duration_per_task_ms": 127243.25451916124, "avg_output_tokens_per_task": 6931.6146059291395, "last_run_timestamp": "2026-10-17T16:03:44.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7305929722222222, "accuracy_std_dev": 0.012805628963003616, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.20431471787317615, "avg_duration_per_task_ms": 243944.4814814815, "avg_output_tokens_per_task": 38124.83193041526, "last_run_timestamp": "2026-10-24T08:29:16.808Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7294042813688214, "accuracy_std_dev": 0.0505548117787454, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.3084836787072243, "avg_duration_per_task_ms": 1347611.5551330799, "avg_output_tokens_per_task": 97858.03675538657, "last_run_timestamp": "2026-10-24T16:28:00.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.72895625, "accuracy_std_dev": 0.024279686950978163, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006164965909090908, "avg_duration_per_task_ms": 374409.91414141416, "avg_output_tokens_per_task": 19987.00505050505, "last_run_timestamp": "2026-10-24T16:02:47.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.7286756666666667, "accuracy_std_dev": 0.008877283080362397, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.19010534595959594, "avg_duration_per_task_ms": 243777.6670875421, "avg_output_tokens_per_task": 37829.979797979795, "last_run_timestamp": "2026-10-24T08:26:35.486Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7281145, "accuracy_std_dev": 0.004208499999999948, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.060785368686868686, "avg_duration_per_task_ms": 121296.46212121213, "avg_output_tokens_per_task": 24226.65404040404, "last_run_timestamp": "2026-10-01T19:12:41.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7273304235890933, "accuracy_std_dev": 0.048401510033262374, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.3023879137603043, "avg_duration_per_task_ms": 1316651.7057704502, "avg_output_tokens_per_task": 95464.91819911223, "last_run_timestamp": "2026-10-24T16:02:28.413Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.72727275, "accuracy_std_dev": 0.008934765069892239, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.18649388005050505, "avg_duration_per_task_ms": 302584.60984848486, "avg_output_tokens_per_task": 37129.88047138047, "last_run_timestamp": "2026-10-24T08:28:43.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.7271808509532062, "accuracy_std_dev": 0.03540222199908109, "run_count": 6, "total_questions": 1154, "avg_cost_per_task_usd": 0.011152869150779895, "avg_duration_per_task_ms": 504501.5831889082, "avg_output_tokens_per_task": 55506.62998266898, "last_run_timestamp": "2026-10-10T08:24:27.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7264923098256736, "accuracy_std_dev": 0.012443953888007772, "run_count": 13, "total_questions": 2524, "avg_cost_per_task_usd": 0.18690665134706813, "avg_duration_per_task_ms": 345005.80625990493, "avg_output_tokens_per_task": 37190.982171156895, "last_run_timestamp": "2026-10-24T08:44:41.412Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.7264309999999999, "accuracy_std_dev": 0.021044000000000007, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1303771111111111, "avg_duration_per_task_ms": 318087.7601010101, "avg_output_tokens_per_task": 43267.416666666664, "last_run_timestamp": "2026-10-04T05:57:29.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.725799875, "accuracy_std_dev": 0.01577998058013301, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005505772095959596, "avg_duration_per_task_ms": 302649.43876262626, "avg_output_tokens_per_task": 8026.409090909091, "last_run_timestamp": "2026-10-24T16:01:05.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.7252524, "accuracy_std_dev": 0.016713400846027716, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004543460606060606, "avg_duration_per_task_ms": 167068.73636363636, "avg_output_tokens_per_task": 7707.889898989899, "last_run_timestamp": "2026-10-03T16:57:10.991Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.7234044255319149, "accuracy_std_dev": 0.1917994784299478, "run_count": 3, "total_questions": 517, "avg_cost_per_task_usd": 0.033565591876208896, "avg_duration_per_task_ms": 242772.43907156674, "avg_output_tokens_per_task": 37102.30560928433, "last_run_timestamp": "2026-09-29T20:57:20.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.7226898888888889, "accuracy_std_dev": 0.010780440739747587, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.004412828282828282, "avg_duration_per_task_ms": 687415.5527497195, "avg_output_tokens_per_task": 7898.555555555556, "last_run_timestamp": "2026-10-24T16:01:45.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.7219817142857143, "accuracy_std_dev": 0.03351855020183082, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007928627705627704, "avg_duration_per_task_ms": 786446.1406926407, "avg_output_tokens_per_task": 27309.676767676767, "last_run_timestamp": "2026-10-24T16:14:06.772Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.7219408075949366, "accuracy_std_dev": 0.025598999999999983, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.27939689620253166, "avg_duration_per_task_ms": 450249.90379746835, "avg_output_tokens_per_task": 45941.00253164557, "last_run_timestamp": "2026-08-15T08:16:15.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.721866038034188, "accuracy_std_dev": 0.03876582276336764, "run_count": 12, "total_questions": 2340, "avg_cost_per_task_usd": 0.010606732051282052, "avg_duration_per_task_ms": 258545.00683760684, "avg_output_tokens_per_task": 48059.278205128205, "last_run_timestamp": "2026-10-24T08:11:11.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.721557710526316, "accuracy_std_dev": 0.013393277763800398, "run_count": 38, "total_questions": 7524, "avg_cost_per_task_usd": 0.19713929492291335, "avg_duration_per_task_ms": 268866.1802232855, "avg_output_tokens_per_task": 36797.53282828283, "last_run_timestamp": "2026-10-24T09:06:24.922Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7204335, "accuracy_std_dev": 0.06469765028190745, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12717121843434342, "avg_duration_per_task_ms": 642265.8712121212, "avg_output_tokens_per_task": 38206.32133838384, "last_run_timestamp": "2026-10-24T16:25:38.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7203351056305258, "accuracy_std_dev": 0.11641213306920471, "run_count": 12, "total_questions": 2149, "avg_cost_per_task_usd": 0.033083575151233134, "avg_duration_per_task_ms": 1068472.419730107, "avg_output_tokens_per_task": 34739.315960912056, "last_run_timestamp": "2026-10-24T16:14:06.772Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Cohere", "accuracy": 0.719697, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.021745989898989897, "avg_duration_per_task_ms": 204822.55555555556, "avg_output_tokens_per_task": 77279.37878787878, "last_run_timestamp": "2026-10-24T08:14:03.073Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "AkashML", "accuracy": 0.7194163333333333, "accuracy_std_dev": 0.011195005920300179, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.014668272727272728, "avg_duration_per_task_ms": 1270908.3501683502, "avg_output_tokens_per_task": 60914.632996632994, "last_run_timestamp": "2026-10-30T00:57:12.245Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7174523333333331, "accuracy_std_dev": 0.013163488932987655, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.018911546296296298, "avg_duration_per_task_ms": 531218.9570707071, "avg_output_tokens_per_task": 67191.76851851853, "last_run_timestamp": "2026-10-24T09:17:16.298Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.7164982000000001, "accuracy_std_dev": 0.007939319804618048, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007827072727272727, "avg_duration_per_task_ms": 80947.86969696969, "avg_output_tokens_per_task": 9940.124242424243, "last_run_timestamp": "2026-10-03T16:40:12.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.7138721298568508, "accuracy_std_dev": 0.014796710480373683, "run_count": 5, "total_questions": 978, "avg_cost_per_task_usd": 0.011163598159509203, "avg_duration_per_task_ms": 105524.10429447853, "avg_output_tokens_per_task": 10243.394683026585, "last_run_timestamp": "2026-10-03T16:43:17.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.713468, "accuracy_std_dev": 0.009367361122536085, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010825097979797981, "avg_duration_per_task_ms": 93154.30303030302, "avg_output_tokens_per_task": 10165.817171717172, "last_run_timestamp": "2026-10-03T16:59:33.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Darkbloom", "accuracy": 0.713213, "accuracy_std_dev": null, "run_count": 1, "total_questions": 111, "avg_cost_per_task_usd": 0.007665369369369369, "avg_duration_per_task_ms": 1401051.936936937, "avg_output_tokens_per_task": 27183.8018018018, "last_run_timestamp": "2026-10-03T09:34:00.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7124580000000001, "accuracy_std_dev": 0.005150598062361318, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015565995959595959, "avg_duration_per_task_ms": 101217.36060606061, "avg_output_tokens_per_task": 10296.08787878788, "last_run_timestamp": "2026-10-03T16:46:22.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.7117846000000001, "accuracy_std_dev": 0.007633552268767163, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00877359797979798, "avg_duration_per_task_ms": 133676.15555555557, "avg_output_tokens_per_task": 5686.586868686869, "last_run_timestamp": "2026-10-03T16:45:33.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7112403023255814, "accuracy_std_dev": 0.09623599999999999, "run_count": 2, "total_questions": 344, "avg_cost_per_task_usd": 0.06576875, "avg_duration_per_task_ms": 744084.4970930233, "avg_output_tokens_per_task": 27095.636627906977, "last_run_timestamp": "2026-08-15T09:29:41.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.7106959574742269, "accuracy_std_dev": 0.04945132031209455, "run_count": 16, "total_questions": 3104, "avg_cost_per_task_usd": 0.007755754832474224, "avg_duration_per_task_ms": 541486.2728737113, "avg_output_tokens_per_task": 25549.894974226805, "last_run_timestamp": "2026-10-24T16:12:04.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.7098763333333333, "accuracy_std_dev": 0.0209969340354464, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0663709898989899, "avg_duration_per_task_ms": 1050299.8484848484, "avg_output_tokens_per_task": 22129.309764309764, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.7077440000000002, "accuracy_std_dev": 0.006867349678005355, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0059367939393939395, "avg_duration_per_task_ms": 109836.06363636363, "avg_output_tokens_per_task": 5507.80101010101, "last_run_timestamp": "2026-10-03T17:02:24.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.707071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008867828282828283, "avg_duration_per_task_ms": 116045.65151515152, "avg_output_tokens_per_task": 5749.39898989899, "last_run_timestamp": "2026-10-04T03:05:59.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.70665, "accuracy_std_dev": 0.05616899659064598, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006621275252525252, "avg_duration_per_task_ms": 125753.64267676767, "avg_output_tokens_per_task": 7346.292929292929, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.7048612500000001, "accuracy_std_dev": 0.04887384717246537, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.016058294823232323, "avg_duration_per_task_ms": 999836.755050505, "avg_output_tokens_per_task": 49417.84027777778, "last_run_timestamp": "2026-10-24T16:05:13.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.7045455, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01336774494949495, "avg_duration_per_task_ms": 231791.09848484848, "avg_output_tokens_per_task": 33323.15909090909, "last_run_timestamp": "2026-10-01T19:27:22.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7037038000000001, "accuracy_std_dev": 0.015684797906253056, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.01112428686868687, "avg_duration_per_task_ms": 196538.2898989899, "avg_output_tokens_per_task": 5791.187878787879, "last_run_timestamp": "2026-10-03T16:59:57.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7032012317327765, "accuracy_std_dev": 0.20342432055346774, "run_count": 5, "total_questions": 958, "avg_cost_per_task_usd": 0.08802485803757831, "avg_duration_per_task_ms": 1349048.8903966597, "avg_output_tokens_per_task": 33413.259916492694, "last_run_timestamp": "2026-10-24T16:13:05.627Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.7030114672897196, "accuracy_std_dev": 0.02694021764221664, "run_count": 10, "total_questions": 1926, "avg_cost_per_task_usd": 0.009145764278296989, "avg_duration_per_task_ms": 363422.58099688473, "avg_output_tokens_per_task": 44741.07476635514, "last_run_timestamp": "2026-10-24T08:01:41.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7028622499999999, "accuracy_std_dev": 0.08200476941122087, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0070032777777777775, "avg_duration_per_task_ms": 172461.12373737374, "avg_output_tokens_per_task": 7212.60101010101, "last_run_timestamp": "2026-10-03T16:56:36.098Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.7006736, "accuracy_std_dev": 0.029399119487494893, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010841593939393938, "avg_duration_per_task_ms": 65671.14141414141, "avg_output_tokens_per_task": 8990.147474747475, "last_run_timestamp": "2026-10-03T16:36:16.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.7006460556023589, "accuracy_std_dev": 0.008635764504856131, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.010373346251053074, "avg_duration_per_task_ms": 142035.05391743893, "avg_output_tokens_per_task": 9363.947767481044, "last_run_timestamp": "2026-10-08T22:40:37.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6948820649606299, "accuracy_std_dev": 0.027729042227588305, "run_count": 8, "total_questions": 1524, "avg_cost_per_task_usd": 0.015564356299212597, "avg_duration_per_task_ms": 765563.3963254594, "avg_output_tokens_per_task": 77634.76837270342, "last_run_timestamp": "2026-09-26T16:34:21.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6936026666666666, "accuracy_std_dev": 0.010286382189844758, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.005539732323232322, "avg_duration_per_task_ms": 380174.12373737374, "avg_output_tokens_per_task": 6110.575757575758, "last_run_timestamp": "2026-10-08T23:09:15.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.6932966363636364, "accuracy_std_dev": 0.13147150091889792, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.609068142332415, "avg_duration_per_task_ms": 632474.5367309458, "avg_output_tokens_per_task": 40521.938016528926, "last_run_timestamp": "2026-10-24T08:08:54.166Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6932378040665435, "accuracy_std_dev": 0.046335214744527585, "run_count": 11, "total_questions": 2164, "avg_cost_per_task_usd": 0.0074276700554528645, "avg_duration_per_task_ms": 281567.51894639555, "avg_output_tokens_per_task": 40195.87939001848, "last_run_timestamp": "2026-10-24T08:42:05.963Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.691171, "accuracy_std_dev": 0.011421081968593563, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.006491580808080808, "avg_duration_per_task_ms": 106721.03198653199, "avg_output_tokens_per_task": 15952.695847362515, "last_run_timestamp": "2026-10-28T10:35:05.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.6905724000000001, "accuracy_std_dev": 0.021171864222122725, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.009511277777777778, "avg_duration_per_task_ms": 270838.6898989899, "avg_output_tokens_per_task": 6542.475757575758, "last_run_timestamp": "2026-10-03T17:02:48.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.6899941197301855, "accuracy_std_dev": 0.005866752821337055, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.0068884283305227665, "avg_duration_per_task_ms": 1677214.3709949409, "avg_output_tokens_per_task": 49011.02360876897, "last_run_timestamp": "2026-10-24T16:08:06.921Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.689815, "accuracy_std_dev": 0.0034448118235979193, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013852137626262627, "avg_duration_per_task_ms": 234308.42045454544, "avg_output_tokens_per_task": 32893.11994949495, "last_run_timestamp": "2026-10-01T19:27:10.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.6874404742120345, "accuracy_std_dev": 0.03143087716808991, "run_count": 6, "total_questions": 698, "avg_cost_per_task_usd": 0.009229828080229225, "avg_duration_per_task_ms": 54932.83810888252, "avg_output_tokens_per_task": 5977.71346704871, "last_run_timestamp": "2026-10-24T16:45:45.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6855188289251547, "accuracy_std_dev": 0.09527891619753961, "run_count": 9, "total_questions": 1777, "avg_cost_per_task_usd": 0.011474473832301631, "avg_duration_per_task_ms": 906865.2909397861, "avg_output_tokens_per_task": 33586.68429938098, "last_run_timestamp": "2026-10-24T16:17:39.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6854461436619719, "accuracy_std_dev": 0.012155576012164115, "run_count": 4, "total_questions": 710, "avg_cost_per_task_usd": 0.040622619718309855, "avg_duration_per_task_ms": 770310.514084507, "avg_output_tokens_per_task": 43146.859154929574, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403:free", "model_slug": null, "model_name": "Gemma 4 26B A4B IT (free)", "provider_name": "Darkbloom", "accuracy": 0.684722, "accuracy_std_dev": null, "run_count": 1, "total_questions": 120, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 382535.98333333334, "avg_output_tokens_per_task": 6994.291666666667, "last_run_timestamp": "2026-08-04T05:36:23.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.6844233356431357, "accuracy_std_dev": 0.1414237811051766, "run_count": 35, "total_questions": 6927, "avg_cost_per_task_usd": 0.6621923740435977, "avg_duration_per_task_ms": 602080.901255955, "avg_output_tokens_per_task": 41272.465136422696, "last_run_timestamp": "2026-10-24T08:43:07.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.6843435000000001, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01356670707070707, "avg_duration_per_task_ms": 229761.702020202, "avg_output_tokens_per_task": 33820.56313131313, "last_run_timestamp": "2026-10-01T19:28:34.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.6830809999999999, "accuracy_std_dev": 0.14005659745795623, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.6164722958754209, "avg_duration_per_task_ms": 635625.3400673401, "avg_output_tokens_per_task": 40941.11069023569, "last_run_timestamp": "2026-10-24T08:26:35.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.6825664722222222, "accuracy_std_dev": 0.136152835465126, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.6585870970819304, "avg_duration_per_task_ms": 454569.8709315376, "avg_output_tokens_per_task": 40985.43560606061, "last_run_timestamp": "2026-10-24T08:24:10.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.6805556666666668, "accuracy_std_dev": 0.14593486187813, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.6194684494949494, "avg_duration_per_task_ms": 636905.944023569, "avg_output_tokens_per_task": 41202.27062289562, "last_run_timestamp": "2026-10-24T08:14:03.137Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.680394694980695, "accuracy_std_dev": 0.020660473202167934, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.0022303307593307593, "avg_duration_per_task_ms": 211402.3449163449, "avg_output_tokens_per_task": 4069.488416988417, "last_run_timestamp": "2026-10-24T16:39:13.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.6796669584736251, "accuracy_std_dev": 0.014488707679507579, "run_count": 12, "total_questions": 1782, "avg_cost_per_task_usd": 0.004952217732884399, "avg_duration_per_task_ms": 782700.4062850729, "avg_output_tokens_per_task": 49359.42255892256, "last_run_timestamp": "2026-10-30T00:43:53.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6751971669865644, "accuracy_std_dev": 0.03877467265315564, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.02805620857325656, "avg_duration_per_task_ms": 841992.2539987204, "avg_output_tokens_per_task": 18377.529750479847, "last_run_timestamp": "2026-10-24T16:22:30.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.6726822675159235, "accuracy_std_dev": 0.0862043071017272, "run_count": 3, "total_questions": 471, "avg_cost_per_task_usd": 0.07077279405520169, "avg_duration_per_task_ms": 1745687.9214437366, "avg_output_tokens_per_task": 24749.823779193204, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.6691920000000001, "accuracy_std_dev": 0.06399689168567932, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008361025974025976, "avg_duration_per_task_ms": 1523891.8455988455, "avg_output_tokens_per_task": 65650.8253968254, "last_run_timestamp": "2026-10-30T04:19:55.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.6689113333333334, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0026409175084175086, "avg_duration_per_task_ms": 47935.90572390572, "avg_output_tokens_per_task": 2383.6902356902356, "last_run_timestamp": "2026-10-02T15:53:31.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.6683503999999999, "accuracy_std_dev": 0.046945141186708554, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015881437373737373, "avg_duration_per_task_ms": 296657.8616161616, "avg_output_tokens_per_task": 8706.40404040404, "last_run_timestamp": "2026-10-03T17:36:48.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.66835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013946772727272728, "avg_duration_per_task_ms": 324889.0353535354, "avg_output_tokens_per_task": 15315.242424242424, "last_run_timestamp": "2026-10-01T08:41:30.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.66813975, "accuracy_std_dev": 0.0169491097609137, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005768640151515151, "avg_duration_per_task_ms": 418288.6957070707, "avg_output_tokens_per_task": 15810.410353535353, "last_run_timestamp": "2026-10-24T16:11:51.309Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.6671506640851887, "accuracy_std_dev": 0.1203125901358104, "run_count": 8, "total_questions": 1033, "avg_cost_per_task_usd": 0.09392116069699902, "avg_duration_per_task_ms": 1316783.9874152953, "avg_output_tokens_per_task": 26739.650532429816, "last_run_timestamp": "2026-10-24T16:06:30.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.6666666755181347, "accuracy_std_dev": 0.016603057166797653, "run_count": 9, "total_questions": 1544, "avg_cost_per_task_usd": 0.006663814119170984, "avg_duration_per_task_ms": 477892.86852331605, "avg_output_tokens_per_task": 47426.00971502591, "last_run_timestamp": "2026-10-24T16:02:28.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6662925555555553, "accuracy_std_dev": 0.007644274285896932, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.005087682379349046, "avg_duration_per_task_ms": 104399.63636363637, "avg_output_tokens_per_task": 9580.892817059485, "last_run_timestamp": "2026-10-03T09:23:08.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6662457499999999, "accuracy_std_dev": 0.017551141520923935, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005718719696969698, "avg_duration_per_task_ms": 566951.7815656565, "avg_output_tokens_per_task": 16218.064393939394, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.6658247500000001, "accuracy_std_dev": 0.0064656044332684175, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0027968030303030305, "avg_duration_per_task_ms": 81352.02777777778, "avg_output_tokens_per_task": 2993.090909090909, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.6649832, "accuracy_std_dev": 0.006981815133616774, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.002071141414141414, "avg_duration_per_task_ms": 228896.86666666667, "avg_output_tokens_per_task": 3033.7636363636366, "last_run_timestamp": "2026-10-05T20:44:48.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6637718184210527, "accuracy_std_dev": 0.20207857374716895, "run_count": 10, "total_questions": 1900, "avg_cost_per_task_usd": 0.012164477368421052, "avg_duration_per_task_ms": 311613.41736842104, "avg_output_tokens_per_task": 14454.243684210527, "last_run_timestamp": "2026-10-17T08:17:57.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.66372075, "accuracy_std_dev": 0.00844903779654822, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00248682702020202, "avg_duration_per_task_ms": 87766.55681818182, "avg_output_tokens_per_task": 2304.1906565656564, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.6633000000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013461267676767677, "avg_duration_per_task_ms": 327894.71717171714, "avg_output_tokens_per_task": 14738.742424242424, "last_run_timestamp": "2026-10-01T08:27:45.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.6627089841688654, "accuracy_std_dev": 0.007909198264520861, "run_count": 4, "total_questions": 379, "avg_cost_per_task_usd": 0.004262828496042217, "avg_duration_per_task_ms": 535749.2849604222, "avg_output_tokens_per_task": 28502.488126649077, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.6622475, "accuracy_std_dev": 0.019541184080039777, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0077082720959595966, "avg_duration_per_task_ms": 563098.2026515151, "avg_output_tokens_per_task": 59511.905303030304, "last_run_timestamp": "2026-10-24T16:15:47.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.6616163125, "accuracy_std_dev": 0.014371619927476648, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.00900978345959596, "avg_duration_per_task_ms": 319538.81597222225, "avg_output_tokens_per_task": 59717.22032828283, "last_run_timestamp": "2026-10-24T16:22:30.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.6601731428571429, "accuracy_std_dev": 0.023160494075833857, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008067838383838384, "avg_duration_per_task_ms": 808632.0974025974, "avg_output_tokens_per_task": 55153.242424242424, "last_run_timestamp": "2026-10-24T16:14:24.977Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.659933, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014984520202020202, "avg_duration_per_task_ms": 348987.1767676768, "avg_output_tokens_per_task": 16461.863636363636, "last_run_timestamp": "2026-10-01T08:28:22.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6599326666666666, "accuracy_std_dev": 0.013255807590972711, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015927558922558923, "avg_duration_per_task_ms": 14095.8367003367, "avg_output_tokens_per_task": 1794.7289562289561, "last_run_timestamp": "2026-10-02T23:17:48.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.658249, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0066776414141414145, "avg_duration_per_task_ms": 101359.9898989899, "avg_output_tokens_per_task": 9307.358585858587, "last_run_timestamp": "2026-10-04T03:07:35.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.65736, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.014864365482233502, "avg_duration_per_task_ms": 1717906.416243655, "avg_output_tokens_per_task": 64415.350253807104, "last_run_timestamp": "2026-10-01T08:48:38.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.6570467857142857, "accuracy_std_dev": 0.14261200486794257, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.6177520173160173, "avg_duration_per_task_ms": 633109.20995671, "avg_output_tokens_per_task": 41054.18795093795, "last_run_timestamp": "2026-10-24T08:12:02.408Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6560171418530352, "accuracy_std_dev": 0.056453171596991564, "run_count": 8, "total_questions": 1565, "avg_cost_per_task_usd": 0.05637429073482428, "avg_duration_per_task_ms": 1175439.4670926516, "avg_output_tokens_per_task": 24772.420447284345, "last_run_timestamp": "2026-10-24T16:10:43.378Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.65277775, "accuracy_std_dev": 0.007840200328276042, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.003156868686868687, "avg_duration_per_task_ms": 41742.583333333336, "avg_output_tokens_per_task": 2990.8636363636365, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.652357, "accuracy_std_dev": 0.01262599999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004161952020202021, "avg_duration_per_task_ms": 27160.11111111111, "avg_output_tokens_per_task": 2408.8232323232323, "last_run_timestamp": "2026-10-01T19:12:31.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.6523569999999999, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01652235606060606, "avg_duration_per_task_ms": 1917916.494949495, "avg_output_tokens_per_task": 65966.15909090909, "last_run_timestamp": "2026-10-09T00:15:37.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.651515, "accuracy_std_dev": 0.1905456270048375, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.058297676767676776, "avg_duration_per_task_ms": 1098090.5791245792, "avg_output_tokens_per_task": 23174.20202020202, "last_run_timestamp": "2026-10-17T16:03:44.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6492706666666667, "accuracy_std_dev": 0.010498645193018418, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0041127255892255896, "avg_duration_per_task_ms": 24949.132996632998, "avg_output_tokens_per_task": 2375.9343434343436, "last_run_timestamp": "2026-10-02T23:15:53.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.6484773807106599, "accuracy_std_dev": 0.0054344006339245585, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.00290907614213198, "avg_duration_per_task_ms": 58072.35659898477, "avg_output_tokens_per_task": 2371.812182741117, "last_run_timestamp": "2026-10-27T00:30:02.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.6477272500000001, "accuracy_std_dev": 0.00710503227913143, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.003938823232323232, "avg_duration_per_task_ms": 63531.12626262626, "avg_output_tokens_per_task": 2389.935606060606, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "CoreWeave", "accuracy": 0.647493, "accuracy_std_dev": null, "run_count": 1, "total_questions": 113, "avg_cost_per_task_usd": 0.005189893805309735, "avg_duration_per_task_ms": 619234.2654867256, "avg_output_tokens_per_task": 34365.4778761062, "last_run_timestamp": "2026-10-01T08:28:32.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.6470586811145511, "accuracy_std_dev": 0.06899576470516142, "run_count": 4, "total_questions": 646, "avg_cost_per_task_usd": 0.006973526315789473, "avg_duration_per_task_ms": 477220.6222910217, "avg_output_tokens_per_task": 36135.41950464396, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.6468855, "accuracy_std_dev": 0.133548982014091, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.015117938762626265, "avg_duration_per_task_ms": 431100.12184343435, "avg_output_tokens_per_task": 12974.737373737375, "last_run_timestamp": "2026-10-24T16:02:06.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.646675125, "accuracy_std_dev": 0.02302836491936359, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006796745580808081, "avg_duration_per_task_ms": 647569.8042929292, "avg_output_tokens_per_task": 44075.63952020202, "last_run_timestamp": "2026-10-24T16:13:05.627Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.644781, "accuracy_std_dev": 0.0023808285322550727, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0024933156565656567, "avg_duration_per_task_ms": 50490.56060606061, "avg_output_tokens_per_task": 2691.5126262626263, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.6439055298507463, "accuracy_std_dev": 0.20314360210271282, "run_count": 7, "total_questions": 1340, "avg_cost_per_task_usd": 0.07648400820895522, "avg_duration_per_task_ms": 1782783.4514925373, "avg_output_tokens_per_task": 36935.7671641791, "last_run_timestamp": "2026-10-24T16:17:39.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.64309775, "accuracy_std_dev": 0.010030734778045907, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0023864103535353536, "avg_duration_per_task_ms": 55257.29924242424, "avg_output_tokens_per_task": 2792.7045454545455, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.6419754444444444, "accuracy_std_dev": 0.014018014458475362, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.007092590347923681, "avg_duration_per_task_ms": 241593.06509539843, "avg_output_tokens_per_task": 11377.103254769921, "last_run_timestamp": "2026-10-03T10:20:18.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.641751, "accuracy_std_dev": 0.012926960106691747, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0027229565656565658, "avg_duration_per_task_ms": 151188.63838383838, "avg_output_tokens_per_task": 9770.50707070707, "last_run_timestamp": "2026-10-03T17:16:11.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.641751, "accuracy_std_dev": 0.015110385329302487, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0044971595959595965, "avg_duration_per_task_ms": 76672.9303030303, "avg_output_tokens_per_task": 8453.274747474747, "last_run_timestamp": "2026-10-03T16:51:25.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.641414, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00284089898989899, "avg_duration_per_task_ms": 209352.64646464647, "avg_output_tokens_per_task": 9266.040404040405, "last_run_timestamp": "2026-10-09T15:00:06.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.6401515, "accuracy_std_dev": 0.002759842885020819, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004124544191919192, "avg_duration_per_task_ms": 34476.35353535353, "avg_output_tokens_per_task": 2259.632575757576, "last_run_timestamp": "2026-10-01T19:13:29.560Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6388888630705394, "accuracy_std_dev": 0.22030686687149445, "run_count": 8, "total_questions": 1446, "avg_cost_per_task_usd": 0.008508840940525588, "avg_duration_per_task_ms": 167853.79668049794, "avg_output_tokens_per_task": 14018.163208852005, "last_run_timestamp": "2026-10-24T16:26:15.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.6381674285714285, "accuracy_std_dev": 0.012871844288514171, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.001091096681096681, "avg_duration_per_task_ms": 351783.54761904763, "avg_output_tokens_per_task": 11560.167388167389, "last_run_timestamp": "2026-10-30T01:21:27.783Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6378052777777776, "accuracy_std_dev": 0.01862890271779676, "run_count": 9, "total_questions": 1692, "avg_cost_per_task_usd": 0.005482690307328605, "avg_duration_per_task_ms": 821174.3670212766, "avg_output_tokens_per_task": 35064.53132387707, "last_run_timestamp": "2026-10-30T02:09:50.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6372055000000001, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.015519383838383837, "avg_duration_per_task_ms": 14506.310606060606, "avg_output_tokens_per_task": 1746.2525252525252, "last_run_timestamp": "2026-10-01T19:12:45.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.6367847499999999, "accuracy_std_dev": 0.012277609526593496, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015237281565656567, "avg_duration_per_task_ms": 18840.242424242424, "avg_output_tokens_per_task": 1618.909090909091, "last_run_timestamp": "2026-10-01T19:14:40.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.6358028333333334, "accuracy_std_dev": 0.012808867337599457, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.001631114478114478, "avg_duration_per_task_ms": 75943.06734006734, "avg_output_tokens_per_task": 8246.542087542088, "last_run_timestamp": "2026-10-03T16:47:32.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.6324910327398047, "accuracy_std_dev": 0.011104531477434478, "run_count": 9, "total_questions": 1741, "avg_cost_per_task_usd": 0.01822674784606548, "avg_duration_per_task_ms": 490237.7823090178, "avg_output_tokens_per_task": 91135.2251579552, "last_run_timestamp": "2026-10-03T17:08:02.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.632155, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005966434343434343, "avg_duration_per_task_ms": 235473.01515151514, "avg_output_tokens_per_task": 48249.11363636364, "last_run_timestamp": "2026-10-01T08:24:52.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.6315160731707317, "accuracy_std_dev": 0.011095205901649571, "run_count": 4, "total_questions": 697, "avg_cost_per_task_usd": 0.008070608321377333, "avg_duration_per_task_ms": 35343.74175035868, "avg_output_tokens_per_task": 13371.598278335725, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.6313132, "accuracy_std_dev": 0.007896357575490096, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0027152242424242426, "avg_duration_per_task_ms": 260211.7595959596, "avg_output_tokens_per_task": 8830.749494949496, "last_run_timestamp": "2026-10-03T17:03:50.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.6306148156028369, "accuracy_std_dev": 0.05281625698168657, "run_count": 3, "total_questions": 282, "avg_cost_per_task_usd": 0.0037584255319148938, "avg_duration_per_task_ms": 208452.59574468085, "avg_output_tokens_per_task": 12308.468085106382, "last_run_timestamp": "2026-10-23T09:29:14.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.6305678657378088, "accuracy_std_dev": 0.007960405916746635, "run_count": 8, "total_questions": 1579, "avg_cost_per_task_usd": 0.00187089613679544, "avg_duration_per_task_ms": 238458.68967701076, "avg_output_tokens_per_task": 10226.877137428752, "last_run_timestamp": "2026-10-24T16:03:25.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Venice", "accuracy": 0.6292411856287424, "accuracy_std_dev": 0.0117815, "run_count": 2, "total_questions": 334, "avg_cost_per_task_usd": 0.0071094161676646716, "avg_duration_per_task_ms": 112258.81736526947, "avg_output_tokens_per_task": 28232.077844311378, "last_run_timestamp": "2026-08-15T08:02:44.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.6290686666666666, "accuracy_std_dev": 0.03535789523021852, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00678852356902357, "avg_duration_per_task_ms": 106935.86756453423, "avg_output_tokens_per_task": 9465.799102132436, "last_run_timestamp": "2026-10-03T10:32:36.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6285795064570466, "accuracy_std_dev": 0.020919173358428876, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.01718929421673217, "avg_duration_per_task_ms": 631165.4817518248, "avg_output_tokens_per_task": 83912.1010668164, "last_run_timestamp": "2026-10-03T17:40:51.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.6280566473616473, "accuracy_std_dev": 0.014452741297068885, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.01670690218790219, "avg_duration_per_task_ms": 81944.96975546975, "avg_output_tokens_per_task": 55459.59009009009, "last_run_timestamp": "2026-10-24T16:45:19.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6275255000000001, "accuracy_std_dev": 0.010334903785231893, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0036913661616161616, "avg_duration_per_task_ms": 150999.66287878787, "avg_output_tokens_per_task": 12500.281565656565, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.627104375, "accuracy_std_dev": 0.008333089267155095, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.024497946969696968, "avg_duration_per_task_ms": 1060409.1414141415, "avg_output_tokens_per_task": 101874.34027777778, "last_run_timestamp": "2026-10-23T09:03:34.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6265433333333333, "accuracy_std_dev": 0.01630548430913829, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004165183501683501, "avg_duration_per_task_ms": 241184.85185185185, "avg_output_tokens_per_task": 13613.127104377105, "last_run_timestamp": "2026-10-23T08:08:11.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.6212125000000001, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023548611111111114, "avg_duration_per_task_ms": 64938.90151515151, "avg_output_tokens_per_task": 2165.318181818182, "last_run_timestamp": "2026-10-01T19:21:21.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.621212, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00552190909090909, "avg_duration_per_task_ms": 174055.64141414143, "avg_output_tokens_per_task": 10415.944444444445, "last_run_timestamp": "2026-10-04T03:09:40.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.6199326452702703, "accuracy_std_dev": 0.053313699046305146, "run_count": 3, "total_questions": 592, "avg_cost_per_task_usd": 0.037519454391891895, "avg_duration_per_task_ms": 2801762.006756757, "avg_output_tokens_per_task": 26159.864864864863, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.6184415959279561, "accuracy_std_dev": 0.18122512960844483, "run_count": 13, "total_questions": 2554, "avg_cost_per_task_usd": 0.026508938919342206, "avg_duration_per_task_ms": 731184.3801879404, "avg_output_tokens_per_task": 37523.92443226312, "last_run_timestamp": "2026-10-30T02:15:35.781Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.6182949425287356, "accuracy_std_dev": 0.0013475000000000015, "run_count": 2, "total_questions": 348, "avg_cost_per_task_usd": 0.011105244252873564, "avg_duration_per_task_ms": 345775.4454022989, "avg_output_tokens_per_task": 38606.68103448276, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.6170940030769231, "accuracy_std_dev": 0.014320214565431623, "run_count": 5, "total_questions": 975, "avg_cost_per_task_usd": 0.015717948717948717, "avg_duration_per_task_ms": 780521.6543589743, "avg_output_tokens_per_task": 16712.84717948718, "last_run_timestamp": "2026-10-03T11:06:38.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.6139909190897598, "accuracy_std_dev": 0.11036434192418933, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.009721548672566372, "avg_duration_per_task_ms": 349467.04930467764, "avg_output_tokens_per_task": 27722.50379266751, "last_run_timestamp": "2026-10-24T16:05:40.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6123375670628184, "accuracy_std_dev": 0.008273645159312946, "run_count": 3, "total_questions": 589, "avg_cost_per_task_usd": 0.003321100169779287, "avg_duration_per_task_ms": 426798.44821731746, "avg_output_tokens_per_task": 13430.818336162989, "last_run_timestamp": "2026-10-08T22:16:42.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.611953, "accuracy_std_dev": 0.007760307500350757, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005744295454545455, "avg_duration_per_task_ms": 648984.4204545454, "avg_output_tokens_per_task": 19126.670454545456, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.6104378, "accuracy_std_dev": 0.012074440366327535, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.003635177777777778, "avg_duration_per_task_ms": 32727.437373737375, "avg_output_tokens_per_task": 2146.0969696969696, "last_run_timestamp": "2026-10-03T17:03:29.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.6102123225480283, "accuracy_std_dev": 0.005364099163885748, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.004066017189079878, "avg_duration_per_task_ms": 27933.76643073812, "avg_output_tokens_per_task": 2160.4469160768454, "last_run_timestamp": "2026-10-03T17:03:50.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.609428, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00778269696969697, "avg_duration_per_task_ms": 302947.1666666667, "avg_output_tokens_per_task": 15369.68686868687, "last_run_timestamp": "2026-10-08T22:57:53.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6073235, "accuracy_std_dev": 0.010738651928896826, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008191137626262626, "avg_duration_per_task_ms": 318721.5845959596, "avg_output_tokens_per_task": 17321.883838383837, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Mara", "accuracy": 0.6061323018867925, "accuracy_std_dev": 0.029270000000000018, "run_count": 2, "total_questions": 212, "avg_cost_per_task_usd": 0.034763094339622644, "avg_duration_per_task_ms": 71660.68867924529, "avg_output_tokens_per_task": 7461.462264150943, "last_run_timestamp": "2026-10-17T16:06:00.921Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6059645129698252, "accuracy_std_dev": 0.012733602453351516, "run_count": 10, "total_questions": 1889, "avg_cost_per_task_usd": 0.008509499205929062, "avg_duration_per_task_ms": 554697.4531498147, "avg_output_tokens_per_task": 13980.221281101112, "last_run_timestamp": "2026-10-08T22:50:36.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.6042256518448439, "accuracy_std_dev": 0.02010368015853481, "run_count": 6, "total_questions": 1057, "avg_cost_per_task_usd": 0.0021907710501419112, "avg_duration_per_task_ms": 51585.36707663198, "avg_output_tokens_per_task": 2058.464522232734, "last_run_timestamp": "2026-10-08T23:15:07.689Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.6040405999999999, "accuracy_std_dev": 0.007924755859961838, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.003472381818181818, "avg_duration_per_task_ms": 58103.21111111111, "avg_output_tokens_per_task": 2119.49797979798, "last_run_timestamp": "2026-10-03T16:57:23.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.6033671999999999, "accuracy_std_dev": 0.005694314722598297, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010998522222222222, "avg_duration_per_task_ms": 31713.89797979798, "avg_output_tokens_per_task": 2095.668686868687, "last_run_timestamp": "2026-10-03T17:43:44.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.5997241451304901, "accuracy_std_dev": 0.07108217488715086, "run_count": 8, "total_questions": 1571, "avg_cost_per_task_usd": 0.012881001273074474, "avg_duration_per_task_ms": 1360475.176957352, "avg_output_tokens_per_task": 57168.979630808404, "last_run_timestamp": "2026-10-30T01:56:42.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.599327, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014991068181818182, "avg_duration_per_task_ms": 284433.01515151514, "avg_output_tokens_per_task": 12302.075757575758, "last_run_timestamp": "2026-08-15T15:26:03.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.599327, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012440297979797978, "avg_duration_per_task_ms": 230792.08080808082, "avg_output_tokens_per_task": 41242.71717171717, "last_run_timestamp": "2026-10-04T03:37:16.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.5990461666666667, "accuracy_std_dev": 0.015102210725769769, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003087529461279461, "avg_duration_per_task_ms": 677793.6026936027, "avg_output_tokens_per_task": 13663.569865319865, "last_run_timestamp": "2026-10-30T14:52:53.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.5983168000000001, "accuracy_std_dev": 0.003626295707743628, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.003269590909090909, "avg_duration_per_task_ms": 65326.75858585859, "avg_output_tokens_per_task": 2077.4313131313133, "last_run_timestamp": "2026-10-03T16:59:44.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5963805, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01735105808080808, "avg_duration_per_task_ms": 112232.29292929293, "avg_output_tokens_per_task": 43198.833333333336, "last_run_timestamp": "2026-10-01T19:14:48.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5948373333333333, "accuracy_std_dev": 0.010316724200162651, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04306378282828283, "avg_duration_per_task_ms": 401108.6296296296, "avg_output_tokens_per_task": 21434.68181818182, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.5942281016850668, "accuracy_std_dev": 0.00965016140081236, "run_count": 9, "total_questions": 1721, "avg_cost_per_task_usd": 0.009055742010459035, "avg_duration_per_task_ms": 523063.0418361418, "avg_output_tokens_per_task": 45085.088901801275, "last_run_timestamp": "2026-10-03T17:04:03.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.591751125, "accuracy_std_dev": 0.0083327609685731, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01027929356060606, "avg_duration_per_task_ms": 11398.482323232323, "avg_output_tokens_per_task": 13251.253787878788, "last_run_timestamp": "2026-10-24T16:20:41.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.591751, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0029208080808080807, "avg_duration_per_task_ms": 521963.51515151514, "avg_output_tokens_per_task": 11688.588383838383, "last_run_timestamp": "2026-10-01T16:32:22.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.5878379610492847, "accuracy_std_dev": 0.08614130418814413, "run_count": 8, "total_questions": 1258, "avg_cost_per_task_usd": 0.056543784578696354, "avg_duration_per_task_ms": 347246.7694753577, "avg_output_tokens_per_task": 21441.431637519872, "last_run_timestamp": "2026-10-24T16:28:50.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.5875422, "accuracy_std_dev": 0.007299620439447542, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006416512121212121, "avg_duration_per_task_ms": 659187.0757575758, "avg_output_tokens_per_task": 42517.9595959596, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5835020000000001, "accuracy_std_dev": 0.037153058162148644, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02795441818181818, "avg_duration_per_task_ms": 1395358.9363636363, "avg_output_tokens_per_task": 69775.76262626263, "last_run_timestamp": "2026-10-13T21:28:41.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.5800594410646388, "accuracy_std_dev": 0.007005609872630659, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.025549254752851712, "avg_duration_per_task_ms": 1152259.712294043, "avg_output_tokens_per_task": 67901.9062103929, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Google", "accuracy": 0.5774414999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.010289136363636365, "avg_duration_per_task_ms": 331432.74242424243, "avg_output_tokens_per_task": 40883.856060606064, "last_run_timestamp": "2026-10-17T16:06:05.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.5736152536443148, "accuracy_std_dev": 0.03731433483807747, "run_count": 7, "total_questions": 686, "avg_cost_per_task_usd": 0.03136954518950437, "avg_duration_per_task_ms": 62083.20991253644, "avg_output_tokens_per_task": 6724.625364431487, "last_run_timestamp": "2026-10-24T16:01:29.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Fireworks", "accuracy": 0.569024, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.019884464646464647, "avg_duration_per_task_ms": 348444.5303030303, "avg_output_tokens_per_task": 99240.31818181818, "last_run_timestamp": "2026-10-30T21:00:04.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.56060625, "accuracy_std_dev": 0.009297342103391676, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.017428513888888886, "avg_duration_per_task_ms": 127446.60858585859, "avg_output_tokens_per_task": 43389.579545454544, "last_run_timestamp": "2026-10-01T19:27:01.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.5558924000000001, "accuracy_std_dev": 0.010399706661247687, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004659020202020202, "avg_duration_per_task_ms": 490788.4282828283, "avg_output_tokens_per_task": 16414.2101010101, "last_run_timestamp": "2026-10-08T23:30:03.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.5526449293849659, "accuracy_std_dev": 0.02398069638824749, "run_count": 7, "total_questions": 1317, "avg_cost_per_task_usd": 0.005556834472285498, "avg_duration_per_task_ms": 809029.5223993926, "avg_output_tokens_per_task": 14330.80941533789, "last_run_timestamp": "2026-10-30T20:09:39.337Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.5525624444444444, "accuracy_std_dev": 0.13700937369238736, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0030681475869809203, "avg_duration_per_task_ms": 217856.8911335578, "avg_output_tokens_per_task": 7427.39898989899, "last_run_timestamp": "2026-10-24T16:03:25.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5521885000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01778226515151515, "avg_duration_per_task_ms": 116667.00757575757, "avg_output_tokens_per_task": 44273.378787878784, "last_run_timestamp": "2026-10-01T19:15:36.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.54798, "accuracy_std_dev": 0.06951982075998184, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005136612373737373, "avg_duration_per_task_ms": 120175.0505050505, "avg_output_tokens_per_task": 4605.713383838384, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5420490229357798, "accuracy_std_dev": 0.06975384672944968, "run_count": 3, "total_questions": 218, "avg_cost_per_task_usd": 0.03485861009174312, "avg_duration_per_task_ms": 397279.0229357798, "avg_output_tokens_per_task": 16660.94495412844, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.5415076988564168, "accuracy_std_dev": 0.08088323645385365, "run_count": 4, "total_questions": 787, "avg_cost_per_task_usd": 0.0033490050825921223, "avg_duration_per_task_ms": 120991.54510800508, "avg_output_tokens_per_task": 3568.362134688691, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.538721, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005473, "avg_duration_per_task_ms": 117831.5808080808, "avg_output_tokens_per_task": 11819.661616161617, "last_run_timestamp": "2026-10-04T03:06:12.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.5373176666666667, "accuracy_std_dev": 0.013403444242266851, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0019087592592592591, "avg_duration_per_task_ms": 97924.95622895623, "avg_output_tokens_per_task": 1886.641414141414, "last_run_timestamp": "2026-10-08T22:10:22.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.537037, "accuracy_std_dev": null, "run_count": 1, "total_questions": 126, "avg_cost_per_task_usd": 0.007595468253968254, "avg_duration_per_task_ms": 1060044.9047619049, "avg_output_tokens_per_task": 22408.52380952381, "last_run_timestamp": "2026-08-08T10:11:59.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.5280586666666666, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01799902861952862, "avg_duration_per_task_ms": 14939.786195286195, "avg_output_tokens_per_task": 1607.080808080808, "last_run_timestamp": "2026-10-02T23:16:52.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.5266555, "accuracy_std_dev": 0.023895888382383546, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01547567760942761, "avg_duration_per_task_ms": 13716.89898989899, "avg_output_tokens_per_task": 1354.1801346801346, "last_run_timestamp": "2026-10-02T23:17:37.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.5238700798479087, "accuracy_std_dev": 0.008445731536551466, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.016655803548795946, "avg_duration_per_task_ms": 761511.7452471483, "avg_output_tokens_per_task": 42502.96070975919, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.5202019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017511833333333334, "avg_duration_per_task_ms": 16177.118686868687, "avg_output_tokens_per_task": 1559.209595959596, "last_run_timestamp": "2026-10-01T16:04:12.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.5193605, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.003164820707070707, "avg_duration_per_task_ms": 106661.49242424243, "avg_output_tokens_per_task": 2729.3585858585857, "last_run_timestamp": "2026-08-15T08:46:42.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.5168355, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017194939393939394, "avg_duration_per_task_ms": 13199.247474747475, "avg_output_tokens_per_task": 1523.2777777777778, "last_run_timestamp": "2026-10-01T16:28:34.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.5151519999999999, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018041959595959597, "avg_duration_per_task_ms": 13498.479797979799, "avg_output_tokens_per_task": 1611.5353535353536, "last_run_timestamp": "2026-10-01T16:26:08.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.5151515, "accuracy_std_dev": 0.008417499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004014873737373737, "avg_duration_per_task_ms": 26643.18181818182, "avg_output_tokens_per_task": 2582.7247474747473, "last_run_timestamp": "2026-08-15T09:25:42.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.510943, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013169762626262625, "avg_duration_per_task_ms": 12925.411616161617, "avg_output_tokens_per_task": 1121.608585858586, "last_run_timestamp": "2026-10-01T16:03:45.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.5084175, "accuracy_std_dev": 0.025252499999999983, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004674189393939395, "avg_duration_per_task_ms": 291399.3181818182, "avg_output_tokens_per_task": 2695.719696969697, "last_run_timestamp": "2026-08-15T09:11:29.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.5075759999999999, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008372954545454546, "avg_duration_per_task_ms": 96066.76515151515, "avg_output_tokens_per_task": 2787.2348484848485, "last_run_timestamp": "2026-08-15T08:49:03.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.5050510000000001, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008273518518518518, "avg_duration_per_task_ms": 18715.93265993266, "avg_output_tokens_per_task": 1873.8602693602693, "last_run_timestamp": "2026-10-02T23:16:50.042Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009755954545454545, "avg_duration_per_task_ms": 700759.0555555555, "avg_output_tokens_per_task": 64762.606060606064, "last_run_timestamp": "2026-10-01T08:24:17.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.4995792500000001, "accuracy_std_dev": 0.02025051336849268, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0008581388888888889, "avg_duration_per_task_ms": 12992.723484848484, "avg_output_tokens_per_task": 1853.0088383838383, "last_run_timestamp": "2026-10-01T19:12:29.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.496001625, "accuracy_std_dev": 0.012540032196305345, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0023573857323232325, "avg_duration_per_task_ms": 9976.099116161617, "avg_output_tokens_per_task": 2337.1717171717173, "last_run_timestamp": "2026-10-24T16:02:08.875Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.49242400000000003, "accuracy_std_dev": 0.0008419999999999817, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0017634166666666666, "avg_duration_per_task_ms": 30891.38888888889, "avg_output_tokens_per_task": 1731.7373737373737, "last_run_timestamp": "2026-10-01T16:05:56.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.49168807800511505, "accuracy_std_dev": 0.022345304455019185, "run_count": 4, "total_questions": 782, "avg_cost_per_task_usd": 0.010985424552429669, "avg_duration_per_task_ms": 526999.0754475703, "avg_output_tokens_per_task": 27562.6368286445, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.49074049999999997, "accuracy_std_dev": 0.009259500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008383863636363636, "avg_duration_per_task_ms": 22708.954545454544, "avg_output_tokens_per_task": 1902.2954545454545, "last_run_timestamp": "2026-10-01T19:13:37.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.48782048076923085, "accuracy_std_dev": 0.015008531315555174, "run_count": 4, "total_questions": 780, "avg_cost_per_task_usd": 0.011197282051282051, "avg_duration_per_task_ms": 480526.36282051285, "avg_output_tokens_per_task": 27926.11923076923, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.4844273749999999, "accuracy_std_dev": 0.0055676718190258885, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0023532323232323235, "avg_duration_per_task_ms": 15477.722222222223, "avg_output_tokens_per_task": 2211.7329545454545, "last_run_timestamp": "2026-10-24T16:21:36.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.4819933646302252, "accuracy_std_dev": 0.014921479863223815, "run_count": 16, "total_questions": 3110, "avg_cost_per_task_usd": 0.0012891855305466235, "avg_duration_per_task_ms": 59448.33536977492, "avg_output_tokens_per_task": 2320.65884244373, "last_run_timestamp": "2026-10-30T00:42:34.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.48148112499999995, "accuracy_std_dev": 0.009923980116836928, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0026001357323232324, "avg_duration_per_task_ms": 50611.480429292926, "avg_output_tokens_per_task": 2853.9835858585857, "last_run_timestamp": "2026-10-24T16:03:59.174Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.478956, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.048212727272727275, "avg_duration_per_task_ms": 51496.83838383838, "avg_output_tokens_per_task": 16895.032828282827, "last_run_timestamp": "2026-08-15T14:46:10.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.47664124999999996, "accuracy_std_dev": 0.014045976548019012, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0013353724747474747, "avg_duration_per_task_ms": 46855.55997474748, "avg_output_tokens_per_task": 2393.6155303030305, "last_run_timestamp": "2026-10-24T16:03:59.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4765793529411765, "accuracy_std_dev": 0.006944000000000006, "run_count": 2, "total_questions": 306, "avg_cost_per_task_usd": 0.0011232875816993464, "avg_duration_per_task_ms": 126677.51307189543, "avg_output_tokens_per_task": 2069.3627450980393, "last_run_timestamp": "2026-10-01T16:04:45.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.47558900000000004, "accuracy_std_dev": 0.011198623844026553, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010472727272727272, "avg_duration_per_task_ms": 222278.38005050505, "avg_output_tokens_per_task": 2355.248737373737, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.47474712499999994, "accuracy_std_dev": 0.009182339672402404, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0014147474747474747, "avg_duration_per_task_ms": 78596.79671717172, "avg_output_tokens_per_task": 2360.6445707070707, "last_run_timestamp": "2026-10-24T16:04:55.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.4728232857142857, "accuracy_std_dev": 0.008882080414508524, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0031378203463203465, "avg_duration_per_task_ms": 63855.042568542565, "avg_output_tokens_per_task": 2211.1046176046175, "last_run_timestamp": "2026-10-17T16:05:55.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.469697, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006110727272727273, "avg_duration_per_task_ms": 42879.368686868685, "avg_output_tokens_per_task": 2318.919191919192, "last_run_timestamp": "2026-10-04T03:02:23.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.46877630000000003, "accuracy_std_dev": 0.013737037653161437, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.0013216873417721518, "avg_duration_per_task_ms": 88415.32848101266, "avg_output_tokens_per_task": 3031.9816455696205, "last_run_timestamp": "2026-10-24T16:02:28.569Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.468294, "accuracy_std_dev": 0.012796634635715754, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0007987283950617283, "avg_duration_per_task_ms": 163190.25925925927, "avg_output_tokens_per_task": 2245.347923681257, "last_run_timestamp": "2026-10-24T16:22:30.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.4637588786346397, "accuracy_std_dev": 0.005380526437998433, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.0022411302149178258, "avg_duration_per_task_ms": 37374.46017699115, "avg_output_tokens_per_task": 2724.637168141593, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.461279, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0011232171717171719, "avg_duration_per_task_ms": 136788.7626262626, "avg_output_tokens_per_task": 2036.2272727272727, "last_run_timestamp": "2026-10-01T16:27:00.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.45959600000000006, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010278964646464647, "avg_duration_per_task_ms": 96379.95959595959, "avg_output_tokens_per_task": 1848.388888888889, "last_run_timestamp": "2026-10-01T16:47:11.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.457071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0017076717171717172, "avg_duration_per_task_ms": 27081.62626262626, "avg_output_tokens_per_task": 1337.1363636363637, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.442761, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0013494898989898992, "avg_duration_per_task_ms": 69228.69696969698, "avg_output_tokens_per_task": 1528.5050505050506, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.43602700000000005, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010042146464646464, "avg_duration_per_task_ms": 21558.090909090908, "avg_output_tokens_per_task": 1481.030303030303, "last_run_timestamp": "2026-10-01T19:16:07.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.43139700000000003, "accuracy_std_dev": 0.012951914704011906, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010746982323232325, "avg_duration_per_task_ms": 24227.715909090908, "avg_output_tokens_per_task": 1509.2487373737374, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.429293, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0015700353535353536, "avg_duration_per_task_ms": 59122.85858585859, "avg_output_tokens_per_task": 1664.8737373737374, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4276093333333334, "accuracy_std_dev": 0.00629925838668507, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0010292508417508418, "avg_duration_per_task_ms": 16725.542087542086, "avg_output_tokens_per_task": 1523.1902356902358, "last_run_timestamp": "2026-10-02T23:19:57.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.422559, "accuracy_std_dev": 0.09221941654915557, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.019149646464646466, "avg_duration_per_task_ms": 3630846.4646464647, "avg_output_tokens_per_task": 17674.203703703704, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.421027614334471, "accuracy_std_dev": 0.09192114718900765, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.004391554607508533, "avg_duration_per_task_ms": 42970.98634812287, "avg_output_tokens_per_task": 1850.198521046644, "last_run_timestamp": "2026-10-24T16:02:47.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.405724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0003259848484848485, "avg_duration_per_task_ms": 29056.348484848484, "avg_output_tokens_per_task": 1917.1515151515152, "last_run_timestamp": "2026-10-08T22:01:16.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy-mt2-30b-a3b-20260521", "model_slug": "tencent/hy-mt2-30b-a3b", "model_name": "Tencent: Hy-MT2-30B-A3B", "provider_name": "Reka", "accuracy": 0.388889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00023147979797979797, "avg_duration_per_task_ms": 8604.353535353535, "avg_output_tokens_per_task": 710.540404040404, "last_run_timestamp": "2026-09-24T06:00:03.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.3855215, "accuracy_std_dev": 0.011293261320362695, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0001567070707070707, "avg_duration_per_task_ms": 7111.70202020202, "avg_output_tokens_per_task": 932.0808080808081, "last_run_timestamp": "2026-10-01T19:39:32.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "NextBit", "accuracy": 0.3804713333333334, "accuracy_std_dev": 0.013950341891469493, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00034128451178451176, "avg_duration_per_task_ms": 43460.74410774411, "avg_output_tokens_per_task": 1987.3602693602693, "last_run_timestamp": "2026-10-30T14:15:27.188Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.346801, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008616373737373737, "avg_duration_per_task_ms": 1179181.5303030303, "avg_output_tokens_per_task": 32995.232323232325, "last_run_timestamp": "2026-10-03T00:19:24.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.32428450000000003, "accuracy_std_dev": 0.09813129033340996, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008669871843434344, "avg_duration_per_task_ms": 575303.3933080808, "avg_output_tokens_per_task": 21440.64835858586, "last_run_timestamp": "2026-10-24T16:36:16.779Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.28756325, "accuracy_std_dev": 0.017090551700793627, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.002848802398989899, "avg_duration_per_task_ms": 50657.708333333336, "avg_output_tokens_per_task": 35609.12626262626, "last_run_timestamp": "2026-10-24T16:02:28.572Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.2873176666666667, "accuracy_std_dev": 0.0007938452130121062, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00006128619528619528, "avg_duration_per_task_ms": 10231.205387205388, "avg_output_tokens_per_task": 580.9023569023569, "last_run_timestamp": "2026-10-08T22:00:27.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.286195, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005941969696969697, "avg_duration_per_task_ms": 14998.156565656565, "avg_output_tokens_per_task": 1642.1363636363637, "last_run_timestamp": "2026-10-13T18:56:05.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.280303, "accuracy_std_dev": 0.021444131149337797, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0013256224747474747, "avg_duration_per_task_ms": 144884.44255050505, "avg_output_tokens_per_task": 18578.765151515152, "last_run_timestamp": "2026-10-24T16:08:06.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.27693599999999996, "accuracy_std_dev": 0.010943000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0007441464646464646, "avg_duration_per_task_ms": 48473.70202020202, "avg_output_tokens_per_task": 2545.467171717172, "last_run_timestamp": "2026-08-15T09:28:07.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.27665533333333336, "accuracy_std_dev": 0.0007938452130121063, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00006201683501683503, "avg_duration_per_task_ms": 9496.395622895623, "avg_output_tokens_per_task": 580.2053872053872, "last_run_timestamp": "2026-10-01T20:43:57.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.2745902896174863, "accuracy_std_dev": 0.009941000000000005, "run_count": 2, "total_questions": 366, "avg_cost_per_task_usd": 0.0000641912568306011, "avg_duration_per_task_ms": 8433.43169398907, "avg_output_tokens_per_task": 532.0136612021857, "last_run_timestamp": "2026-10-01T16:01:04.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.27161569453924916, "accuracy_std_dev": 0.020450238993441835, "run_count": 3, "total_questions": 586, "avg_cost_per_task_usd": 0.000055156996587030725, "avg_duration_per_task_ms": 10158.571672354949, "avg_output_tokens_per_task": 550.3515358361775, "last_run_timestamp": "2026-10-01T20:47:49.128Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-4b-it", "model_slug": "google/gemma-3-4b-it", "model_name": "Google: Gemma 3 4B", "provider_name": "Parasail", "accuracy": 0.271044, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00011306565656565657, "avg_duration_per_task_ms": 10846.11616161616, "avg_output_tokens_per_task": 1804.4646464646464, "last_run_timestamp": "2026-10-15T03:00:03.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.27046308294930876, "accuracy_std_dev": 0.013958073966682333, "run_count": 8, "total_questions": 1519, "avg_cost_per_task_usd": 0.00941191836734694, "avg_duration_per_task_ms": 387844.9934167215, "avg_output_tokens_per_task": 42072.92560895326, "last_run_timestamp": "2026-10-24T16:10:43.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.265432, "accuracy_std_dev": 0.010735715625890985, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00039356734006734006, "avg_duration_per_task_ms": 391096.85185185185, "avg_output_tokens_per_task": 9436.173961840628, "last_run_timestamp": "2026-10-24T16:01:29.844Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.254208875, "accuracy_std_dev": 0.012051921116957867, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0004300145202020202, "avg_duration_per_task_ms": 75497.70643939394, "avg_output_tokens_per_task": 8276.654671717171, "last_run_timestamp": "2026-10-24T16:02:28.577Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.253367, "accuracy_std_dev": 0.23989899999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.016424065656565658, "avg_duration_per_task_ms": 206615.3207070707, "avg_output_tokens_per_task": 20468.073232323233, "last_run_timestamp": "2026-08-15T15:24:35.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.21801350000000003, "accuracy_std_dev": 0.11886565594085619, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0018304709595959596, "avg_duration_per_task_ms": 56840.131313131315, "avg_output_tokens_per_task": 1584.419191919192, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.17239079999999998, "accuracy_std_dev": 0.1395319132204529, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.001048532323232323, "avg_duration_per_task_ms": 421240.27575757576, "avg_output_tokens_per_task": 3399.7606060606063, "last_run_timestamp": "2026-10-24T16:03:19.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.17174282233502539, "accuracy_std_dev": 0.01788397279378631, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.019897788494077834, "avg_duration_per_task_ms": 1784246.4196277496, "avg_output_tokens_per_task": 110294.06598984772, "last_run_timestamp": "2026-10-08T23:13:58.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.1161615, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017850737373737373, "avg_duration_per_task_ms": 46853.982323232325, "avg_output_tokens_per_task": 11776.444444444445, "last_run_timestamp": "2026-10-01T16:00:58.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.112795, "accuracy_std_dev": 0.006733999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018038613636363634, "avg_duration_per_task_ms": 21125.79292929293, "avg_output_tokens_per_task": 11795.222222222223, "last_run_timestamp": "2026-10-01T16:00:58.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017984406565656568, "avg_duration_per_task_ms": 21554.590909090908, "avg_output_tokens_per_task": 11812.823232323231, "last_run_timestamp": "2026-10-01T16:02:13.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.0025255000000000004, "accuracy_std_dev": 0.0008415000000000001, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0034195530303030306, "avg_duration_per_task_ms": 159765.32575757575, "avg_output_tokens_per_task": 3219.0782828282827, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.001684, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0034544065656565656, "avg_duration_per_task_ms": 164367.952020202, "avg_output_tokens_per_task": 3253.931818181818, "last_run_timestamp": "2026-08-15T14:54:58.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8368423578947368, "accuracy_std_dev": 0.014073500000000017, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.008221831578947368, "avg_duration_per_task_ms": 31565.694736842106, "avg_output_tokens_per_task": 9295.157894736842, "last_run_timestamp": "2026-10-10T11:02:59.339Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8233334999999999, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.7708268799999999, "avg_duration_per_task_ms": 139666.89, "avg_output_tokens_per_task": 5287.85, "last_run_timestamp": "2026-08-15T08:09:14.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.813333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00614692, "avg_duration_per_task_ms": 141813.64, "avg_output_tokens_per_task": 15680.88, "last_run_timestamp": "2026-08-08T08:30:38.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20567228, "avg_duration_per_task_ms": 69455.26, "avg_output_tokens_per_task": 15737.28, "last_run_timestamp": "2026-08-15T00:15:11.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8013334, "accuracy_std_dev": 0.021249926537284785, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.8758712919999999, "avg_duration_per_task_ms": 139145.43, "avg_output_tokens_per_task": 5724.488, "last_run_timestamp": "2026-10-24T11:41:42.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8, "accuracy_std_dev": 0.02326845525989678, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.8815157218181818, "avg_duration_per_task_ms": 126457.99454545455, "avg_output_tokens_per_task": 5745.552727272728, "last_run_timestamp": "2026-10-24T11:46:02.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.8, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.39313749000000003, "avg_duration_per_task_ms": 477750.16, "avg_output_tokens_per_task": 34082.08, "last_run_timestamp": "2026-10-01T16:00:45.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.7996450638297874, "accuracy_std_dev": 0.011700000000000044, "run_count": 2, "total_questions": 94, "avg_cost_per_task_usd": 0.06506034042553191, "avg_duration_per_task_ms": 304476.60638297873, "avg_output_tokens_per_task": 17465.68085106383, "last_run_timestamp": "2026-10-24T19:25:11.376Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.7981576231155779, "accuracy_std_dev": 0.02702867818632647, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.20028302512562812, "avg_duration_per_task_ms": 842159.8542713568, "avg_output_tokens_per_task": 34248.3391959799, "last_run_timestamp": "2026-10-24T19:22:19.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.7977776666666667, "accuracy_std_dev": 0.01911642045177097, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008363246666666668, "avg_duration_per_task_ms": 272879.14, "avg_output_tokens_per_task": 12625.026666666667, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.7975757272727273, "accuracy_std_dev": 0.026707871060082322, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.8921026254545454, "avg_duration_per_task_ms": 120870.10727272727, "avg_output_tokens_per_task": 5752.245454545455, "last_run_timestamp": "2026-10-24T11:40:21.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.7939394545454546, "accuracy_std_dev": 0.01716339453797282, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.8242855072727272, "avg_duration_per_task_ms": 128426.18181818182, "avg_output_tokens_per_task": 5739.361818181818, "last_run_timestamp": "2026-10-24T12:06:18.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7936708987341772, "accuracy_std_dev": 0.03037925132800345, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.12511070886075948, "avg_duration_per_task_ms": 2933394.1088607595, "avg_output_tokens_per_task": 27206.47088607595, "last_run_timestamp": "2026-10-17T19:41:14.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7936365, "accuracy_std_dev": 0.0214125331263343, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.9285239381818181, "avg_duration_per_task_ms": 94655, "avg_output_tokens_per_task": 5727.099090909091, "last_run_timestamp": "2026-10-24T11:52:13.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00828808, "avg_duration_per_task_ms": 196026.56, "avg_output_tokens_per_task": 16303, "last_run_timestamp": "2026-10-14T19:21:35.384Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03146312, "avg_duration_per_task_ms": 636210.72, "avg_output_tokens_per_task": 16745.44, "last_run_timestamp": "2026-10-14T19:17:59.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.7920001999999999, "accuracy_std_dev": 0.014847359662916514, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.05966906, "avg_duration_per_task_ms": 918022.212, "avg_output_tokens_per_task": 41767.624, "last_run_timestamp": "2026-10-24T12:47:38.794Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.79166675, "accuracy_std_dev": 0.012801703448662584, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.71616581, "avg_duration_per_task_ms": 111830.815, "avg_output_tokens_per_task": 6900.955, "last_run_timestamp": "2026-10-01T08:03:01.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.7906666, "accuracy_std_dev": 0.009977824303925176, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.814870324, "avg_duration_per_task_ms": 182126.652, "avg_output_tokens_per_task": 7046.908, "last_run_timestamp": "2026-10-01T08:05:22.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.7890909090909091, "accuracy_std_dev": 0.021324416182798293, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.47806725999999994, "avg_duration_per_task_ms": 120590.92909090909, "avg_output_tokens_per_task": 7057.872727272727, "last_run_timestamp": "2026-10-24T11:55:24.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.7889448542713567, "accuracy_std_dev": 0.022421675465662684, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.02856220603015075, "avg_duration_per_task_ms": 373105.72361809044, "avg_output_tokens_per_task": 11144.979899497488, "last_run_timestamp": "2026-08-29T17:06:21.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.7883330000000001, "accuracy_std_dev": 0.02598076211353313, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06002512, "avg_duration_per_task_ms": 888300.44, "avg_output_tokens_per_task": 41523.41, "last_run_timestamp": "2026-10-24T12:33:32.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.7880001999999999, "accuracy_std_dev": 0.03709450273773731, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.043486136, "avg_duration_per_task_ms": 113472.604, "avg_output_tokens_per_task": 4980.336, "last_run_timestamp": "2026-10-30T19:04:49.114Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.7876542314814815, "accuracy_std_dev": 0.022797342804780508, "run_count": 11, "total_questions": 540, "avg_cost_per_task_usd": 0.07938424444444445, "avg_duration_per_task_ms": 1140818.088888889, "avg_output_tokens_per_task": 26072.82222222222, "last_run_timestamp": "2026-10-24T19:36:21.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.7874881680993314, "accuracy_std_dev": 0.015935514274356157, "run_count": 21, "total_questions": 1047, "avg_cost_per_task_usd": 0.08563230085959884, "avg_duration_per_task_ms": 141393.38681948424, "avg_output_tokens_per_task": 12084.589302769818, "last_run_timestamp": "2026-10-29T22:52:40.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01468154, "avg_duration_per_task_ms": 178223.4, "avg_output_tokens_per_task": 11352.52, "last_run_timestamp": "2026-08-08T08:29:14.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.49828548, "avg_duration_per_task_ms": 348417.68, "avg_output_tokens_per_task": 21971.04, "last_run_timestamp": "2026-10-14T15:16:05.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.78666675, "accuracy_std_dev": 0.024944159861328273, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.15998305999999998, "avg_duration_per_task_ms": 73714.275, "avg_output_tokens_per_task": 6299.305, "last_run_timestamp": "2026-10-24T11:26:51.227Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7862320000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.029582717391304348, "avg_duration_per_task_ms": 2435445.304347826, "avg_output_tokens_per_task": 108721.58695652174, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7860360641891891, "accuracy_std_dev": 0.025554613476174253, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.09440423986486486, "avg_duration_per_task_ms": 381879.5878378378, "avg_output_tokens_per_task": 16484.91891891892, "last_run_timestamp": "2026-10-10T11:07:21.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.78583325, "accuracy_std_dev": 0.02190571019021068, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.18706350749999998, "avg_duration_per_task_ms": 433314.44, "avg_output_tokens_per_task": 30794.75, "last_run_timestamp": "2026-10-24T18:46:07.800Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.78583325, "accuracy_std_dev": 0.016811747713652493, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07747623, "avg_duration_per_task_ms": 47946.0625, "avg_output_tokens_per_task": 4735.27, "last_run_timestamp": "2026-10-24T11:58:13.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7857144285714285, "accuracy_std_dev": 0.013058229959434387, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.17514343714285713, "avg_duration_per_task_ms": 72438.17428571428, "avg_output_tokens_per_task": 6286.754285714285, "last_run_timestamp": "2026-10-29T00:13:53.374Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.7853333, "accuracy_std_dev": 0.011469724399914758, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.17582804400000004, "avg_duration_per_task_ms": 67798.288, "avg_output_tokens_per_task": 6240.81, "last_run_timestamp": "2026-10-24T12:30:32.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.7842432787356322, "accuracy_std_dev": 0.019450509073790118, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.09254919540229886, "avg_duration_per_task_ms": 69468.46551724138, "avg_output_tokens_per_task": 4697.962643678161, "last_run_timestamp": "2026-10-24T13:01:18.254Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.783333375, "accuracy_std_dev": 0.023804813940343558, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.19070967249999998, "avg_duration_per_task_ms": 778974.4025, "avg_output_tokens_per_task": 30803.295, "last_run_timestamp": "2026-10-24T19:20:47.328Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7833330000000001, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07443635, "avg_duration_per_task_ms": 98397.15, "avg_output_tokens_per_task": 21518.67, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7833330000000001, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06795213, "avg_duration_per_task_ms": 191712.46, "avg_output_tokens_per_task": 8639.07, "last_run_timestamp": "2026-08-15T08:17:17.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.7831111333333334, "accuracy_std_dev": 0.01851801018780244, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.181601536, "avg_duration_per_task_ms": 65069.36666666667, "avg_output_tokens_per_task": 6347.210666666667, "last_run_timestamp": "2026-10-24T12:24:09.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.7831052054794521, "accuracy_std_dev": 0.017155537965592597, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.14992966438356164, "avg_duration_per_task_ms": 183129.3904109589, "avg_output_tokens_per_task": 16313.164383561643, "last_run_timestamp": "2026-10-15T21:26:28.195Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7826668, "accuracy_std_dev": 0.01717857063204037, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.01173768, "avg_duration_per_task_ms": 599702.892, "avg_output_tokens_per_task": 22568, "last_run_timestamp": "2026-10-24T14:27:48.875Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.7817936907630522, "accuracy_std_dev": 0.018188779316930522, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.08406518072289156, "avg_duration_per_task_ms": 255202.53012048194, "avg_output_tokens_per_task": 12331.429718875503, "last_run_timestamp": "2026-10-24T19:28:54.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.78166675, "accuracy_std_dev": 0.015183827290492351, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.014457230000000001, "avg_duration_per_task_ms": 401220.695, "avg_output_tokens_per_task": 21060.795, "last_run_timestamp": "2026-10-24T13:36:02.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.7809524285714285, "accuracy_std_dev": 0.025053077443455044, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08778324, "avg_duration_per_task_ms": 269653.90285714285, "avg_output_tokens_per_task": 20106.945714285714, "last_run_timestamp": "2026-10-24T12:40:07.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.7809524285714285, "accuracy_std_dev": 0.011507741653303821, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08243106857142857, "avg_duration_per_task_ms": 42328.47714285714, "avg_output_tokens_per_task": 4763.78, "last_run_timestamp": "2026-10-24T12:26:24.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7807406666666666, "accuracy_std_dev": 0.036709605237140126, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0812698177777778, "avg_duration_per_task_ms": 207249.8177777778, "avg_output_tokens_per_task": 25401.606666666667, "last_run_timestamp": "2026-10-28T03:33:36.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.7807406666666666, "accuracy_std_dev": 0.01923075098204262, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08175265555555557, "avg_duration_per_task_ms": 137863.5, "avg_output_tokens_per_task": 4670.331111111111, "last_run_timestamp": "2026-10-24T13:21:36.785Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.7804090721205597, "accuracy_std_dev": 0.030493720143653286, "run_count": 19, "total_questions": 929, "avg_cost_per_task_usd": 0.07802858557588804, "avg_duration_per_task_ms": 76989.98815931109, "avg_output_tokens_per_task": 4254.709364908504, "last_run_timestamp": "2026-10-24T12:32:51.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7801419645390072, "accuracy_std_dev": 0.026804102476217243, "run_count": 3, "total_questions": 141, "avg_cost_per_task_usd": 0.09157175886524822, "avg_duration_per_task_ms": 67531.68794326241, "avg_output_tokens_per_task": 7567.7943262411345, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.7800001111111111, "accuracy_std_dev": 0.0235177999322709, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06914864222222221, "avg_duration_per_task_ms": 267980.70666666667, "avg_output_tokens_per_task": 14178.462222222222, "last_run_timestamp": "2026-10-24T12:19:13.995Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00686524, "avg_duration_per_task_ms": 150471.32, "avg_output_tokens_per_task": 16577.68, "last_run_timestamp": "2026-08-08T08:27:33.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7799998, "accuracy_std_dev": 0.01633001326882497, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.055028052, "avg_duration_per_task_ms": 146838.836, "avg_output_tokens_per_task": 4580.112, "last_run_timestamp": "2026-10-17T10:59:26.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.7793333, "accuracy_std_dev": 0.02393506864853326, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.09867859, "avg_duration_per_task_ms": 301600.814, "avg_output_tokens_per_task": 20737.49, "last_run_timestamp": "2026-10-24T11:44:08.040Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.7792640401337794, "accuracy_std_dev": 0.028913483122066055, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.1886598294314381, "avg_duration_per_task_ms": 56557.29765886287, "avg_output_tokens_per_task": 4653.71237458194, "last_run_timestamp": "2026-10-24T12:16:13.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.779214709770115, "accuracy_std_dev": 0.033260406159852735, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.19238710632183906, "avg_duration_per_task_ms": 621470.8362068966, "avg_output_tokens_per_task": 29821.287356321838, "last_run_timestamp": "2026-10-24T19:18:27.046Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7783333125, "accuracy_std_dev": 0.02006927412650602, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.49599691625, "avg_duration_per_task_ms": 120067.109375, "avg_output_tokens_per_task": 7093.541875, "last_run_timestamp": "2026-10-24T11:54:38.630Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.77833325, "accuracy_std_dev": 0.011902275693643617, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0336993, "avg_duration_per_task_ms": 220470.06, "avg_output_tokens_per_task": 12832.385, "last_run_timestamp": "2026-10-24T11:48:43.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.77833325, "accuracy_std_dev": 0.005527984369324845, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12165100000000001, "avg_duration_per_task_ms": 123453.655, "avg_output_tokens_per_task": 4597.39, "last_run_timestamp": "2026-10-28T05:13:02.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.7781817878787878, "accuracy_std_dev": 0.022993088488233407, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.5197950163636365, "avg_duration_per_task_ms": 109495.11454545455, "avg_output_tokens_per_task": 7111.084848484848, "last_run_timestamp": "2026-10-24T11:58:58.417Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7781817272727273, "accuracy_std_dev": 0.03929598310743943, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1061598909090909, "avg_duration_per_task_ms": 180685.46181818182, "avg_output_tokens_per_task": 6783.716363636364, "last_run_timestamp": "2026-10-24T11:43:03.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.777778, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.012714583333333333, "avg_duration_per_task_ms": 516568.7291666667, "avg_output_tokens_per_task": 14230.208333333334, "last_run_timestamp": "2026-08-03T17:20:47.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.7771429999999999, "accuracy_std_dev": 0.017036713725699246, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.15415246857142859, "avg_duration_per_task_ms": 315516.36, "avg_output_tokens_per_task": 19966.505714285715, "last_run_timestamp": "2026-10-24T14:03:17.639Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.7771428571428571, "accuracy_std_dev": 0.018120213341763423, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09363316857142856, "avg_duration_per_task_ms": 130500.70857142858, "avg_output_tokens_per_task": 5167.157142857143, "last_run_timestamp": "2026-10-24T13:07:35.631Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.7769697272727273, "accuracy_std_dev": 0.02235858980790933, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.47973416363636356, "avg_duration_per_task_ms": 102499.14090909091, "avg_output_tokens_per_task": 7159.897272727273, "last_run_timestamp": "2026-10-24T11:37:51.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.77666675, "accuracy_std_dev": 0.015986122456290018, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.040007865, "avg_duration_per_task_ms": 202967.295, "avg_output_tokens_per_task": 13307.9975, "last_run_timestamp": "2026-10-24T19:48:57.247Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7766665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.017346550000000002, "avg_duration_per_task_ms": 335231.13, "avg_output_tokens_per_task": 20113.88, "last_run_timestamp": "2026-10-03T08:17:48.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.7766665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.41419341000000004, "avg_duration_per_task_ms": 38158.51, "avg_output_tokens_per_task": 9152, "last_run_timestamp": "2026-08-15T14:16:11.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.7764583125, "accuracy_std_dev": 0.027168853280730192, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.13175464625, "avg_duration_per_task_ms": 318529.455, "avg_output_tokens_per_task": 19311.1725, "last_run_timestamp": "2026-10-28T04:30:12.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.7764101538461539, "accuracy_std_dev": 0.017853329682318696, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.44019213846153843, "avg_duration_per_task_ms": 91000.5123076923, "avg_output_tokens_per_task": 5712.275384615385, "last_run_timestamp": "2026-10-24T12:19:07.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.7759999000000001, "accuracy_std_dev": 0.02174607193242035, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.08178571999999999, "avg_duration_per_task_ms": 336667.87, "avg_output_tokens_per_task": 20194.232, "last_run_timestamp": "2026-10-24T12:55:53.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.7758974615384615, "accuracy_std_dev": 0.022726804434125412, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.508684303076923, "avg_duration_per_task_ms": 124701.7123076923, "avg_output_tokens_per_task": 8041.3538461538465, "last_run_timestamp": "2026-10-24T12:26:01.097Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.7755556666666666, "accuracy_std_dev": 0.029313124352981113, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07678593777777779, "avg_duration_per_task_ms": 340037.8177777778, "avg_output_tokens_per_task": 17962.597777777777, "last_run_timestamp": "2026-10-27T03:02:28.656Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7755444170854271, "accuracy_std_dev": 0.02087028870637634, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.15095997738693467, "avg_duration_per_task_ms": 341068.3417085427, "avg_output_tokens_per_task": 20195.442211055277, "last_run_timestamp": "2026-10-24T13:15:20.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.7753625384615385, "accuracy_std_dev": 0.017323958686038136, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.06710363879598663, "avg_duration_per_task_ms": 1125092.9698996656, "avg_output_tokens_per_task": 36573.67558528428, "last_run_timestamp": "2026-10-24T13:26:35.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.7753334, "accuracy_std_dev": 0.019561963092696, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.06818830799999999, "avg_duration_per_task_ms": 45366.578, "avg_output_tokens_per_task": 4737.112, "last_run_timestamp": "2026-10-28T11:10:06.273Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.7753333, "accuracy_std_dev": 0.025307041621058755, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.48832906800000003, "avg_duration_per_task_ms": 125426.254, "avg_output_tokens_per_task": 7994.034, "last_run_timestamp": "2026-10-24T12:05:00.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.7746668, "accuracy_std_dev": 0.012927338819726197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.758212416, "avg_duration_per_task_ms": 112131.876, "avg_output_tokens_per_task": 6866.944, "last_run_timestamp": "2026-10-01T08:05:27.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7746666, "accuracy_std_dev": 0.02508632562652409, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.16812989199999998, "avg_duration_per_task_ms": 82939.64, "avg_output_tokens_per_task": 6304.564, "last_run_timestamp": "2026-10-24T11:50:50.507Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7745533125, "accuracy_std_dev": 0.016606913050345267, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.08764765178571429, "avg_duration_per_task_ms": 134355.6294642857, "avg_output_tokens_per_task": 4417.004464285715, "last_run_timestamp": "2026-10-24T12:55:13.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7745455454545455, "accuracy_std_dev": 0.01924194798002831, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4526700363636364, "avg_duration_per_task_ms": 121754.2, "avg_output_tokens_per_task": 7111.852727272727, "last_run_timestamp": "2026-10-24T11:43:26.286Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.7745454545454545, "accuracy_std_dev": 0.0297895320002411, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.5158095636363637, "avg_duration_per_task_ms": 126197.32727272727, "avg_output_tokens_per_task": 7983.286363636364, "last_run_timestamp": "2026-10-24T11:35:03.985Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7742857142857142, "accuracy_std_dev": 0.026945723560596418, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21623159142857146, "avg_duration_per_task_ms": 562306.5571428571, "avg_output_tokens_per_task": 12602.945714285714, "last_run_timestamp": "2026-10-10T11:06:43.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.7740741111111111, "accuracy_std_dev": 0.022098307604602994, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05655496222222222, "avg_duration_per_task_ms": 155063.69555555555, "avg_output_tokens_per_task": 16071.293333333333, "last_run_timestamp": "2026-10-24T11:33:51.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.7739393636363636, "accuracy_std_dev": 0.019582366837506575, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.43740748545454544, "avg_duration_per_task_ms": 94129.65818181819, "avg_output_tokens_per_task": 5651.554545454545, "last_run_timestamp": "2026-10-24T11:36:40.498Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.7738693316582915, "accuracy_std_dev": 0.030268145815516595, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.03491607663316582, "avg_duration_per_task_ms": 506838.68467336684, "avg_output_tokens_per_task": 12387.751256281406, "last_run_timestamp": "2026-10-24T19:21:21.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.7736362727272728, "accuracy_std_dev": 0.032503472161631956, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.44677050818181824, "avg_duration_per_task_ms": 79384.7109090909, "avg_output_tokens_per_task": 5652.894545454546, "last_run_timestamp": "2026-10-24T12:17:57.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.7733335, "accuracy_std_dev": 0.02708037419885479, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.13855224, "avg_duration_per_task_ms": 310185.72, "avg_output_tokens_per_task": 21233.035, "last_run_timestamp": "2026-10-24T12:39:23.138Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.7733335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11407636, "avg_duration_per_task_ms": 360681.75, "avg_output_tokens_per_task": 19997.86, "last_run_timestamp": "2026-10-01T22:40:52.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "AtlasCloud", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01881542, "avg_duration_per_task_ms": 96553.18, "avg_output_tokens_per_task": 11186.19, "last_run_timestamp": "2026-10-24T11:34:09.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.7733335, "accuracy_std_dev": 0.03126950946609175, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.068859715, "avg_duration_per_task_ms": 704699.005, "avg_output_tokens_per_task": 13178.825, "last_run_timestamp": "2026-10-24T12:26:26.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.7733334, "accuracy_std_dev": 0.013984054413509715, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.17623406799999997, "avg_duration_per_task_ms": 74817.192, "avg_output_tokens_per_task": 6393.728, "last_run_timestamp": "2026-10-24T11:36:43.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.7733332857142857, "accuracy_std_dev": 0.022537305934283946, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08668051142857143, "avg_duration_per_task_ms": 364495.4428571428, "avg_output_tokens_per_task": 20483.94, "last_run_timestamp": "2026-10-24T12:45:56.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012345539999999999, "avg_duration_per_task_ms": 657478.84, "avg_output_tokens_per_task": 14196.68, "last_run_timestamp": "2026-10-03T00:00:06.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.16341722, "avg_duration_per_task_ms": 174236.4, "avg_output_tokens_per_task": 10664.8, "last_run_timestamp": "2026-08-08T08:27:49.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DeepInfra", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0051899, "avg_duration_per_task_ms": 123477.22, "avg_output_tokens_per_task": 11624.46, "last_run_timestamp": "2026-10-20T20:17:11.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26629643999999997, "avg_duration_per_task_ms": 438741.58, "avg_output_tokens_per_task": 27876.64, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11485508, "avg_duration_per_task_ms": 381375.48, "avg_output_tokens_per_task": 14199.1, "last_run_timestamp": "2026-10-13T03:50:19.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.7727272727272727, "accuracy_std_dev": 0.028632777073235846, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1722844727272727, "avg_duration_per_task_ms": 588293.6218181818, "avg_output_tokens_per_task": 36404.07818181818, "last_run_timestamp": "2026-10-24T18:49:47.645Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7726668, "accuracy_std_dev": 0.0325508757356849, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.065897038, "avg_duration_per_task_ms": 245761.102, "avg_output_tokens_per_task": 11799.874, "last_run_timestamp": "2026-10-24T12:58:13.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.7723808571428572, "accuracy_std_dev": 0.03175867478536295, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0666134542857143, "avg_duration_per_task_ms": 68063.51142857142, "avg_output_tokens_per_task": 12368.354285714286, "last_run_timestamp": "2026-10-28T04:52:11.699Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.7723808571428572, "accuracy_std_dev": 0.03553283429420726, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0410159, "avg_duration_per_task_ms": 732886.1685714286, "avg_output_tokens_per_task": 12629.145714285714, "last_run_timestamp": "2026-10-24T12:41:08.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.7722223333333333, "accuracy_std_dev": 0.051949695516164705, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.2672586, "avg_duration_per_task_ms": 26529.913333333334, "avg_output_tokens_per_task": 12210.046666666667, "last_run_timestamp": "2026-10-24T12:34:49.317Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7722222222222223, "accuracy_std_dev": 0.03095202181433487, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03593436666666667, "avg_duration_per_task_ms": 446735.1666666667, "avg_output_tokens_per_task": 13815.786666666667, "last_run_timestamp": "2026-10-08T22:42:47.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.772, "accuracy_std_dev": 0.02679938639596063, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.808827636, "avg_duration_per_task_ms": 183544.5, "avg_output_tokens_per_task": 6940.7, "last_run_timestamp": "2026-10-01T08:03:00.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.771929874493927, "accuracy_std_dev": 0.030837182396580943, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.011004716599190284, "avg_duration_per_task_ms": 1072772.7165991904, "avg_output_tokens_per_task": 18798.943319838058, "last_run_timestamp": "2026-10-17T12:03:02.755Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.7718518888888889, "accuracy_std_dev": 0.02102986445480936, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05397537555555555, "avg_duration_per_task_ms": 110633.31777777777, "avg_output_tokens_per_task": 4708.497777777778, "last_run_timestamp": "2026-10-24T13:04:30.805Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.7714285714285715, "accuracy_std_dev": 0.02435571983191243, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21782835142857143, "avg_duration_per_task_ms": 377656.6857142857, "avg_output_tokens_per_task": 17802.317142857144, "last_run_timestamp": "2026-10-24T12:37:29.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.7714285714285715, "accuracy_std_dev": 0.023289574128825135, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.044844714285714286, "avg_duration_per_task_ms": 100224.04857142858, "avg_output_tokens_per_task": 4956.551428571429, "last_run_timestamp": "2026-10-24T12:38:46.757Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.7714284285714286, "accuracy_std_dev": 0.043456125344526336, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12002842571428574, "avg_duration_per_task_ms": 270068.88285714283, "avg_output_tokens_per_task": 11820.128571428571, "last_run_timestamp": "2026-10-17T21:03:47.447Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.7713333, "accuracy_std_dev": 0.0351567708046402, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.060528674000000005, "avg_duration_per_task_ms": 302911.6, "avg_output_tokens_per_task": 12614.202, "last_run_timestamp": "2026-10-24T12:18:13.146Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7712821538461538, "accuracy_std_dev": 0.029045614221682316, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.5353010338461538, "avg_duration_per_task_ms": 182572.49076923076, "avg_output_tokens_per_task": 10548.853846153846, "last_run_timestamp": "2026-10-24T12:18:32.884Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.771253766972477, "accuracy_std_dev": 0.0197324823544269, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.20062177064220185, "avg_duration_per_task_ms": 536809.0183486239, "avg_output_tokens_per_task": 28063.546788990825, "last_run_timestamp": "2026-10-24T19:14:04.604Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Alibaba", "accuracy": 0.7711113333333333, "accuracy_std_dev": 0.021999034771755084, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029255286666666668, "avg_duration_per_task_ms": 133490.13333333333, "avg_output_tokens_per_task": 12074.193333333333, "last_run_timestamp": "2026-10-28T03:37:31.074Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7711112499999999, "accuracy_std_dev": 0.014989728995354355, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.13635980666666667, "avg_duration_per_task_ms": 232657.155, "avg_output_tokens_per_task": 24827.668333333335, "last_run_timestamp": "2026-10-29T23:10:20.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.7709090909090909, "accuracy_std_dev": 0.02562719156909333, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.47683379272727267, "avg_duration_per_task_ms": 109957.77818181818, "avg_output_tokens_per_task": 7177.963636363636, "last_run_timestamp": "2026-10-24T11:37:27.338Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.7707407777777777, "accuracy_std_dev": 0.023188894510460925, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.11877632888888891, "avg_duration_per_task_ms": 486710.77777777775, "avg_output_tokens_per_task": 21345.64, "last_run_timestamp": "2026-10-17T11:37:24.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7707070606060606, "accuracy_std_dev": 0.02775262413138882, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.5647490236363636, "avg_duration_per_task_ms": 115196.03454545455, "avg_output_tokens_per_task": 8110.365454545455, "last_run_timestamp": "2026-10-24T12:27:09.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.7706666, "accuracy_std_dev": 0.0204828947622156, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10916058, "avg_duration_per_task_ms": 258128.42, "avg_output_tokens_per_task": 11359.708, "last_run_timestamp": "2026-10-03T16:07:05.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.770416625, "accuracy_std_dev": 0.0211762125800242, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.079187595, "avg_duration_per_task_ms": 95610.7625, "avg_output_tokens_per_task": 4413.6475, "last_run_timestamp": "2026-10-24T13:02:59.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.7703030909090909, "accuracy_std_dev": 0.021151618478604965, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5374423745454546, "avg_duration_per_task_ms": 175150.91636363635, "avg_output_tokens_per_task": 10708.305454545454, "last_run_timestamp": "2026-10-24T11:55:03.241Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.77, "accuracy_std_dev": 0.027688565631863254, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021299527500000002, "avg_duration_per_task_ms": 231325.375, "avg_output_tokens_per_task": 12153.2125, "last_run_timestamp": "2026-10-24T19:11:05.620Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.77, "accuracy_std_dev": 0.018855956915521412, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.065556992, "avg_duration_per_task_ms": 170454.924, "avg_output_tokens_per_task": 10274.532, "last_run_timestamp": "2026-10-24T19:27:30.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.77, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.35129818, "avg_duration_per_task_ms": 177012.35, "avg_output_tokens_per_task": 12679.64, "last_run_timestamp": "2026-10-01T19:06:01.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "StreamLake", "accuracy": 0.77, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05546486, "avg_duration_per_task_ms": 677432.76, "avg_output_tokens_per_task": 33432.56, "last_run_timestamp": "2026-10-24T13:08:41.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.77, "accuracy_std_dev": 0.022852121225391757, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.047772247500000004, "avg_duration_per_task_ms": 117312.3025, "avg_output_tokens_per_task": 4701.9325, "last_run_timestamp": "2026-10-24T12:40:31.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7699997500000001, "accuracy_std_dev": 0.029999861111136825, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08088121, "avg_duration_per_task_ms": 203815.155, "avg_output_tokens_per_task": 9386.985, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.7697777333333333, "accuracy_std_dev": 0.019147228922106605, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.12604373866666665, "avg_duration_per_task_ms": 93167.28, "avg_output_tokens_per_task": 4490.572, "last_run_timestamp": "2026-10-28T04:16:35.871Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.7696297777777777, "accuracy_std_dev": 0.028477632067516422, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1409867, "avg_duration_per_task_ms": 323727.4288888889, "avg_output_tokens_per_task": 20180.546666666665, "last_run_timestamp": "2026-10-24T12:56:30.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.7696296666666667, "accuracy_std_dev": 0.03639449000421171, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01613732888888889, "avg_duration_per_task_ms": 298156.97555555555, "avg_output_tokens_per_task": 11483.206666666667, "last_run_timestamp": "2026-10-24T19:22:55.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.7696296296296297, "accuracy_std_dev": 0.0190587770392855, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.06694535481481483, "avg_duration_per_task_ms": 71687.36444444445, "avg_output_tokens_per_task": 4656.992592592593, "last_run_timestamp": "2026-10-30T16:33:36.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.7695238571428571, "accuracy_std_dev": 0.029354336946199627, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12993965428571427, "avg_duration_per_task_ms": 178407.48, "avg_output_tokens_per_task": 8114.634285714285, "last_run_timestamp": "2026-10-17T12:47:59.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.7695237142857143, "accuracy_std_dev": 0.02386662296246901, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11461242285714283, "avg_duration_per_task_ms": 472929.0742857143, "avg_output_tokens_per_task": 11699.722857142857, "last_run_timestamp": "2026-10-23T07:26:58.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.7691498636363635, "accuracy_std_dev": 0.024155174859563606, "run_count": 16, "total_questions": 792, "avg_cost_per_task_usd": 0.05303868308080808, "avg_duration_per_task_ms": 267537.58207070705, "avg_output_tokens_per_task": 11036.6148989899, "last_run_timestamp": "2026-10-30T00:40:23.011Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.769144, "accuracy_std_dev": 0.05227625153611027, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.03692847297297297, "avg_duration_per_task_ms": 119447.77702702703, "avg_output_tokens_per_task": 4304.4121621621625, "last_run_timestamp": "2026-10-24T12:53:05.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.7688888888888888, "accuracy_std_dev": 0.02914421358022683, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08110119333333334, "avg_duration_per_task_ms": 48308.12, "avg_output_tokens_per_task": 4858.82, "last_run_timestamp": "2026-10-24T12:49:25.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.7688235294117647, "accuracy_std_dev": 0.026394292226378947, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.04698485764705882, "avg_duration_per_task_ms": 91736.21058823529, "avg_output_tokens_per_task": 4343.287058823529, "last_run_timestamp": "2026-10-24T12:48:21.011Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.768333375, "accuracy_std_dev": 0.032787198976649036, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05900729999999999, "avg_duration_per_task_ms": 114247.045, "avg_output_tokens_per_task": 6145.8525, "last_run_timestamp": "2026-10-24T12:32:52.472Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.76833325, "accuracy_std_dev": 0.023746338900712666, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05045579, "avg_duration_per_task_ms": 229267.9725, "avg_output_tokens_per_task": 12600.1625, "last_run_timestamp": "2026-10-24T19:41:33.487Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.768080787878788, "accuracy_std_dev": 0.02061397303207526, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.5330831266666668, "avg_duration_per_task_ms": 119035.52545454545, "avg_output_tokens_per_task": 8065.8436363636365, "last_run_timestamp": "2026-10-24T12:05:52.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.7680176486486487, "accuracy_std_dev": 0.018065309426264085, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.06668277702702702, "avg_duration_per_task_ms": 92879.05405405405, "avg_output_tokens_per_task": 4281.108108108108, "last_run_timestamp": "2026-10-24T13:05:47.020Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.768, "accuracy_std_dev": 0.028875333573830795, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.15511289399999997, "avg_duration_per_task_ms": 603929.03, "avg_output_tokens_per_task": 12401.186, "last_run_timestamp": "2026-10-24T12:59:12.979Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.768, "accuracy_std_dev": 0.011469922022402766, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030947572, "avg_duration_per_task_ms": 58222.324, "avg_output_tokens_per_task": 4803.824, "last_run_timestamp": "2026-10-24T12:19:36.645Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.7679998, "accuracy_std_dev": 0.04721585044622197, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07315105200000001, "avg_duration_per_task_ms": 1015313.192, "avg_output_tokens_per_task": 46954.436, "last_run_timestamp": "2026-10-24T13:12:06.662Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.767879, "accuracy_std_dev": 0.0204628104530234, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.49366084363636353, "avg_duration_per_task_ms": 125790.01454545454, "avg_output_tokens_per_task": 8065.552727272728, "last_run_timestamp": "2026-10-24T11:39:26.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.767500125, "accuracy_std_dev": 0.01839017891727471, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05273846000000001, "avg_duration_per_task_ms": 114111.54, "avg_output_tokens_per_task": 4459.3875, "last_run_timestamp": "2026-10-24T13:02:56.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.7675, "accuracy_std_dev": 0.018085537502380192, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1069855275, "avg_duration_per_task_ms": 249511.1875, "avg_output_tokens_per_task": 18702.1275, "last_run_timestamp": "2026-10-24T19:31:34.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.7675, "accuracy_std_dev": 0.019561171859068173, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0578507225, "avg_duration_per_task_ms": 50974.42, "avg_output_tokens_per_task": 4677.1725, "last_run_timestamp": "2026-10-24T12:36:03.010Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.7672728181818181, "accuracy_std_dev": 0.023690757149795346, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.41836188, "avg_duration_per_task_ms": 93044.15818181819, "avg_output_tokens_per_task": 5611.463636363636, "last_run_timestamp": "2026-10-24T12:23:35.338Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7666669999999999, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2750355, "avg_duration_per_task_ms": 98712.99, "avg_output_tokens_per_task": 5708.24, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "ModelRun", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01300872, "avg_duration_per_task_ms": 90405.02, "avg_output_tokens_per_task": 13808.24, "last_run_timestamp": "2026-10-29T03:35:28.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Modal", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.30783086, "avg_duration_per_task_ms": 74384.36, "avg_output_tokens_per_task": 8511.9, "last_run_timestamp": "2026-10-24T11:55:43.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "AtlasCloud", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06503968, "avg_duration_per_task_ms": 307621.28, "avg_output_tokens_per_task": 13420.2, "last_run_timestamp": "2026-10-27T04:07:26.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0912162, "avg_duration_per_task_ms": 191018.42, "avg_output_tokens_per_task": 9260.76, "last_run_timestamp": "2026-08-04T03:12:10.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7666668, "accuracy_std_dev": 0.007302967435227963, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.220517392, "avg_duration_per_task_ms": 158612.068, "avg_output_tokens_per_task": 23228.836, "last_run_timestamp": "2026-10-01T19:04:58.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.76666675, "accuracy_std_dev": 0.024267078752653787, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.036940722499999995, "avg_duration_per_task_ms": 209470.17, "avg_output_tokens_per_task": 17358.2575, "last_run_timestamp": "2026-10-03T08:39:48.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.76666675, "accuracy_std_dev": 0.0141420177728463, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01053462, "avg_duration_per_task_ms": 598882.16, "avg_output_tokens_per_task": 21582.78, "last_run_timestamp": "2026-10-27T03:23:59.523Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08312547333333332, "avg_duration_per_task_ms": 156615.94, "avg_output_tokens_per_task": 8699.633333333333, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.766666625, "accuracy_std_dev": 0.017320700526663898, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.029551312500000003, "avg_duration_per_task_ms": 128791.4, "avg_output_tokens_per_task": 12071.38, "last_run_timestamp": "2026-10-28T10:41:11.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.7666666, "accuracy_std_dev": 0.03169302061400902, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.435956584, "avg_duration_per_task_ms": 91965.526, "avg_output_tokens_per_task": 5586.83, "last_run_timestamp": "2026-10-24T11:44:03.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.7666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.27949224, "avg_duration_per_task_ms": 363191.33, "avg_output_tokens_per_task": 35460.86, "last_run_timestamp": "2026-10-15T21:31:19.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "ParetoAlpha", "accuracy": 0.7666665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29799199, "avg_duration_per_task_ms": 1675038.6, "avg_output_tokens_per_task": 11020.82, "last_run_timestamp": "2026-10-30T07:12:44.253Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.7666665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01352679, "avg_duration_per_task_ms": 90848.6, "avg_output_tokens_per_task": 4507.38, "last_run_timestamp": "2026-10-10T11:18:50.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7658335, "accuracy_std_dev": 0.028369961618761493, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1399948425, "avg_duration_per_task_ms": 291221.6025, "avg_output_tokens_per_task": 11931.6425, "last_run_timestamp": "2026-10-24T19:21:41.094Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.7658335, "accuracy_std_dev": 0.029095183832552104, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.11472753, "avg_duration_per_task_ms": 363943.795, "avg_output_tokens_per_task": 19034.395, "last_run_timestamp": "2026-10-24T12:25:43.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.76583325, "accuracy_std_dev": 0.029142835503559023, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.10458012499999998, "avg_duration_per_task_ms": 230385.535, "avg_output_tokens_per_task": 11203.13, "last_run_timestamp": "2026-10-24T19:49:41.079Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7657141428571429, "accuracy_std_dev": 0.02265784142050462, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01868964, "avg_duration_per_task_ms": 296461.9485714286, "avg_output_tokens_per_task": 14208.894285714287, "last_run_timestamp": "2026-10-24T13:09:04.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.7655824146341464, "accuracy_std_dev": 0.030599644599243303, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.07519595528455283, "avg_duration_per_task_ms": 226711.57317073172, "avg_output_tokens_per_task": 10093.731707317073, "last_run_timestamp": "2026-10-10T19:17:08.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7655204498567336, "accuracy_std_dev": 0.018269240228608635, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.012520114613180514, "avg_duration_per_task_ms": 376237.81088825216, "avg_output_tokens_per_task": 17341.69340974212, "last_run_timestamp": "2026-10-27T15:46:29.407Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.7654902352941176, "accuracy_std_dev": 0.022548186516434778, "run_count": 34, "total_questions": 1700, "avg_cost_per_task_usd": 0.5963552552941177, "avg_duration_per_task_ms": 71890.29588235295, "avg_output_tokens_per_task": 5710.195294117647, "last_run_timestamp": "2026-10-24T12:22:02.797Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7654544545454546, "accuracy_std_dev": 0.0392678247038871, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.09729278181818182, "avg_duration_per_task_ms": 320080.3927272727, "avg_output_tokens_per_task": 16582.44, "last_run_timestamp": "2026-10-24T19:50:25.118Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.7652466315789472, "accuracy_std_dev": 0.0204935205903598, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.010908814536340851, "avg_duration_per_task_ms": 294625.86215538846, "avg_output_tokens_per_task": 12715.822055137845, "last_run_timestamp": "2026-10-24T19:59:46.522Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.7651851111111111, "accuracy_std_dev": 0.030107298743897986, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.009121904444444445, "avg_duration_per_task_ms": 193621.67333333334, "avg_output_tokens_per_task": 4487.197777777777, "last_run_timestamp": "2026-10-24T13:10:16.181Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.7651515252525253, "accuracy_std_dev": 0.02642279772753065, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.07224725252525253, "avg_duration_per_task_ms": 56760.18939393939, "avg_output_tokens_per_task": 4516.429292929293, "last_run_timestamp": "2026-10-24T12:38:39.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.765, "accuracy_std_dev": 0.03840572873934304, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08522504, "avg_duration_per_task_ms": 558480.23, "avg_output_tokens_per_task": 11022.365, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.765, "accuracy_std_dev": 0.02682235800596212, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.093536965, "avg_duration_per_task_ms": 395836.625, "avg_output_tokens_per_task": 14656.925, "last_run_timestamp": "2026-10-24T12:32:30.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7648483636363637, "accuracy_std_dev": 0.02528071162430767, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.021531063636363635, "avg_duration_per_task_ms": 268995.7272727273, "avg_output_tokens_per_task": 15699.76909090909, "last_run_timestamp": "2026-10-24T12:30:46.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.7645742242152466, "accuracy_std_dev": 0.03220926464716497, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.05124673542600897, "avg_duration_per_task_ms": 396898.4529147982, "avg_output_tokens_per_task": 12964.300448430493, "last_run_timestamp": "2026-10-24T12:42:05.792Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7645455, "accuracy_std_dev": 0.032559987304205146, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.5562211936363636, "avg_duration_per_task_ms": 182577.9390909091, "avg_output_tokens_per_task": 10629.811818181819, "last_run_timestamp": "2026-10-24T12:21:43.024Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "GMICloud", "accuracy": 0.7644446666666667, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01656822, "avg_duration_per_task_ms": 126283.51333333334, "avg_output_tokens_per_task": 12553.306666666667, "last_run_timestamp": "2026-10-27T00:46:05.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.7644445555555556, "accuracy_std_dev": 0.027216570843314122, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04539358888888889, "avg_duration_per_task_ms": 189556.9111111111, "avg_output_tokens_per_task": 9729.92888888889, "last_run_timestamp": "2026-10-24T19:32:27.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.7644443333333334, "accuracy_std_dev": 0.04565023765059233, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01549936, "avg_duration_per_task_ms": 297786.8, "avg_output_tokens_per_task": 16565.086666666666, "last_run_timestamp": "2026-10-28T04:10:41.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.7641668749999999, "accuracy_std_dev": 0.02623356496188377, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.334670415, "avg_duration_per_task_ms": 656088.585, "avg_output_tokens_per_task": 28108.1825, "last_run_timestamp": "2026-10-24T18:52:57.046Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.7636566923076923, "accuracy_std_dev": 0.016849090853317095, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.028887086956521736, "avg_duration_per_task_ms": 106220.35785953177, "avg_output_tokens_per_task": 18019.585284280936, "last_run_timestamp": "2026-10-24T11:41:59.518Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7636092192393736, "accuracy_std_dev": 0.031035672419704556, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.11008267114093959, "avg_duration_per_task_ms": 162725.5794183445, "avg_output_tokens_per_task": 9299.664429530201, "last_run_timestamp": "2026-10-28T04:07:24.080Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.7633820364963503, "accuracy_std_dev": 0.022249781810054704, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.09270584489051097, "avg_duration_per_task_ms": 50291.538321167885, "avg_output_tokens_per_task": 4353.293795620438, "last_run_timestamp": "2026-10-29T23:26:26.346Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.7633335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.021769780000000002, "avg_duration_per_task_ms": 1044368.78, "avg_output_tokens_per_task": 12563.69, "last_run_timestamp": "2026-08-15T08:15:52.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.7633334166666667, "accuracy_std_dev": 0.02501853017084981, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.021799716666666667, "avg_duration_per_task_ms": 153361.12, "avg_output_tokens_per_task": 16488.393333333333, "last_run_timestamp": "2026-10-27T00:49:28.181Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7633333333333333, "accuracy_std_dev": 0.017105316480231798, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.15788212333333335, "avg_duration_per_task_ms": 383915.8333333333, "avg_output_tokens_per_task": 18570.623333333333, "last_run_timestamp": "2026-10-24T20:11:33.520Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.7633332777777778, "accuracy_std_dev": 0.01808766376046748, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.010060852222222218, "avg_duration_per_task_ms": 327666.52, "avg_output_tokens_per_task": 14799.732222222223, "last_run_timestamp": "2026-10-24T13:02:06.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.03605539719220273, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.157135485, "avg_duration_per_task_ms": 411704.8, "avg_output_tokens_per_task": 26150.13, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.013743584364622665, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01189864, "avg_duration_per_task_ms": 642869.1, "avg_output_tokens_per_task": 23140.98, "last_run_timestamp": "2026-10-24T13:28:08.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.762963, "accuracy_std_dev": 0.025745351040959277, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08904649555555556, "avg_duration_per_task_ms": 300656.5133333333, "avg_output_tokens_per_task": 12058.504444444445, "last_run_timestamp": "2026-10-24T12:17:31.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7629628888888889, "accuracy_std_dev": 0.02354127566374736, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01074868, "avg_duration_per_task_ms": 1059485.7177777777, "avg_output_tokens_per_task": 13128.448888888888, "last_run_timestamp": "2026-10-24T19:49:45.575Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.7629628888888889, "accuracy_std_dev": 0.031128845034949126, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.008403426666666667, "avg_duration_per_task_ms": 205005.57333333333, "avg_output_tokens_per_task": 4237.573333333334, "last_run_timestamp": "2026-10-24T13:15:23.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.7629022732240437, "accuracy_std_dev": 0.028478290118661355, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.01388206375227687, "avg_duration_per_task_ms": 226746.868852459, "avg_output_tokens_per_task": 15915.701275045538, "last_run_timestamp": "2026-10-17T11:00:48.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7628571428571429, "accuracy_std_dev": 0.023328310442945685, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11986269428571429, "avg_duration_per_task_ms": 415507.14, "avg_output_tokens_per_task": 18962.734285714287, "last_run_timestamp": "2026-10-03T08:33:31.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.7627778333333333, "accuracy_std_dev": 0.03313374040368652, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07174823666666666, "avg_duration_per_task_ms": 893696.3066666666, "avg_output_tokens_per_task": 51834.78666666667, "last_run_timestamp": "2026-10-27T03:58:39.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.7625640769230769, "accuracy_std_dev": 0.027985121905594868, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.5249607092307692, "avg_duration_per_task_ms": 215709.9046153846, "avg_output_tokens_per_task": 9987.407692307692, "last_run_timestamp": "2026-10-24T12:29:32.505Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.7625627587939698, "accuracy_std_dev": 0.017640643984489356, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.01927761809045226, "avg_duration_per_task_ms": 121120.84924623116, "avg_output_tokens_per_task": 14540.939698492462, "last_run_timestamp": "2026-10-24T11:49:30.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.762500125, "accuracy_std_dev": 0.06959710991384181, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.25472334, "avg_duration_per_task_ms": 482869.145, "avg_output_tokens_per_task": 29497.4675, "last_run_timestamp": "2026-10-24T19:02:15.249Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7624327096774194, "accuracy_std_dev": 0.039878336715063745, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.09450495362903226, "avg_duration_per_task_ms": 748971.7096774194, "avg_output_tokens_per_task": 10301.362903225807, "last_run_timestamp": "2026-10-24T19:52:35.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.030469494967188978, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029655940000000006, "avg_duration_per_task_ms": 256385.88, "avg_output_tokens_per_task": 18856.086666666666, "last_run_timestamp": "2026-10-08T22:27:05.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.762143986599665, "accuracy_std_dev": 0.04074106053036531, "run_count": 12, "total_questions": 597, "avg_cost_per_task_usd": 0.06808563986599665, "avg_duration_per_task_ms": 207102.986599665, "avg_output_tokens_per_task": 11782.765494137353, "last_run_timestamp": "2026-10-24T12:36:04.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.762, "accuracy_std_dev": 0.02386079071196091, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.5457165, "avg_duration_per_task_ms": 180407.204, "avg_output_tokens_per_task": 10745.142, "last_run_timestamp": "2026-10-24T11:51:51.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.7618181818181818, "accuracy_std_dev": 0.03607974434957347, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.14144602545454546, "avg_duration_per_task_ms": 856323.6963636364, "avg_output_tokens_per_task": 31885.77090909091, "last_run_timestamp": "2026-10-24T19:43:40.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.7617847878787878, "accuracy_std_dev": 0.03876959831491683, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.02413183333333333, "avg_duration_per_task_ms": 113963.81313131313, "avg_output_tokens_per_task": 15997.81313131313, "last_run_timestamp": "2026-10-24T12:54:33.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.7617776666666667, "accuracy_std_dev": 0.02671855322596808, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.011852372, "avg_duration_per_task_ms": 180670.42666666667, "avg_output_tokens_per_task": 5590.144, "last_run_timestamp": "2026-10-29T22:50:44.363Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7617447986577182, "accuracy_std_dev": 0.00897518183195566, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.007206587248322147, "avg_duration_per_task_ms": 288157.1812080537, "avg_output_tokens_per_task": 12110.157718120805, "last_run_timestamp": "2026-10-24T20:14:29.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.76166675, "accuracy_std_dev": 0.02229839929653024, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.065100265, "avg_duration_per_task_ms": 116150.4225, "avg_output_tokens_per_task": 4604.0675, "last_run_timestamp": "2026-10-24T12:22:07.097Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.7616666428571429, "accuracy_std_dev": 0.025346699169734525, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.09051784142857143, "avg_duration_per_task_ms": 191721.94857142857, "avg_output_tokens_per_task": 11065.187142857143, "last_run_timestamp": "2026-10-28T04:31:19.438Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.761666625, "accuracy_std_dev": 0.015545671961493828, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08149474, "avg_duration_per_task_ms": 103926, "avg_output_tokens_per_task": 22421.02, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.7615039124579126, "accuracy_std_dev": 0.01612860952882591, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.1581539494949495, "avg_duration_per_task_ms": 807649.4747474748, "avg_output_tokens_per_task": 30684.649831649833, "last_run_timestamp": "2026-10-24T18:54:54.259Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.7612120909090909, "accuracy_std_dev": 0.030986186680386097, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5405228218181818, "avg_duration_per_task_ms": 191201.12545454546, "avg_output_tokens_per_task": 10721.865454545454, "last_run_timestamp": "2026-10-24T12:29:33.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.761111, "accuracy_std_dev": 0.016063062451890448, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05401205, "avg_duration_per_task_ms": 73585.80666666667, "avg_output_tokens_per_task": 4477.26, "last_run_timestamp": "2026-10-24T12:38:41.606Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.7610546224489795, "accuracy_std_dev": 0.016475935184003962, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.2764416326530612, "avg_duration_per_task_ms": 159672.02551020408, "avg_output_tokens_per_task": 7623.877551020408, "last_run_timestamp": "2026-10-24T12:38:26.300Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.7609524285714285, "accuracy_std_dev": 0.025053305529593985, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06623187714285714, "avg_duration_per_task_ms": 1264087.4542857143, "avg_output_tokens_per_task": 64691.61714285714, "last_run_timestamp": "2026-10-30T02:58:57.544Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7609091818181818, "accuracy_std_dev": 0.03684790544884393, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.19680807272727271, "avg_duration_per_task_ms": 478180.9890909091, "avg_output_tokens_per_task": 27737.445454545454, "last_run_timestamp": "2026-10-28T04:04:29.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.760833375, "accuracy_std_dev": 0.017460477198071506, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.17206684999999997, "avg_duration_per_task_ms": 195271.6725, "avg_output_tokens_per_task": 11186.875, "last_run_timestamp": "2026-10-24T19:43:42.996Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.76083325, "accuracy_std_dev": 0.022897649686976593, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.018742625, "avg_duration_per_task_ms": 681240.3, "avg_output_tokens_per_task": 12523.52, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.7607406666666667, "accuracy_std_dev": 0.034633054297881376, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.006436806666666667, "avg_duration_per_task_ms": 125547.04444444444, "avg_output_tokens_per_task": 4262.8533333333335, "last_run_timestamp": "2026-10-24T14:25:23.734Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.7606666, "accuracy_std_dev": 0.031048465341140474, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.027007422000000003, "avg_duration_per_task_ms": 150781.166, "avg_output_tokens_per_task": 8984.718, "last_run_timestamp": "2026-10-24T19:16:17.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.7606059090909091, "accuracy_std_dev": 0.02402931176124294, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03610860181818182, "avg_duration_per_task_ms": 72225.87454545454, "avg_output_tokens_per_task": 4745.927272727273, "last_run_timestamp": "2026-10-24T12:22:54.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.760476, "accuracy_std_dev": 0.023999188385086207, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21848949428571432, "avg_duration_per_task_ms": 711094.3942857143, "avg_output_tokens_per_task": 57307.56, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.7603748473895582, "accuracy_std_dev": 0.02026019621425223, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.03634712449799197, "avg_duration_per_task_ms": 152002.0281124498, "avg_output_tokens_per_task": 23794.738955823294, "last_run_timestamp": "2026-10-24T12:53:48.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.7603120367892977, "accuracy_std_dev": 0.027860534151775026, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.06456417725752508, "avg_duration_per_task_ms": 376665.0802675585, "avg_output_tokens_per_task": 52393.28428093645, "last_run_timestamp": "2026-10-24T19:21:23.089Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7600000909090909, "accuracy_std_dev": 0.023441265523758994, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03510028181818181, "avg_duration_per_task_ms": 167423.50181818183, "avg_output_tokens_per_task": 16452.421818181818, "last_run_timestamp": "2026-10-24T12:19:39.208Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.76, "accuracy_std_dev": 0.02160235326879922, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.027435835, "avg_duration_per_task_ms": 200904.6775, "avg_output_tokens_per_task": 13494.075, "last_run_timestamp": "2026-10-24T20:44:23.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.054461335, "avg_duration_per_task_ms": 91417, "avg_output_tokens_per_task": 15101.33, "last_run_timestamp": "2026-10-03T08:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.76, "accuracy_std_dev": 0.014401594564491825, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.019761673333333334, "avg_duration_per_task_ms": 291193.25333333336, "avg_output_tokens_per_task": 17811.393333333333, "last_run_timestamp": "2026-10-10T11:23:23.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01597332, "avg_duration_per_task_ms": 42256.58, "avg_output_tokens_per_task": 1764.66, "last_run_timestamp": "2026-10-01T23:50:45.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.3443486, "avg_duration_per_task_ms": 184596.7, "avg_output_tokens_per_task": 12502.76, "last_run_timestamp": "2026-10-01T19:04:40.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.76, "accuracy_std_dev": 0.02494442712310708, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.022151744999999997, "avg_duration_per_task_ms": 485324.6075, "avg_output_tokens_per_task": 14080.3675, "last_run_timestamp": "2026-10-24T18:47:38.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Venice", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19103682, "avg_duration_per_task_ms": 233230.24, "avg_output_tokens_per_task": 15236.66, "last_run_timestamp": "2026-08-04T03:12:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.76, "accuracy_std_dev": 0.005443582705045132, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.40005456666666667, "avg_duration_per_task_ms": 394539.76666666666, "avg_output_tokens_per_task": 34690.113333333335, "last_run_timestamp": "2026-10-02T23:12:57.355Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.76, "accuracy_std_dev": 0.02108195647941624, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08341662, "avg_duration_per_task_ms": 533566.115, "avg_output_tokens_per_task": 14923.395, "last_run_timestamp": "2026-10-24T12:36:09.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Cohere", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10151575999999998, "avg_duration_per_task_ms": 434936.03, "avg_output_tokens_per_task": 13962.46, "last_run_timestamp": "2026-10-24T12:32:07.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.7599998888888889, "accuracy_std_dev": 0.03899989237148808, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.009081111111111111, "avg_duration_per_task_ms": 187614.08, "avg_output_tokens_per_task": 15582.557777777778, "last_run_timestamp": "2026-10-27T01:09:17.824Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.7595238571428571, "accuracy_std_dev": 0.02174373003832582, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.04527854285714286, "avg_duration_per_task_ms": 136160.14428571428, "avg_output_tokens_per_task": 4500.364285714286, "last_run_timestamp": "2026-10-24T12:17:41.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.7594445, "accuracy_std_dev": 0.03225570769776, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.01983395166666667, "avg_duration_per_task_ms": 192524.21833333332, "avg_output_tokens_per_task": 14719.435, "last_run_timestamp": "2026-10-24T12:16:56.429Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.759408620967742, "accuracy_std_dev": 0.023253272624729635, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.012160826612903226, "avg_duration_per_task_ms": 309190.9919354839, "avg_output_tokens_per_task": 4125.725806451613, "last_run_timestamp": "2026-10-17T11:47:18.976Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7593939090909091, "accuracy_std_dev": 0.017397280855741624, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.031511507272727265, "avg_duration_per_task_ms": 229078.9090909091, "avg_output_tokens_per_task": 15370.054545454546, "last_run_timestamp": "2026-10-24T13:28:03.080Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.7592594444444444, "accuracy_std_dev": 0.017340275597650393, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.15033249555555556, "avg_duration_per_task_ms": 165755.92888888888, "avg_output_tokens_per_task": 20125.08, "last_run_timestamp": "2026-10-24T11:39:34.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7592592436412315, "accuracy_std_dev": 0.02502658689065603, "run_count": 15, "total_questions": 747, "avg_cost_per_task_usd": 0.05607525167336011, "avg_duration_per_task_ms": 275516.45113788487, "avg_output_tokens_per_task": 12015.718875502009, "last_run_timestamp": "2026-10-24T11:39:58.119Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.759166625, "accuracy_std_dev": 0.02777639760631993, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0257473675, "avg_duration_per_task_ms": 253688.2675, "avg_output_tokens_per_task": 13099.1275, "last_run_timestamp": "2026-10-24T19:09:33.707Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.7591665000000001, "accuracy_std_dev": 0.030264945803685155, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.03163724750000001, "avg_duration_per_task_ms": 184467.965, "avg_output_tokens_per_task": 14426.225, "last_run_timestamp": "2026-10-28T03:36:02.600Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7591665000000001, "accuracy_std_dev": 0.003632454025861855, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.09515249499999999, "avg_duration_per_task_ms": 195938.94, "avg_output_tokens_per_task": 9656.23, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Wafer", "accuracy": 0.7588889999999999, "accuracy_std_dev": 0.019019067756683928, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.009268686666666666, "avg_duration_per_task_ms": 94260.51333333334, "avg_output_tokens_per_task": 13026.133333333333, "last_run_timestamp": "2026-10-29T22:30:49.139Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7588886666666668, "accuracy_std_dev": 0.011811189506377245, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.010121153333333334, "avg_duration_per_task_ms": 244060.09, "avg_output_tokens_per_task": 16283.283333333333, "last_run_timestamp": "2026-10-24T13:01:16.778Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.7588075528455285, "accuracy_std_dev": 0.013043211721044773, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.0340294837398374, "avg_duration_per_task_ms": 524064.9918699187, "avg_output_tokens_per_task": 12340.776422764227, "last_run_timestamp": "2026-10-30T01:17:31.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.7584615384615384, "accuracy_std_dev": 0.02110672856848985, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.021854490769230767, "avg_duration_per_task_ms": 124647.87076923077, "avg_output_tokens_per_task": 17096.62, "last_run_timestamp": "2026-10-24T12:20:00.386Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.7583335, "accuracy_std_dev": 0.034760921546328406, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12354291499999999, "avg_duration_per_task_ms": 1021866.15, "avg_output_tokens_per_task": 26151.615, "last_run_timestamp": "2026-10-24T12:47:02.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.758333375, "accuracy_std_dev": 0.012360224066916245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.034803532500000005, "avg_duration_per_task_ms": 275650.13, "avg_output_tokens_per_task": 13717.48, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.758333375, "accuracy_std_dev": 0.03329163433558611, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.044925605, "avg_duration_per_task_ms": 183156.9125, "avg_output_tokens_per_task": 9429.4925, "last_run_timestamp": "2026-10-24T19:48:57.855Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.014433785597946933, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.42515429, "avg_duration_per_task_ms": 460450.54, "avg_output_tokens_per_task": 34342.89, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.03460084697211181, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0886916525, "avg_duration_per_task_ms": 581833.97, "avg_output_tokens_per_task": 12683.3375, "last_run_timestamp": "2026-10-24T12:19:32.894Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.029767707149485012, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1867523675, "avg_duration_per_task_ms": 181894.3925, "avg_output_tokens_per_task": 9712.265, "last_run_timestamp": "2026-10-24T19:21:02.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.7581818181818182, "accuracy_std_dev": 0.034942042529720004, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.0916818309090909, "avg_duration_per_task_ms": 244976.56, "avg_output_tokens_per_task": 12409.656363636364, "last_run_timestamp": "2026-10-24T12:33:35.006Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.7579907534246576, "accuracy_std_dev": 0.01133630408123485, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.06852380136986301, "avg_duration_per_task_ms": 156936.7876712329, "avg_output_tokens_per_task": 10408.410958904109, "last_run_timestamp": "2026-10-24T18:33:39.875Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.7579486923076924, "accuracy_std_dev": 0.021623878857143158, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.0068238523076923074, "avg_duration_per_task_ms": 261267.2046153846, "avg_output_tokens_per_task": 15563.432307692308, "last_run_timestamp": "2026-10-24T13:09:56.475Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.757778, "accuracy_std_dev": 0.025724014817546128, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03362461666666666, "avg_duration_per_task_ms": 226709.77666666667, "avg_output_tokens_per_task": 35231.33666666667, "last_run_timestamp": "2026-10-24T11:55:02.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.7575758811188811, "accuracy_std_dev": 0.010732768960317529, "run_count": 3, "total_questions": 143, "avg_cost_per_task_usd": 0.27724311888111886, "avg_duration_per_task_ms": 338699.5594405594, "avg_output_tokens_per_task": 6540.4965034965035, "last_run_timestamp": "2026-10-24T11:53:36.261Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.7575504530201342, "accuracy_std_dev": 0.028065656837905205, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.19407245637583895, "avg_duration_per_task_ms": 482897.4211409396, "avg_output_tokens_per_task": 20396.08389261745, "last_run_timestamp": "2026-10-24T14:01:08.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7574072222222222, "accuracy_std_dev": 0.030298005023351983, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.015441244444444443, "avg_duration_per_task_ms": 72124.47777777778, "avg_output_tokens_per_task": 4310.673333333333, "last_run_timestamp": "2026-10-17T11:14:15.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.032000033334982636, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.028732296, "avg_duration_per_task_ms": 171140.192, "avg_output_tokens_per_task": 15777.572, "last_run_timestamp": "2026-10-24T12:23:36.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.03991088294989224, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023026752, "avg_duration_per_task_ms": 161149.652, "avg_output_tokens_per_task": 18473.992, "last_run_timestamp": "2026-10-03T08:55:31.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7571428571428571, "accuracy_std_dev": 0.024651550802730272, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.012830634285714286, "avg_duration_per_task_ms": 221282.58285714284, "avg_output_tokens_per_task": 4997.4742857142855, "last_run_timestamp": "2026-10-24T12:58:11.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.7568475720930232, "accuracy_std_dev": 0.02459803490467476, "run_count": 13, "total_questions": 645, "avg_cost_per_task_usd": 0.09061050697674419, "avg_duration_per_task_ms": 702536.334883721, "avg_output_tokens_per_task": 21508.96434108527, "last_run_timestamp": "2026-10-24T12:57:12.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.7567265784753363, "accuracy_std_dev": 0.024115065606584538, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.015529553811659193, "avg_duration_per_task_ms": 194340.35874439462, "avg_output_tokens_per_task": 5359.600896860987, "last_run_timestamp": "2026-10-30T14:59:37.060Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "SiliconFlow", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.023368849999999997, "avg_duration_per_task_ms": 302412.09, "avg_output_tokens_per_task": 11606.21, "last_run_timestamp": "2026-10-24T12:08:01.600Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7566666666666667, "accuracy_std_dev": 0.04895203245581898, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02191566, "avg_duration_per_task_ms": 332956.19333333336, "avg_output_tokens_per_task": 12348.723333333333, "last_run_timestamp": "2026-10-24T12:55:32.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.7566666, "accuracy_std_dev": 0.015846499791436604, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.175855144, "avg_duration_per_task_ms": 317393.27, "avg_output_tokens_per_task": 10799.458, "last_run_timestamp": "2026-10-28T03:37:05.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.7566665, "accuracy_std_dev": 0.028087175801422264, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0129076525, "avg_duration_per_task_ms": 225184.265, "avg_output_tokens_per_task": 14188.9, "last_run_timestamp": "2026-10-24T19:54:36.988Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.7566665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018845109999999998, "avg_duration_per_task_ms": 92258.91, "avg_output_tokens_per_task": 11470.03, "last_run_timestamp": "2026-10-24T12:23:52.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.7564972372881356, "accuracy_std_dev": 0.0180813407986352, "run_count": 12, "total_questions": 590, "avg_cost_per_task_usd": 0.032619, "avg_duration_per_task_ms": 164901.18983050846, "avg_output_tokens_per_task": 16019.486440677965, "last_run_timestamp": "2026-10-28T03:43:00.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.7563636363636363, "accuracy_std_dev": 0.028620044734329858, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1009524109090909, "avg_duration_per_task_ms": 283080.54363636364, "avg_output_tokens_per_task": 12195.158181818182, "last_run_timestamp": "2026-10-28T03:44:06.462Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7563351081871345, "accuracy_std_dev": 0.013338594901982123, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.033182684210526316, "avg_duration_per_task_ms": 2146298.769005848, "avg_output_tokens_per_task": 50213.23391812865, "last_run_timestamp": "2026-10-09T04:05:56.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.756, "accuracy_std_dev": 0.04249977777824259, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.03474274, "avg_duration_per_task_ms": 260320.88, "avg_output_tokens_per_task": 13874.396, "last_run_timestamp": "2026-10-29T23:09:03.074Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.756, "accuracy_std_dev": 0.024073867807230344, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07011482399999999, "avg_duration_per_task_ms": 334056.692, "avg_output_tokens_per_task": 14633.816, "last_run_timestamp": "2026-10-24T12:59:11.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.756, "accuracy_std_dev": 0.022151057162131126, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.064562922, "avg_duration_per_task_ms": 168412.394, "avg_output_tokens_per_task": 4657.626, "last_run_timestamp": "2026-10-27T01:58:28.298Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.7559648947368421, "accuracy_std_dev": 0.020191449941144954, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.29131978947368414, "avg_duration_per_task_ms": 264712.04, "avg_output_tokens_per_task": 11238.59157894737, "last_run_timestamp": "2026-10-24T12:24:12.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Modal", "accuracy": 0.755892303030303, "accuracy_std_dev": 0.022789500000000018, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.027549767676767676, "avg_duration_per_task_ms": 104945.30303030302, "avg_output_tokens_per_task": 10081.373737373737, "last_run_timestamp": "2026-10-24T12:19:36.298Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.75583325, "accuracy_std_dev": 0.019982613972839007, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.008674565, "avg_duration_per_task_ms": 119249.875, "avg_output_tokens_per_task": 4714.72, "last_run_timestamp": "2026-10-24T12:46:25.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.7557780862865947, "accuracy_std_dev": 0.028031620161237676, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.09079595223420647, "avg_duration_per_task_ms": 135126.0123266564, "avg_output_tokens_per_task": 12093.19722650231, "last_run_timestamp": "2026-10-28T04:53:14.750Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.7557576363636364, "accuracy_std_dev": 0.027008915986704473, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5314156181818182, "avg_duration_per_task_ms": 187359.03636363638, "avg_output_tokens_per_task": 10435.296363636364, "last_run_timestamp": "2026-10-24T11:54:04.367Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7556052242152467, "accuracy_std_dev": 0.025010430481878735, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.020433513452914796, "avg_duration_per_task_ms": 233457.43946188342, "avg_output_tokens_per_task": 4244.6569506726455, "last_run_timestamp": "2026-10-24T12:41:47.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.7555555555555555, "accuracy_std_dev": 0.02330683151215116, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.007330602222222223, "avg_duration_per_task_ms": 193346.8088888889, "avg_output_tokens_per_task": 4415.6, "last_run_timestamp": "2026-10-24T14:44:52.720Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.7554901764705882, "accuracy_std_dev": 0.03402120717371878, "run_count": 34, "total_questions": 1700, "avg_cost_per_task_usd": 0.2160354817647059, "avg_duration_per_task_ms": 94519.24058823529, "avg_output_tokens_per_task": 7506.205882352941, "last_run_timestamp": "2026-10-24T12:46:28.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7554166562500002, "accuracy_std_dev": 0.029647766247654916, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.20572255374999998, "avg_duration_per_task_ms": 117215.80375, "avg_output_tokens_per_task": 7761.9825, "last_run_timestamp": "2026-10-24T12:18:37.751Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.7551515454545454, "accuracy_std_dev": 0.030363520838375783, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.2013825509090909, "avg_duration_per_task_ms": 122033.56545454546, "avg_output_tokens_per_task": 7806.483636363636, "last_run_timestamp": "2026-10-24T11:58:02.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.755000125, "accuracy_std_dev": 0.02576585003176445, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.025328775, "avg_duration_per_task_ms": 341291.3375, "avg_output_tokens_per_task": 10920.815, "last_run_timestamp": "2026-10-24T19:18:15.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.755, "accuracy_std_dev": 0.04014562657874455, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.161091306, "avg_duration_per_task_ms": 311751.64, "avg_output_tokens_per_task": 23196.802, "last_run_timestamp": "2026-10-24T12:52:46.316Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.755, "accuracy_std_dev": 0.008333000000000035, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07072132, "avg_duration_per_task_ms": 164484.88, "avg_output_tokens_per_task": 7418.93, "last_run_timestamp": "2026-08-15T08:13:33.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.7548484545454546, "accuracy_std_dev": 0.021805424301655014, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.01682924181818182, "avg_duration_per_task_ms": 119778.08181818182, "avg_output_tokens_per_task": 4470.658181818182, "last_run_timestamp": "2026-10-24T13:29:07.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.7545454545454545, "accuracy_std_dev": 0.03163137195128692, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.008343416363636365, "avg_duration_per_task_ms": 139678.11272727273, "avg_output_tokens_per_task": 4440.807272727273, "last_run_timestamp": "2026-10-24T13:04:30.804Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.7545453636363637, "accuracy_std_dev": 0.01578076033582857, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03706595454545454, "avg_duration_per_task_ms": 272204.82727272727, "avg_output_tokens_per_task": 21398.014545454545, "last_run_timestamp": "2026-10-24T19:02:49.020Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.7542953608247424, "accuracy_std_dev": 0.02652500000000002, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.021299783505154636, "avg_duration_per_task_ms": 633495.5257731959, "avg_output_tokens_per_task": 26267.113402061856, "last_run_timestamp": "2026-08-08T08:30:53.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.7542424545454546, "accuracy_std_dev": 0.03230505110621568, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.2011599981818182, "avg_duration_per_task_ms": 97937.50545454545, "avg_output_tokens_per_task": 7615.459090909091, "last_run_timestamp": "2026-10-24T12:48:21.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7540288473282444, "accuracy_std_dev": 0.03768553089030821, "run_count": 8, "total_questions": 393, "avg_cost_per_task_usd": 0.12041368702290077, "avg_duration_per_task_ms": 478479.56488549616, "avg_output_tokens_per_task": 22514.213740458013, "last_run_timestamp": "2026-10-03T08:18:33.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7539394545454545, "accuracy_std_dev": 0.02627826791498197, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.01945476181818182, "avg_duration_per_task_ms": 219416.4818181818, "avg_output_tokens_per_task": 15698.581818181818, "last_run_timestamp": "2026-10-24T12:17:05.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7539394545454545, "accuracy_std_dev": 0.10561804966556154, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.15414004727272726, "avg_duration_per_task_ms": 605208.28, "avg_output_tokens_per_task": 28608.598181818183, "last_run_timestamp": "2026-10-24T19:26:44.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.753478061769616, "accuracy_std_dev": 0.01967899930322652, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.027070736227045072, "avg_duration_per_task_ms": 546959.5626043406, "avg_output_tokens_per_task": 22325.539232053423, "last_run_timestamp": "2026-10-30T02:35:47.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.16966541999999996, "avg_duration_per_task_ms": 147439, "avg_output_tokens_per_task": 22903.86, "last_run_timestamp": "2026-10-01T19:04:26.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7533335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.13189366, "avg_duration_per_task_ms": 651730.99, "avg_output_tokens_per_task": 24955.07, "last_run_timestamp": "2026-08-15T08:15:35.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33087047999999997, "avg_duration_per_task_ms": 241177.17, "avg_output_tokens_per_task": 11982.77, "last_run_timestamp": "2026-10-01T16:01:02.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7533334615384615, "accuracy_std_dev": 0.02958406824787076, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.34445012307692313, "avg_duration_per_task_ms": 228394.87538461538, "avg_output_tokens_per_task": 10886.436923076923, "last_run_timestamp": "2026-10-24T11:48:43.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.753333375, "accuracy_std_dev": 0.01452975873799614, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1921822125, "avg_duration_per_task_ms": 294611.6675, "avg_output_tokens_per_task": 11751.955, "last_run_timestamp": "2026-10-24T11:55:23.806Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.7533333636363636, "accuracy_std_dev": 0.03856000964087188, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.19848266727272726, "avg_duration_per_task_ms": 100895.72727272728, "avg_output_tokens_per_task": 7619.894545454546, "last_run_timestamp": "2026-10-24T12:18:01.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.024944382578492966, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09710899333333334, "avg_duration_per_task_ms": 165189.49333333335, "avg_output_tokens_per_task": 9718.706666666667, "last_run_timestamp": "2026-10-30T01:11:01.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7533332500000001, "accuracy_std_dev": 0.030731887168501403, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.029856812500000003, "avg_duration_per_task_ms": 481555.77, "avg_output_tokens_per_task": 11765.71, "last_run_timestamp": "2026-10-24T19:23:19.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.7533331428571429, "accuracy_std_dev": 0.0230940520074423, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.011956362857142858, "avg_duration_per_task_ms": 437970.4885714286, "avg_output_tokens_per_task": 6008.582857142857, "last_run_timestamp": "2026-10-24T12:34:15.321Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02472732, "avg_duration_per_task_ms": 1279934.76, "avg_output_tokens_per_task": 22994.42, "last_run_timestamp": "2026-08-08T08:29:47.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "BaseTen", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.024162740000000002, "avg_duration_per_task_ms": 65962.82, "avg_output_tokens_per_task": 8135.38, "last_run_timestamp": "2026-10-24T12:37:26.070Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Decart", "accuracy": 0.753333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01197713, "avg_duration_per_task_ms": 212560.45, "avg_output_tokens_per_task": 13957.42, "last_run_timestamp": "2026-10-24T11:49:49.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Baidu", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0185477, "avg_duration_per_task_ms": 97588.58, "avg_output_tokens_per_task": 10717.4, "last_run_timestamp": "2026-10-24T11:41:40.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-codex-20260224", "model_slug": "openai/gpt-5.3-codex", "model_name": "OpenAI: GPT-5.3-Codex", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29971074000000003, "avg_duration_per_task_ms": 180697.5, "avg_output_tokens_per_task": 13569.24, "last_run_timestamp": "2026-10-02T23:12:05.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.7533329999999999, "accuracy_std_dev": 0.03999999999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0194427, "avg_duration_per_task_ms": 127110.83, "avg_output_tokens_per_task": 12815.84, "last_run_timestamp": "2026-10-24T12:03:02.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.7531401420289855, "accuracy_std_dev": 0.02334674149443907, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.0109654, "avg_duration_per_task_ms": 132639.57101449274, "avg_output_tokens_per_task": 12361.4, "last_run_timestamp": "2026-10-24T19:21:18.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.7527273636363636, "accuracy_std_dev": 0.022822115642797527, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07911564, "avg_duration_per_task_ms": 552538.9854545455, "avg_output_tokens_per_task": 35028.91454545454, "last_run_timestamp": "2026-10-24T19:16:16.979Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7527272727272727, "accuracy_std_dev": 0.029875735292863623, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.013207607272727274, "avg_duration_per_task_ms": 136477.7381818182, "avg_output_tokens_per_task": 16262.556363636364, "last_run_timestamp": "2026-10-24T11:51:53.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7525337229729728, "accuracy_std_dev": 0.02409323915008227, "run_count": 12, "total_questions": 592, "avg_cost_per_task_usd": 0.014006842905405404, "avg_duration_per_task_ms": 57600.26858108108, "avg_output_tokens_per_task": 4072.2314189189187, "last_run_timestamp": "2026-10-24T12:20:04.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.752500125, "accuracy_std_dev": 0.02039545667199868, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0145751425, "avg_duration_per_task_ms": 232212.94, "avg_output_tokens_per_task": 14651.365, "last_run_timestamp": "2026-10-24T19:19:42.648Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.7522221666666666, "accuracy_std_dev": 0.023227349249083266, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.0077021400000000005, "avg_duration_per_task_ms": 223752.14166666666, "avg_output_tokens_per_task": 14385.541666666666, "last_run_timestamp": "2026-10-24T12:31:25.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.752, "accuracy_std_dev": 0.01857135189478677, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.33952648, "avg_duration_per_task_ms": 120001.212, "avg_output_tokens_per_task": 7107.392, "last_run_timestamp": "2026-10-24T12:22:52.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7518518333333334, "accuracy_std_dev": 0.017715982107345284, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.36871217222222225, "avg_duration_per_task_ms": 149259.27444444445, "avg_output_tokens_per_task": 7150.745555555555, "last_run_timestamp": "2026-10-24T12:24:28.843Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.7518183181818181, "accuracy_std_dev": 0.02334910466725432, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.2542307654545455, "avg_duration_per_task_ms": 230926.2109090909, "avg_output_tokens_per_task": 10132.794545454546, "last_run_timestamp": "2026-10-30T00:59:12.977Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7517172727272728, "accuracy_std_dev": 0.023898191710814292, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.37267475515151516, "avg_duration_per_task_ms": 157429.93757575756, "avg_output_tokens_per_task": 11808.122424242425, "last_run_timestamp": "2026-10-24T12:39:24.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.751666875, "accuracy_std_dev": 0.03278711001841082, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08480622499999999, "avg_duration_per_task_ms": 70521.9675, "avg_output_tokens_per_task": 11906.8925, "last_run_timestamp": "2026-10-24T19:37:32.789Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.7516667499999999, "accuracy_std_dev": 0.01723988431479461, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.066217935, "avg_duration_per_task_ms": 295605.795, "avg_output_tokens_per_task": 8579.39, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.751666625, "accuracy_std_dev": 0.046874494071769725, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11106282249999999, "avg_duration_per_task_ms": 257380.8975, "avg_output_tokens_per_task": 11828.0925, "last_run_timestamp": "2026-10-24T19:31:21.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.7515151818181818, "accuracy_std_dev": 0.027131013635248374, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5026920418181818, "avg_duration_per_task_ms": 194031.15818181817, "avg_output_tokens_per_task": 10111.854545454546, "last_run_timestamp": "2026-10-24T12:42:05.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7514285714285714, "accuracy_std_dev": 0.021296004758486267, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.009337331428571431, "avg_duration_per_task_ms": 230829.7457142857, "avg_output_tokens_per_task": 4265.0371428571425, "last_run_timestamp": "2026-10-24T13:34:36.032Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7512689898477157, "accuracy_std_dev": 0.021764084474197413, "run_count": 4, "total_questions": 197, "avg_cost_per_task_usd": 0.12904325888324872, "avg_duration_per_task_ms": 273181.16751269036, "avg_output_tokens_per_task": 46030.85279187817, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.7511111666666667, "accuracy_std_dev": 0.02454511593248011, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3494615833333333, "avg_duration_per_task_ms": 100555.84, "avg_output_tokens_per_task": 7111.15, "last_run_timestamp": "2026-10-24T11:44:08.041Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.751111, "accuracy_std_dev": 0.01523486904002351, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3475124233333333, "avg_duration_per_task_ms": 124013.22666666667, "avg_output_tokens_per_task": 7125.08, "last_run_timestamp": "2026-10-24T11:31:04.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Fireworks", "accuracy": 0.7510038032128515, "accuracy_std_dev": 0.01043774738916402, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.023900196787148595, "avg_duration_per_task_ms": 72851.72690763052, "avg_output_tokens_per_task": 12280.232931726907, "last_run_timestamp": "2026-10-28T00:15:49.021Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.7506668, "accuracy_std_dev": 0.025854254755455656, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.264866352, "avg_duration_per_task_ms": 264537.464, "avg_output_tokens_per_task": 10440.212, "last_run_timestamp": "2026-10-24T11:49:28.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.7506666, "accuracy_std_dev": 0.024073823498563778, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.022084748, "avg_duration_per_task_ms": 137222.84, "avg_output_tokens_per_task": 13584.26, "last_run_timestamp": "2026-10-24T19:40:37.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7506666, "accuracy_std_dev": 0.012364666507431593, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.17681883199999998, "avg_duration_per_task_ms": 536508.956, "avg_output_tokens_per_task": 9711.396, "last_run_timestamp": "2026-10-03T16:16:10.656Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7505555833333333, "accuracy_std_dev": 0.034043327967112576, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.102866665, "avg_duration_per_task_ms": 304845.985, "avg_output_tokens_per_task": 18629.698333333334, "last_run_timestamp": "2026-10-27T01:19:50.371Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.7505555, "accuracy_std_dev": 0.03574174773170259, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03120006, "avg_duration_per_task_ms": 258878.755, "avg_output_tokens_per_task": 17037.54, "last_run_timestamp": "2026-10-24T12:34:23.300Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.7503030909090909, "accuracy_std_dev": 0.028193269749597228, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.02549516181818182, "avg_duration_per_task_ms": 155201.13454545455, "avg_output_tokens_per_task": 9057.756363636363, "last_run_timestamp": "2026-10-24T19:12:17.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.7502788193979932, "accuracy_std_dev": 0.03287689055865162, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.01826405016722408, "avg_duration_per_task_ms": 135784.91304347827, "avg_output_tokens_per_task": 4013.7859531772574, "last_run_timestamp": "2026-10-24T12:16:33.907Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.7500001666666666, "accuracy_std_dev": 0.023333261905247805, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.3534387216666667, "avg_duration_per_task_ms": 87888.735, "avg_output_tokens_per_task": 7005.235, "last_run_timestamp": "2026-10-24T12:23:35.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7500001, "accuracy_std_dev": 0.02333341428702625, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.196243942, "avg_duration_per_task_ms": 117146.822, "avg_output_tokens_per_task": 7664.984, "last_run_timestamp": "2026-10-24T12:32:11.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.75, "accuracy_std_dev": 0.010000000000000009, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.300415095, "avg_duration_per_task_ms": 278099.495, "avg_output_tokens_per_task": 10648.38, "last_run_timestamp": "2026-10-24T12:46:10.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29130333999999997, "avg_duration_per_task_ms": 176962.37, "avg_output_tokens_per_task": 10386.26, "last_run_timestamp": "2026-10-01T19:04:36.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.75, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01952235, "avg_duration_per_task_ms": 2267441.92, "avg_output_tokens_per_task": 19667.19, "last_run_timestamp": "2026-10-03T08:13:52.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.75, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08838628, "avg_duration_per_task_ms": 204069.19, "avg_output_tokens_per_task": 7577, "last_run_timestamp": "2026-08-15T08:10:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Amazon Bedrock", "accuracy": 0.75, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09412079000000001, "avg_duration_per_task_ms": 469968.05, "avg_output_tokens_per_task": 4703.42, "last_run_timestamp": "2026-10-24T12:22:38.011Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.7493334, "accuracy_std_dev": 0.035427579965896655, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.014355716000000001, "avg_duration_per_task_ms": 155681.656, "avg_output_tokens_per_task": 14723.468, "last_run_timestamp": "2026-10-03T16:15:22.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7492560625, "accuracy_std_dev": 0.02911246180199882, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.0047560535714285725, "avg_duration_per_task_ms": 260988.63169642858, "avg_output_tokens_per_task": 12223.955357142857, "last_run_timestamp": "2026-10-24T20:11:39.941Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.74916675, "accuracy_std_dev": 0.022095025207894678, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.03993733, "avg_duration_per_task_ms": 621867.6175, "avg_output_tokens_per_task": 16182.08, "last_run_timestamp": "2026-10-24T19:37:04.226Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.749166625, "accuracy_std_dev": 0.022837048057583416, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0985293125, "avg_duration_per_task_ms": 261632.96, "avg_output_tokens_per_task": 12654.5375, "last_run_timestamp": "2026-10-24T19:44:46.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.7490908636363637, "accuracy_std_dev": 0.027527419506071103, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.5507198090909092, "avg_duration_per_task_ms": 199727.5418181818, "avg_output_tokens_per_task": 9759.008181818182, "last_run_timestamp": "2026-10-24T12:58:30.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.7488888888888889, "accuracy_std_dev": 0.016329946740366617, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01262068, "avg_duration_per_task_ms": 336899.4511111111, "avg_output_tokens_per_task": 16364.997777777779, "last_run_timestamp": "2026-10-24T12:24:10.218Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7486274705882353, "accuracy_std_dev": 0.025128604245121908, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.021811628235294114, "avg_duration_per_task_ms": 107903.25411764705, "avg_output_tokens_per_task": 14472.312941176471, "last_run_timestamp": "2026-10-29T22:41:53.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.748333375, "accuracy_std_dev": 0.024664456546503812, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0074231424999999995, "avg_duration_per_task_ms": 153071.365, "avg_output_tokens_per_task": 12771.8775, "last_run_timestamp": "2026-10-24T19:02:53.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.748333375, "accuracy_std_dev": 0.018782375754530515, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0418515775, "avg_duration_per_task_ms": 191146.0675, "avg_output_tokens_per_task": 12827.9725, "last_run_timestamp": "2026-10-03T08:33:09.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7483333, "accuracy_std_dev": 0.02761851834566802, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.045017131999999994, "avg_duration_per_task_ms": 459054.072, "avg_output_tokens_per_task": 23025.238, "last_run_timestamp": "2026-10-24T19:38:23.863Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Nebius", "accuracy": 0.748275551724138, "accuracy_std_dev": 0.011091913731282908, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.00999303448275862, "avg_duration_per_task_ms": 128162.52413793103, "avg_output_tokens_per_task": 3554.88275862069, "last_run_timestamp": "2026-10-24T12:44:22.556Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7480002, "accuracy_std_dev": 0.020396130352593862, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030171100000000003, "avg_duration_per_task_ms": 197857.312, "avg_output_tokens_per_task": 18663.832, "last_run_timestamp": "2026-10-03T08:18:33.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7478788181818182, "accuracy_std_dev": 0.018817276123132574, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3490534545454545, "avg_duration_per_task_ms": 215062.8381818182, "avg_output_tokens_per_task": 11920.356363636363, "last_run_timestamp": "2026-10-24T12:39:07.663Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.7477778333333334, "accuracy_std_dev": 0.022580213671978926, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.018034246666666667, "avg_duration_per_task_ms": 75225.08333333333, "avg_output_tokens_per_task": 4268.986666666667, "last_run_timestamp": "2026-10-24T12:38:26.301Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.747691088161209, "accuracy_std_dev": 0.03008101142513954, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.09758945340050378, "avg_duration_per_task_ms": 397184.00755667506, "avg_output_tokens_per_task": 25302.57934508816, "last_run_timestamp": "2026-10-24T13:02:31.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.747619, "accuracy_std_dev": 0.020292645810736476, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.010855531428571428, "avg_duration_per_task_ms": 226454.37142857144, "avg_output_tokens_per_task": 13871.608571428571, "last_run_timestamp": "2026-10-24T19:37:39.139Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.747619, "accuracy_std_dev": 0.025305279849978234, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.33171734571428574, "avg_duration_per_task_ms": 235739.5942857143, "avg_output_tokens_per_task": 12164.34, "last_run_timestamp": "2026-10-01T16:01:04.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.7475554666666667, "accuracy_std_dev": 0.02281875035978576, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.013525546666666666, "avg_duration_per_task_ms": 139030.82666666666, "avg_output_tokens_per_task": 15549.14, "last_run_timestamp": "2026-10-24T12:22:53.795Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7475, "accuracy_std_dev": 0.02271496080780242, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06216827, "avg_duration_per_task_ms": 205983.465, "avg_output_tokens_per_task": 10804.545, "last_run_timestamp": "2026-10-24T20:08:17.474Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7473276647230321, "accuracy_std_dev": 0.024479710940170173, "run_count": 7, "total_questions": 343, "avg_cost_per_task_usd": 0.13171293294460643, "avg_duration_per_task_ms": 209506.55976676385, "avg_output_tokens_per_task": 11363.093294460641, "last_run_timestamp": "2026-10-24T19:49:56.630Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.7472726363636364, "accuracy_std_dev": 0.03984348764688863, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.04194175272727273, "avg_duration_per_task_ms": 190838.81636363638, "avg_output_tokens_per_task": 19126.147272727274, "last_run_timestamp": "2026-10-24T12:19:21.144Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.7472221666666666, "accuracy_std_dev": 0.011613686586906374, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17439429333333334, "avg_duration_per_task_ms": 161420.59666666668, "avg_output_tokens_per_task": 12499.9, "last_run_timestamp": "2026-10-10T11:19:48.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.747143, "accuracy_std_dev": 0.033305956785965055, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014612914285714287, "avg_duration_per_task_ms": 50786.11142857143, "avg_output_tokens_per_task": 4096.908571428571, "last_run_timestamp": "2026-10-24T13:27:44.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7468778674351586, "accuracy_std_dev": 0.03317731677283287, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.06897791354466859, "avg_duration_per_task_ms": 486659.6887608069, "avg_output_tokens_per_task": 10547.132564841499, "last_run_timestamp": "2026-10-24T19:42:30.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.746667, "accuracy_std_dev": 0.014142135623730963, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01997175, "avg_duration_per_task_ms": 504783.88, "avg_output_tokens_per_task": 14879.43, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.746667, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00955731, "avg_duration_per_task_ms": 207347.51, "avg_output_tokens_per_task": 13127.37, "last_run_timestamp": "2026-10-24T12:22:57.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "auto-routing", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08652918, "avg_duration_per_task_ms": 60257.98, "avg_output_tokens_per_task": 8644.22, "last_run_timestamp": "2026-10-24T12:40:48.538Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Relace", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00930136, "avg_duration_per_task_ms": 89811.58, "avg_output_tokens_per_task": 12020.36, "last_run_timestamp": "2026-10-24T15:18:33.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "GMICloud", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07358494, "avg_duration_per_task_ms": 796913.16, "avg_output_tokens_per_task": 13587.26, "last_run_timestamp": "2026-10-27T10:59:10.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07074242, "avg_duration_per_task_ms": 800280.44, "avg_output_tokens_per_task": 15183.42, "last_run_timestamp": "2026-10-02T06:00:07.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.7466667142857143, "accuracy_std_dev": 0.025448285565123643, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04929559142857143, "avg_duration_per_task_ms": 346170.3714285714, "avg_output_tokens_per_task": 11334.602857142858, "last_run_timestamp": "2026-10-24T21:05:35.420Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.019626097517789383, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05652010888888889, "avg_duration_per_task_ms": 356610.63333333336, "avg_output_tokens_per_task": 14359.217777777778, "last_run_timestamp": "2026-10-29T22:50:33.321Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.7466666000000001, "accuracy_std_dev": 0.030983694638309366, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.022173396, "avg_duration_per_task_ms": 122667.268, "avg_output_tokens_per_task": 7404.18, "last_run_timestamp": "2026-10-24T12:20:04.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.7466665714285714, "accuracy_std_dev": 0.02760272588431981, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.056462091428571434, "avg_duration_per_task_ms": 739730.7085714285, "avg_output_tokens_per_task": 16026.454285714286, "last_run_timestamp": "2026-10-24T19:50:11.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7466665714285714, "accuracy_std_dev": 0.03380611384628165, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03743281428571429, "avg_duration_per_task_ms": 197093.65714285715, "avg_output_tokens_per_task": 13601.3, "last_run_timestamp": "2026-10-03T08:15:57.213Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.7466665714285714, "accuracy_std_dev": 0.029814069333871523, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02163002857142857, "avg_duration_per_task_ms": 164450.89714285714, "avg_output_tokens_per_task": 8529.557142857142, "last_run_timestamp": "2026-10-24T18:59:48.400Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Together", "accuracy": 0.7466665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.019887960000000003, "avg_duration_per_task_ms": 51072.42, "avg_output_tokens_per_task": 13340.71, "last_run_timestamp": "2026-10-24T12:26:21.165Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.7466665, "accuracy_std_dev": 0.0466665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.011416649999999999, "avg_duration_per_task_ms": 95996.46, "avg_output_tokens_per_task": 13669.03, "last_run_timestamp": "2026-10-24T12:07:39.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.011547101609639251, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04422623666666667, "avg_duration_per_task_ms": 266653.04, "avg_output_tokens_per_task": 13360.296666666667, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.58250779, "avg_duration_per_task_ms": 102747.13, "avg_output_tokens_per_task": 7546.85, "last_run_timestamp": "2026-10-01T19:04:42.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7463769999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.05924082608695652, "avg_duration_per_task_ms": 289784.32608695654, "avg_output_tokens_per_task": 9531.695652173914, "last_run_timestamp": "2026-08-15T14:23:29.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.7461904285714286, "accuracy_std_dev": 0.021560263063973885, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12111663714285714, "avg_duration_per_task_ms": 120926.64285714286, "avg_output_tokens_per_task": 10598.828571428572, "last_run_timestamp": "2026-10-17T19:14:08.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7460607272727272, "accuracy_std_dev": 0.03259221120867796, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5475128981818181, "avg_duration_per_task_ms": 181106.1872727273, "avg_output_tokens_per_task": 10316.370909090909, "last_run_timestamp": "2026-10-24T12:39:07.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.7460605454545455, "accuracy_std_dev": 0.027035941116708276, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.52474048, "avg_duration_per_task_ms": 221889.97272727272, "avg_output_tokens_per_task": 10006.24909090909, "last_run_timestamp": "2026-10-24T12:47:43.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.7460000999999999, "accuracy_std_dev": 0.04038166415701563, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.024082161999999997, "avg_duration_per_task_ms": 403869.142, "avg_output_tokens_per_task": 15439.562, "last_run_timestamp": "2026-10-17T11:15:55.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.7459999, "accuracy_std_dev": 0.035080781064423305, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.36040775999999997, "avg_duration_per_task_ms": 382799.108, "avg_output_tokens_per_task": 9698.124, "last_run_timestamp": "2026-10-28T03:50:02.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.745975738515901, "accuracy_std_dev": 0.015812418783302782, "run_count": 17, "total_questions": 849, "avg_cost_per_task_usd": 0.044388724381625444, "avg_duration_per_task_ms": 325041.3757361602, "avg_output_tokens_per_task": 25777.905771495876, "last_run_timestamp": "2026-10-30T01:06:37.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.7459258888888889, "accuracy_std_dev": 0.025809336792221565, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.17245649777777777, "avg_duration_per_task_ms": 324708.56444444443, "avg_output_tokens_per_task": 11055.957777777778, "last_run_timestamp": "2026-10-24T12:33:16.196Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.745833375, "accuracy_std_dev": 0.0427442661854707, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11668912499999999, "avg_duration_per_task_ms": 240026.82, "avg_output_tokens_per_task": 9661.8575, "last_run_timestamp": "2026-10-24T19:48:41.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.74583325, "accuracy_std_dev": 0.03402396010442498, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02499106, "avg_duration_per_task_ms": 929627.78, "avg_output_tokens_per_task": 52531.49, "last_run_timestamp": "2026-10-24T12:31:49.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7457142857142857, "accuracy_std_dev": 0.029692331634924993, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.5093009542857143, "avg_duration_per_task_ms": 175938.91142857142, "avg_output_tokens_per_task": 7450.982857142857, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.745641, "accuracy_std_dev": 0.025997527320290117, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.34545777384615384, "avg_duration_per_task_ms": 208026.4569230769, "avg_output_tokens_per_task": 11868.513846153846, "last_run_timestamp": "2026-10-24T13:09:08.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.7455556666666666, "accuracy_std_dev": 0.020846357411627908, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.018734011111111108, "avg_duration_per_task_ms": 376517.3044444444, "avg_output_tokens_per_task": 24556.922222222223, "last_run_timestamp": "2026-10-24T11:55:52.478Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7454488757062147, "accuracy_std_dev": 0.03153896451944291, "run_count": 11, "total_questions": 531, "avg_cost_per_task_usd": 0.022084410546139358, "avg_duration_per_task_ms": 94955.74764595104, "avg_output_tokens_per_task": 7543.256120527307, "last_run_timestamp": "2026-10-24T18:57:18.438Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7453334, "accuracy_std_dev": 0.02543827737170895, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.346131274, "avg_duration_per_task_ms": 200655.086, "avg_output_tokens_per_task": 11839.45, "last_run_timestamp": "2026-10-24T13:00:12.600Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.7453333, "accuracy_std_dev": 0.027936434196403816, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.02067147, "avg_duration_per_task_ms": 399151.052, "avg_output_tokens_per_task": 11925.912, "last_run_timestamp": "2026-10-17T11:29:07.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.7451852222222222, "accuracy_std_dev": 0.02994276662337214, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06914028222222222, "avg_duration_per_task_ms": 256444.49333333335, "avg_output_tokens_per_task": 9708.04, "last_run_timestamp": "2026-10-24T19:33:48.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.7451851111111111, "accuracy_std_dev": 0.030270966502370805, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4099032177777778, "avg_duration_per_task_ms": 619397.1977777778, "avg_output_tokens_per_task": 9848.733333333334, "last_run_timestamp": "2026-10-24T12:05:22.704Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.745000125, "accuracy_std_dev": 0.03312433437609538, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.015775572499999998, "avg_duration_per_task_ms": 148938.23, "avg_output_tokens_per_task": 15126.975, "last_run_timestamp": "2026-10-24T12:05:54.269Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.745, "accuracy_std_dev": 0.026822730826670128, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.15284815, "avg_duration_per_task_ms": 494527.49, "avg_output_tokens_per_task": 28246.935, "last_run_timestamp": "2026-10-17T19:47:21.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.745, "accuracy_std_dev": 0.03362364412879724, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1356366175, "avg_duration_per_task_ms": 342258.37, "avg_output_tokens_per_task": 11856.1675, "last_run_timestamp": "2026-10-24T19:38:37.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.745, "accuracy_std_dev": 0.028332999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01250761, "avg_duration_per_task_ms": 355978.98, "avg_output_tokens_per_task": 23604.2, "last_run_timestamp": "2026-10-24T12:29:47.596Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.745, "accuracy_std_dev": 0.040380248197602775, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.007515549999999999, "avg_duration_per_task_ms": 328112.605, "avg_output_tokens_per_task": 3998.705, "last_run_timestamp": "2026-10-24T13:13:17.945Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.7448485454545455, "accuracy_std_dev": 0.021944100032763564, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.4515615263636363, "avg_duration_per_task_ms": 587567.8727272727, "avg_output_tokens_per_task": 10197.047272727274, "last_run_timestamp": "2026-10-24T11:44:25.226Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.7448484545454546, "accuracy_std_dev": 0.024468606225356824, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.14367746363636363, "avg_duration_per_task_ms": 628527.9981818182, "avg_output_tokens_per_task": 26968.087272727273, "last_run_timestamp": "2026-10-24T20:02:49.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.7448276551724138, "accuracy_std_dev": 0.0035438860340341496, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.022715406896551726, "avg_duration_per_task_ms": 289516.0620689655, "avg_output_tokens_per_task": 10103.63448275862, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.744681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.05595836170212766, "avg_duration_per_task_ms": 34218.255319148935, "avg_output_tokens_per_task": 6745.085106382979, "last_run_timestamp": "2026-08-04T03:12:18.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7445560603015074, "accuracy_std_dev": 0.018800290223823686, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.0472956608040201, "avg_duration_per_task_ms": 744919.2989949749, "avg_output_tokens_per_task": 14283.391959798995, "last_run_timestamp": "2026-10-24T19:19:10.662Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04041968666666667, "avg_duration_per_task_ms": 291588.68666666665, "avg_output_tokens_per_task": 24330.48, "last_run_timestamp": "2026-09-30T11:25:05.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7442960934891485, "accuracy_std_dev": 0.02614118042622386, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.011909569282136896, "avg_duration_per_task_ms": 437548.9916527546, "avg_output_tokens_per_task": 14218.791318864774, "last_run_timestamp": "2026-10-24T12:42:25.209Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.7442423636363636, "accuracy_std_dev": 0.04102458399827359, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.011285496363636365, "avg_duration_per_task_ms": 301862.60363636364, "avg_output_tokens_per_task": 16012.072727272727, "last_run_timestamp": "2026-10-24T12:46:15.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.74416675, "accuracy_std_dev": 0.03012731088543251, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.064155875, "avg_duration_per_task_ms": 1722921.32, "avg_output_tokens_per_task": 8915.945, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7441665000000001, "accuracy_std_dev": 0.036045969878753456, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0744832625, "avg_duration_per_task_ms": 216241.835, "avg_output_tokens_per_task": 9512.8075, "last_run_timestamp": "2026-10-24T19:32:20.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.744131569416499, "accuracy_std_dev": 0.0294471423722235, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.06635609456740442, "avg_duration_per_task_ms": 1016068.6076458753, "avg_output_tokens_per_task": 74326.28571428571, "last_run_timestamp": "2026-10-24T19:38:23.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7440001333333333, "accuracy_std_dev": 0.026195840213964414, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.059455711999999994, "avg_duration_per_task_ms": 139058.504, "avg_output_tokens_per_task": 9866.470666666666, "last_run_timestamp": "2026-10-28T07:08:54.400Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7439998000000001, "accuracy_std_dev": 0.025508095447524134, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.010447856000000002, "avg_duration_per_task_ms": 427694.796, "avg_output_tokens_per_task": 12591.384, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.743738, "accuracy_std_dev": 0.01688456971344439, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.00938970520231214, "avg_duration_per_task_ms": 347914.32658959535, "avg_output_tokens_per_task": 3877.916184971098, "last_run_timestamp": "2026-10-03T08:46:08.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.7437037777777777, "accuracy_std_dev": 0.029832554190115407, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.020828822222222224, "avg_duration_per_task_ms": 153282.48666666666, "avg_output_tokens_per_task": 16327.455555555556, "last_run_timestamp": "2026-10-27T00:50:04.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.7436363636363637, "accuracy_std_dev": 0.02775984111584323, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.32704656909090907, "avg_duration_per_task_ms": 196939.35090909092, "avg_output_tokens_per_task": 11566.430909090908, "last_run_timestamp": "2026-10-24T12:37:04.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Mancer 2", "accuracy": 0.7433335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06141422999999999, "avg_duration_per_task_ms": 1060280.1, "avg_output_tokens_per_task": 12094.68, "last_run_timestamp": "2026-10-24T12:42:07.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.7433335, "accuracy_std_dev": 0.023333285714832386, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.006871515, "avg_duration_per_task_ms": 97438.585, "avg_output_tokens_per_task": 9380.645, "last_run_timestamp": "2026-10-24T12:26:07.751Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.7433335, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03822515, "avg_duration_per_task_ms": 176603.12, "avg_output_tokens_per_task": 16434.51, "last_run_timestamp": "2026-10-24T18:56:11.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.7433335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03870923, "avg_duration_per_task_ms": 112805.69, "avg_output_tokens_per_task": 14592.13, "last_run_timestamp": "2026-10-24T12:30:47.912Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7433335, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64823893, "avg_duration_per_task_ms": 264133.83, "avg_output_tokens_per_task": 10511, "last_run_timestamp": "2026-10-01T16:00:37.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.017532868606008445, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012350623333333333, "avg_duration_per_task_ms": 125234.99333333333, "avg_output_tokens_per_task": 4198.8, "last_run_timestamp": "2026-10-03T08:23:09.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.7433333, "accuracy_std_dev": 0.03441571669470216, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.028436006, "avg_duration_per_task_ms": 438756.644, "avg_output_tokens_per_task": 14726.93, "last_run_timestamp": "2026-10-17T13:55:17.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.7433332500000001, "accuracy_std_dev": 0.01774311029726281, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.013128406666666667, "avg_duration_per_task_ms": 72447.74833333334, "avg_output_tokens_per_task": 10872.788333333334, "last_run_timestamp": "2026-10-24T12:05:47.786Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.7433332500000001, "accuracy_std_dev": 0.023804656407675817, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.055247185, "avg_duration_per_task_ms": 380831.14, "avg_output_tokens_per_task": 8536.295, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.742963111111111, "accuracy_std_dev": 0.025936488854483866, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07987847333333332, "avg_duration_per_task_ms": 298551.6066666667, "avg_output_tokens_per_task": 11298.702222222222, "last_run_timestamp": "2026-10-24T23:01:57.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.7429629444444444, "accuracy_std_dev": 0.018622075637599298, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.35968263222222224, "avg_duration_per_task_ms": 87204.23444444445, "avg_output_tokens_per_task": 7123.5233333333335, "last_run_timestamp": "2026-10-24T12:18:54.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.7428572857142857, "accuracy_std_dev": 0.03884962929300717, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.025685354285714286, "avg_duration_per_task_ms": 140747.64, "avg_output_tokens_per_task": 11913.908571428572, "last_run_timestamp": "2026-10-27T01:02:57.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.7426668, "accuracy_std_dev": 0.03690245640821219, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011645404, "avg_duration_per_task_ms": 585840.916, "avg_output_tokens_per_task": 18794.172, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7426666000000001, "accuracy_std_dev": 0.02908997383704564, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.504919826, "avg_duration_per_task_ms": 191444.46, "avg_output_tokens_per_task": 9912.906, "last_run_timestamp": "2026-10-24T11:43:06.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.7426265425709516, "accuracy_std_dev": 0.027093567829324792, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.04797976794657762, "avg_duration_per_task_ms": 180887.38898163606, "avg_output_tokens_per_task": 9161.572621035059, "last_run_timestamp": "2026-10-24T19:33:32.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.7425926666666667, "accuracy_std_dev": 0.03165311687654157, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.013586744444444443, "avg_duration_per_task_ms": 236886.98, "avg_output_tokens_per_task": 22693.746666666666, "last_run_timestamp": "2026-10-24T12:29:34.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.7425595803571429, "accuracy_std_dev": 0.02504151285854947, "run_count": 7, "total_questions": 336, "avg_cost_per_task_usd": 0.15002946726190475, "avg_duration_per_task_ms": 711525.6666666666, "avg_output_tokens_per_task": 29928.91369047619, "last_run_timestamp": "2026-10-24T19:30:34.911Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.7425, "accuracy_std_dev": 0.01697637247028941, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.22007109249999998, "avg_duration_per_task_ms": 261814.77, "avg_output_tokens_per_task": 11695.5675, "last_run_timestamp": "2026-10-24T11:58:56.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7425, "accuracy_std_dev": 0.03281884621296123, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.101450235, "avg_duration_per_task_ms": 312758.3475, "avg_output_tokens_per_task": 7418.4, "last_run_timestamp": "2026-10-24T19:37:38.705Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.7422716733870968, "accuracy_std_dev": 0.017259512677071726, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.01751647782258065, "avg_duration_per_task_ms": 79788.1754032258, "avg_output_tokens_per_task": 4101.441532258064, "last_run_timestamp": "2026-10-28T10:48:35.415Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0028365266666666666, "avg_duration_per_task_ms": 68565.14666666667, "avg_output_tokens_per_task": 11675.026666666667, "last_run_timestamp": "2026-10-01T16:01:02.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.27693500666666665, "avg_duration_per_task_ms": 100083.4, "avg_output_tokens_per_task": 5197.013333333333, "last_run_timestamp": "2026-10-30T00:15:31.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.7417778, "accuracy_std_dev": 0.0192398742345162, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.016580628, "avg_duration_per_task_ms": 168506.08666666667, "avg_output_tokens_per_task": 4247.114666666666, "last_run_timestamp": "2026-10-24T13:06:52.407Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.7416667499999999, "accuracy_std_dev": 0.02074966532712037, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.019690735, "avg_duration_per_task_ms": 215383.61, "avg_output_tokens_per_task": 12170.26, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.7416665, "accuracy_std_dev": 0.03140250343921645, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012469425000000001, "avg_duration_per_task_ms": 90319.205, "avg_output_tokens_per_task": 10893.135, "last_run_timestamp": "2026-10-24T12:16:35.471Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.7413793735632184, "accuracy_std_dev": 0.05076740924399175, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.08319576436781609, "avg_duration_per_task_ms": 840263.8620689656, "avg_output_tokens_per_task": 51575.90517241379, "last_run_timestamp": "2026-10-24T18:56:22.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.7410191879699248, "accuracy_std_dev": 0.023857300276526983, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.22161520300751877, "avg_duration_per_task_ms": 375984.74686716794, "avg_output_tokens_per_task": 9942.047619047618, "last_run_timestamp": "2026-10-24T12:46:10.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7406779661016949, "accuracy_std_dev": 0.0192449897289312, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.02004586440677966, "avg_duration_per_task_ms": 1117850.5423728814, "avg_output_tokens_per_task": 22657.922033898307, "last_run_timestamp": "2026-09-12T08:11:08.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.7406668999999999, "accuracy_std_dev": 0.04294179161015525, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.009957568, "avg_duration_per_task_ms": 419095.964, "avg_output_tokens_per_task": 14787.174, "last_run_timestamp": "2026-10-17T11:26:35.790Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.7406667, "accuracy_std_dev": 0.05001767643153769, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.009102672, "avg_duration_per_task_ms": 299089.77, "avg_output_tokens_per_task": 15164.772, "last_run_timestamp": "2026-10-24T11:53:54.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7405555, "accuracy_std_dev": 0.01855080944101005, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.0133585, "avg_duration_per_task_ms": 101370.275, "avg_output_tokens_per_task": 10830.071666666667, "last_run_timestamp": "2026-10-24T11:56:06.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7403685728643216, "accuracy_std_dev": 0.02451522166373828, "run_count": 12, "total_questions": 597, "avg_cost_per_task_usd": 0.05593858793969849, "avg_duration_per_task_ms": 169768.54773869348, "avg_output_tokens_per_task": 30975.941373534337, "last_run_timestamp": "2026-10-24T12:06:17.420Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7403274419642857, "accuracy_std_dev": 0.036586619279744714, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.07693905803571428, "avg_duration_per_task_ms": 873819.4196428572, "avg_output_tokens_per_task": 14604.90625, "last_run_timestamp": "2026-10-24T20:35:30.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.7402083125000002, "accuracy_std_dev": 0.030068559537369668, "run_count": 32, "total_questions": 1600, "avg_cost_per_task_usd": 0.37145770687499996, "avg_duration_per_task_ms": 200089.441875, "avg_output_tokens_per_task": 11938.27625, "last_run_timestamp": "2026-10-24T13:14:09.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.7401633919821825, "accuracy_std_dev": 0.02335439453822158, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.061262200445434296, "avg_duration_per_task_ms": 141198.91759465478, "avg_output_tokens_per_task": 4110.719376391982, "last_run_timestamp": "2026-10-30T02:02:14.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7400001666666666, "accuracy_std_dev": 0.01154690915954955, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.034900296666666664, "avg_duration_per_task_ms": 129527.63333333333, "avg_output_tokens_per_task": 16132.923333333334, "last_run_timestamp": "2026-10-27T00:42:49.766Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.740000125, "accuracy_std_dev": 0.02924983380908983, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.072280915, "avg_duration_per_task_ms": 322638.715, "avg_output_tokens_per_task": 11472.485, "last_run_timestamp": "2026-10-24T22:27:21.158Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.7400001111111111, "accuracy_std_dev": 0.021314690929781274, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3873409377777778, "avg_duration_per_task_ms": 192992.06, "avg_output_tokens_per_task": 11071.72, "last_run_timestamp": "2026-10-24T12:36:29.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.74, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.23335095, "avg_duration_per_task_ms": 96091.26, "avg_output_tokens_per_task": 5555.9, "last_run_timestamp": "2026-10-01T19:04:26.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Mistral", "accuracy": 0.74, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.047416770000000004, "avg_duration_per_task_ms": 82837.68, "avg_output_tokens_per_task": 13780.79, "last_run_timestamp": "2026-10-30T16:15:51.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "OpenInference", "accuracy": 0.74, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02650346, "avg_duration_per_task_ms": 245657.46, "avg_output_tokens_per_task": 33745.71, "last_run_timestamp": "2026-10-24T12:08:32.744Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.74, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.37915447, "avg_duration_per_task_ms": 148084.91, "avg_output_tokens_per_task": 12525.4, "last_run_timestamp": "2026-10-01T19:04:55.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Together", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08715152, "avg_duration_per_task_ms": 69751.7, "avg_output_tokens_per_task": 8640, "last_run_timestamp": "2026-10-24T11:33:58.313Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Cohere", "accuracy": 0.74, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01884005, "avg_duration_per_task_ms": 67349.35, "avg_output_tokens_per_task": 22014.7, "last_run_timestamp": "2026-10-24T11:48:47.797Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.74, "accuracy_std_dev": 0.009427854713560266, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.022918865, "avg_duration_per_task_ms": 72639.31, "avg_output_tokens_per_task": 4366.75, "last_run_timestamp": "2026-10-29T23:37:44.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7399999, "accuracy_std_dev": 0.028906131607152153, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.34949705000000003, "avg_duration_per_task_ms": 220413.082, "avg_output_tokens_per_task": 10856.716, "last_run_timestamp": "2026-10-24T12:39:24.524Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.7399998571428572, "accuracy_std_dev": 0.023094052007442304, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01886730285714286, "avg_duration_per_task_ms": 206247.68, "avg_output_tokens_per_task": 4550.928571428572, "last_run_timestamp": "2026-10-10T11:50:34.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7395835, "accuracy_std_dev": 0.022012461748973016, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.042535722500000005, "avg_duration_per_task_ms": 134957.37, "avg_output_tokens_per_task": 25971.215, "last_run_timestamp": "2026-10-23T03:54:04.118Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.7393333, "accuracy_std_dev": 0.04912106993144592, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.04842688399999999, "avg_duration_per_task_ms": 284804.346, "avg_output_tokens_per_task": 14992.81, "last_run_timestamp": "2026-10-24T12:17:04.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.7392592222222222, "accuracy_std_dev": 0.04548160272700695, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.16905808444444445, "avg_duration_per_task_ms": 289473.89555555553, "avg_output_tokens_per_task": 9765.786666666667, "last_run_timestamp": "2026-10-24T11:42:23.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7392592222222222, "accuracy_std_dev": 0.03955854406875342, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.12468993111111111, "avg_duration_per_task_ms": 675089.5488888889, "avg_output_tokens_per_task": 35068.42222222222, "last_run_timestamp": "2026-10-24T11:39:42.358Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.739166625, "accuracy_std_dev": 0.013919448327228176, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11955471250000001, "avg_duration_per_task_ms": 375155.945, "avg_output_tokens_per_task": 10031.9425, "last_run_timestamp": "2026-10-24T20:00:28.580Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.7391665000000001, "accuracy_std_dev": 0.021196743723978007, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06600346500000001, "avg_duration_per_task_ms": 259839.1925, "avg_output_tokens_per_task": 12680.6925, "last_run_timestamp": "2026-10-24T20:26:13.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.7387879090909091, "accuracy_std_dev": 0.03704407867651368, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.02604207454545454, "avg_duration_per_task_ms": 112310.20909090909, "avg_output_tokens_per_task": 9031.76, "last_run_timestamp": "2026-10-24T19:07:29.704Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.7386668, "accuracy_std_dev": 0.03166503013357166, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.009092168000000001, "avg_duration_per_task_ms": 101935.708, "avg_output_tokens_per_task": 5027.116, "last_run_timestamp": "2026-10-17T11:17:02.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7385729799331104, "accuracy_std_dev": 0.028249376510327148, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.1314999698996656, "avg_duration_per_task_ms": 63487.39464882943, "avg_output_tokens_per_task": 9641.89297658863, "last_run_timestamp": "2026-10-24T18:49:16.415Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7384824959349594, "accuracy_std_dev": 0.030529621622286764, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.012032963414634147, "avg_duration_per_task_ms": 61127.26829268293, "avg_output_tokens_per_task": 4200.292682926829, "last_run_timestamp": "2026-10-24T12:38:23.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7383335, "accuracy_std_dev": 0.0016665000000000152, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08170183999999998, "avg_duration_per_task_ms": 327224.56, "avg_output_tokens_per_task": 8775.96, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.73833325, "accuracy_std_dev": 0.017240319347027768, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.063434195, "avg_duration_per_task_ms": 520281.665, "avg_output_tokens_per_task": 14632.895, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.73833325, "accuracy_std_dev": 0.03210231473254696, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03571221, "avg_duration_per_task_ms": 388203.89, "avg_output_tokens_per_task": 8383.965, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.73833325, "accuracy_std_dev": 0.02153795745161318, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.34754801999999996, "avg_duration_per_task_ms": 197375.055, "avg_output_tokens_per_task": 8811.4275, "last_run_timestamp": "2026-10-24T12:17:34.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.7382021797752809, "accuracy_std_dev": 0.021880590311850887, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.020498339325842697, "avg_duration_per_task_ms": 307841.11235955055, "avg_output_tokens_per_task": 10994.101123595505, "last_run_timestamp": "2026-10-17T11:09:59.070Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7380952857142857, "accuracy_std_dev": 0.031414080354035444, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08033971999999999, "avg_duration_per_task_ms": 526120.5771428571, "avg_output_tokens_per_task": 11993.431428571428, "last_run_timestamp": "2026-10-24T23:08:37.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7380950401002507, "accuracy_std_dev": 0.022649397541656617, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.053091303258145366, "avg_duration_per_task_ms": 777194.3383458647, "avg_output_tokens_per_task": 14178.676691729323, "last_run_timestamp": "2026-10-24T19:40:41.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.7377776666666667, "accuracy_std_dev": 0.017777750001367207, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.014543953333333333, "avg_duration_per_task_ms": 73473.39777777778, "avg_output_tokens_per_task": 4452.611111111111, "last_run_timestamp": "2026-10-24T12:18:30.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.7375757272727272, "accuracy_std_dev": 0.02655621201797806, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07909413272727274, "avg_duration_per_task_ms": 154073.8890909091, "avg_output_tokens_per_task": 8791.48909090909, "last_run_timestamp": "2026-10-24T19:32:01.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.7375756363636364, "accuracy_std_dev": 0.03206980449035491, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3100524036363636, "avg_duration_per_task_ms": 339490.60909090907, "avg_output_tokens_per_task": 9723.709090909091, "last_run_timestamp": "2026-10-24T13:17:19.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.737500125, "accuracy_std_dev": 0.03799676247207615, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.042760110000000004, "avg_duration_per_task_ms": 594181.5325, "avg_output_tokens_per_task": 9082.065, "last_run_timestamp": "2026-10-24T19:13:45.181Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.7374243541247485, "accuracy_std_dev": 0.021992186721879213, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.34455653319919516, "avg_duration_per_task_ms": 409342.0885311871, "avg_output_tokens_per_task": 9871.545271629779, "last_run_timestamp": "2026-10-24T12:29:09.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.7373334, "accuracy_std_dev": 0.020482933819157834, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.039714615999999994, "avg_duration_per_task_ms": 174773.632, "avg_output_tokens_per_task": 27822.08, "last_run_timestamp": "2026-10-21T12:36:51.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.7371794615384615, "accuracy_std_dev": 0.04268194927847135, "run_count": 10, "total_questions": 494, "avg_cost_per_task_usd": 0.025897730769230763, "avg_duration_per_task_ms": 112126.24089068826, "avg_output_tokens_per_task": 9272.680161943319, "last_run_timestamp": "2026-10-24T19:18:47.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.7371428571428571, "accuracy_std_dev": 0.023866691370602513, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.022481400000000002, "avg_duration_per_task_ms": 144173.6, "avg_output_tokens_per_task": 16498.17142857143, "last_run_timestamp": "2026-10-24T12:33:37.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.7366890939597315, "accuracy_std_dev": 0.025312184492760705, "run_count": 15, "total_questions": 745, "avg_cost_per_task_usd": 0.0245328711409396, "avg_duration_per_task_ms": 223382.59731543623, "avg_output_tokens_per_task": 20623.554362416107, "last_run_timestamp": "2026-10-29T23:36:12.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.736667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.47698523000000004, "avg_duration_per_task_ms": 128774.34, "avg_output_tokens_per_task": 7206.5, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7366668, "accuracy_std_dev": 0.022900761755015906, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.063241744, "avg_duration_per_task_ms": 1530537.112, "avg_output_tokens_per_task": 25222.716, "last_run_timestamp": "2026-10-03T08:06:43.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.736666625, "accuracy_std_dev": 0.03018462173001967, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.16366423000000002, "avg_duration_per_task_ms": 225879.0775, "avg_output_tokens_per_task": 9618.825, "last_run_timestamp": "2026-10-24T19:37:14.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Io Net", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02539905, "avg_duration_per_task_ms": 216631.23, "avg_output_tokens_per_task": 14336.27, "last_run_timestamp": "2026-10-24T12:02:36.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.7366665, "accuracy_std_dev": 0.014529701381996819, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.057118495000000005, "avg_duration_per_task_ms": 215486.77, "avg_output_tokens_per_task": 9145.625, "last_run_timestamp": "2026-10-04T00:00:34.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.7365744652049571, "accuracy_std_dev": 0.02859181594676679, "run_count": 21, "total_questions": 1049, "avg_cost_per_task_usd": 0.13010050238322207, "avg_duration_per_task_ms": 159321.12011439467, "avg_output_tokens_per_task": 9364.21639656816, "last_run_timestamp": "2026-10-24T19:48:18.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.7365591209677419, "accuracy_std_dev": 0.016931426040354632, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.23607216935483874, "avg_duration_per_task_ms": 378114.625, "avg_output_tokens_per_task": 11505.427419354839, "last_run_timestamp": "2026-10-24T11:40:39.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.7362962222222222, "accuracy_std_dev": 0.03939194698236847, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06971035555555556, "avg_duration_per_task_ms": 294980.4511111111, "avg_output_tokens_per_task": 12708.4, "last_run_timestamp": "2026-10-24T19:51:25.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.7362762003395587, "accuracy_std_dev": 0.03269045093786308, "run_count": 12, "total_questions": 589, "avg_cost_per_task_usd": 0.06473759422750425, "avg_duration_per_task_ms": 132369.44312393887, "avg_output_tokens_per_task": 8373.801358234296, "last_run_timestamp": "2026-10-24T19:13:30.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7361904285714286, "accuracy_std_dev": 0.035964376429450086, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.024463299999999997, "avg_duration_per_task_ms": 153502.6, "avg_output_tokens_per_task": 16394.091428571428, "last_run_timestamp": "2026-10-24T11:47:20.359Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.7359999, "accuracy_std_dev": 0.025157337786220563, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.008132438, "avg_duration_per_task_ms": 403849.82, "avg_output_tokens_per_task": 14485.44, "last_run_timestamp": "2026-10-17T12:09:02.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7359998000000001, "accuracy_std_dev": 0.019596013200648757, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.3276280200000001, "avg_duration_per_task_ms": 156068.02, "avg_output_tokens_per_task": 21546.7, "last_run_timestamp": "2026-10-01T19:05:00.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.04157400066013479, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.48569376, "avg_duration_per_task_ms": 125885.67333333334, "avg_output_tokens_per_task": 7296.606666666667, "last_run_timestamp": "2026-10-02T23:08:08.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7354165, "accuracy_std_dev": 0.0429125354005097, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.006874592499999999, "avg_duration_per_task_ms": 421319.53, "avg_output_tokens_per_task": 14336.97, "last_run_timestamp": "2026-10-10T15:31:23.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.7353332, "accuracy_std_dev": 0.03370472568290686, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.033665999999999995, "avg_duration_per_task_ms": 426833.488, "avg_output_tokens_per_task": 9005.292, "last_run_timestamp": "2026-10-24T20:40:57.027Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7352380000000001, "accuracy_std_dev": 0.01698343330593503, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3052330542857143, "avg_duration_per_task_ms": 191492.84285714285, "avg_output_tokens_per_task": 10672.025714285714, "last_run_timestamp": "2026-10-01T19:05:18.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7351191205357143, "accuracy_std_dev": 0.049992352433912764, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.30342583258928574, "avg_duration_per_task_ms": 387074.1026785714, "avg_output_tokens_per_task": 11534.020089285714, "last_run_timestamp": "2026-10-24T13:33:47.625Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.735, "accuracy_std_dev": 0.026822730826670128, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012519140000000002, "avg_duration_per_task_ms": 95304.945, "avg_output_tokens_per_task": 10824.09, "last_run_timestamp": "2026-10-24T11:50:10.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.735, "accuracy_std_dev": 0.03227493867926013, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0975708025, "avg_duration_per_task_ms": 334496.3, "avg_output_tokens_per_task": 11110.8, "last_run_timestamp": "2026-10-24T20:22:47.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.734694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.25339910204081634, "avg_duration_per_task_ms": 146531.16326530612, "avg_output_tokens_per_task": 29831.102040816328, "last_run_timestamp": "2026-10-01T08:03:06.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.73458325, "accuracy_std_dev": 0.027785801480927293, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0151816525, "avg_duration_per_task_ms": 268037.2675, "avg_output_tokens_per_task": 21770.385, "last_run_timestamp": "2026-10-24T12:56:13.097Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.73416675, "accuracy_std_dev": 0.015789390779491794, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07111907, "avg_duration_per_task_ms": 230769.835, "avg_output_tokens_per_task": 13083.0325, "last_run_timestamp": "2026-10-24T19:41:01.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Mistral", "accuracy": 0.73416675, "accuracy_std_dev": 0.033530506426051786, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02004302, "avg_duration_per_task_ms": 88766.04, "avg_output_tokens_per_task": 5752.91, "last_run_timestamp": "2026-10-30T16:30:54.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.7341665, "accuracy_std_dev": 0.02066601477668108, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.012598340000000001, "avg_duration_per_task_ms": 74617.86, "avg_output_tokens_per_task": 10550.06, "last_run_timestamp": "2026-10-24T11:45:42.778Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.7340742222222222, "accuracy_std_dev": 0.027296881245453555, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.017404215555555554, "avg_duration_per_task_ms": 575071.7644444444, "avg_output_tokens_per_task": 18088.03111111111, "last_run_timestamp": "2026-10-24T11:35:52.428Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7340741111111111, "accuracy_std_dev": 0.03257589639747104, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05069578000000001, "avg_duration_per_task_ms": 248940.97333333333, "avg_output_tokens_per_task": 10685.184444444445, "last_run_timestamp": "2026-10-27T01:22:44.294Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.7339394545454545, "accuracy_std_dev": 0.03041204391613666, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.026801145454545455, "avg_duration_per_task_ms": 98249, "avg_output_tokens_per_task": 9318.925454545455, "last_run_timestamp": "2026-10-24T19:08:17.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7339181879699248, "accuracy_std_dev": 0.03240656404548892, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.06522364912280701, "avg_duration_per_task_ms": 152721.04010025063, "avg_output_tokens_per_task": 11337.393483709273, "last_run_timestamp": "2026-10-24T19:41:50.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7338889166666667, "accuracy_std_dev": 0.03382511166391603, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.3533906716666666, "avg_duration_per_task_ms": 207450.37666666668, "avg_output_tokens_per_task": 10346.453333333333, "last_run_timestamp": "2026-10-24T12:18:44.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.733796, "accuracy_std_dev": 0.0059017228557995535, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.20467558333333333, "avg_duration_per_task_ms": 370954.7361111111, "avg_output_tokens_per_task": 23212.9375, "last_run_timestamp": "2026-10-24T19:18:46.938Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Ionstream", "accuracy": 0.7333335, "accuracy_std_dev": 0.033333500000000016, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018280730000000002, "avg_duration_per_task_ms": 88642.31, "avg_output_tokens_per_task": 11899.17, "last_run_timestamp": "2026-10-24T11:54:38.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.7333335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12301775, "avg_duration_per_task_ms": 155307.55, "avg_output_tokens_per_task": 18812.75, "last_run_timestamp": "2026-10-01T19:05:47.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08394800999999999, "avg_duration_per_task_ms": 134148.76, "avg_output_tokens_per_task": 8723.94, "last_run_timestamp": "2026-10-17T19:55:39.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.047618709999999995, "avg_duration_per_task_ms": 115029.78, "avg_output_tokens_per_task": 15522.32, "last_run_timestamp": "2026-10-24T12:31:48.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7333335, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12687428, "avg_duration_per_task_ms": 110480.49, "avg_output_tokens_per_task": 5152.54, "last_run_timestamp": "2026-10-01T19:04:55.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.733333375, "accuracy_std_dev": 0.027688655921412557, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05093239000000001, "avg_duration_per_task_ms": 359484.675, "avg_output_tokens_per_task": 10187.6175, "last_run_timestamp": "2026-10-24T20:19:38.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008480786666666665, "avg_duration_per_task_ms": 397846.0266666667, "avg_output_tokens_per_task": 19127.326666666668, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.05760637824832321, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04898614, "avg_duration_per_task_ms": 106829.63333333333, "avg_output_tokens_per_task": 9897.326666666666, "last_run_timestamp": "2026-10-10T11:51:48.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.73333325, "accuracy_std_dev": 0.02357013176007932, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06276583000000001, "avg_duration_per_task_ms": 510113.315, "avg_output_tokens_per_task": 14674.185, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.73333325, "accuracy_std_dev": 0.02357013176007932, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04045966999999999, "avg_duration_per_task_ms": 165902.775, "avg_output_tokens_per_task": 35276.36, "last_run_timestamp": "2026-10-24T12:26:22.907Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.7333332, "accuracy_std_dev": 0.026499341711823724, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.29219796600000003, "avg_duration_per_task_ms": 358228.216, "avg_output_tokens_per_task": 10539.826, "last_run_timestamp": "2026-10-24T12:45:08.706Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Particle AI", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.027065799999999997, "avg_duration_per_task_ms": 71689.04, "avg_output_tokens_per_task": 9149.63, "last_run_timestamp": "2026-10-28T22:54:17.860Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01490002, "avg_duration_per_task_ms": 87918.52, "avg_output_tokens_per_task": 12302.3, "last_run_timestamp": "2026-10-24T11:33:56.850Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12365041999999998, "avg_duration_per_task_ms": 156560.04, "avg_output_tokens_per_task": 19704.13, "last_run_timestamp": "2026-10-01T19:04:26.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10928178, "avg_duration_per_task_ms": 75479.44, "avg_output_tokens_per_task": 18824.05, "last_run_timestamp": "2026-10-01T19:05:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.038872460000000005, "avg_duration_per_task_ms": 158714.4, "avg_output_tokens_per_task": 14542.76, "last_run_timestamp": "2026-10-24T13:34:38.188Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02595887, "avg_duration_per_task_ms": 77664, "avg_output_tokens_per_task": 16596.59, "last_run_timestamp": "2026-10-24T12:17:54.586Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.7332197098976109, "accuracy_std_dev": 0.022906695901829432, "run_count": 6, "total_questions": 293, "avg_cost_per_task_usd": 0.04657865870307167, "avg_duration_per_task_ms": 71132.01023890785, "avg_output_tokens_per_task": 19231.02389078498, "last_run_timestamp": "2026-10-24T12:19:39.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.73291675, "accuracy_std_dev": 0.033600323453614245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08912411, "avg_duration_per_task_ms": 173126.64, "avg_output_tokens_per_task": 10513.8925, "last_run_timestamp": "2026-10-24T19:53:44.150Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.7327272727272728, "accuracy_std_dev": 0.025098504195164927, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.36473022363636365, "avg_duration_per_task_ms": 228969.27363636365, "avg_output_tokens_per_task": 10924.667272727273, "last_run_timestamp": "2026-10-24T12:16:43.052Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7327272727272728, "accuracy_std_dev": 0.06263266295137199, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.15801903636363634, "avg_duration_per_task_ms": 1429178.5054545454, "avg_output_tokens_per_task": 28482.28181818182, "last_run_timestamp": "2026-10-24T20:45:53.033Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.7326668, "accuracy_std_dev": 0.024801517876130098, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045864716, "avg_duration_per_task_ms": 120831.192, "avg_output_tokens_per_task": 26031.788, "last_run_timestamp": "2026-10-29T23:56:26.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7325, "accuracy_std_dev": 0.04596937990662913, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.012367795, "avg_duration_per_task_ms": 953069.875, "avg_output_tokens_per_task": 22737, "last_run_timestamp": "2026-10-17T10:32:07.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.7325, "accuracy_std_dev": 0.029474565306379014, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08144697000000001, "avg_duration_per_task_ms": 228561.56, "avg_output_tokens_per_task": 29757.14, "last_run_timestamp": "2026-10-24T12:23:33.838Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.7315151818181818, "accuracy_std_dev": 0.012091042761393707, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.23347585818181818, "avg_duration_per_task_ms": 257016.06181818183, "avg_output_tokens_per_task": 10120.247272727273, "last_run_timestamp": "2026-10-24T12:33:16.806Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.7315151818181818, "accuracy_std_dev": 0.04321351479007923, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08041129999999999, "avg_duration_per_task_ms": 162692.34181818183, "avg_output_tokens_per_task": 8235.127272727274, "last_run_timestamp": "2026-10-24T19:35:51.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7311113333333332, "accuracy_std_dev": 0.0332591470759883, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03472362, "avg_duration_per_task_ms": 267399.06, "avg_output_tokens_per_task": 33925.37333333334, "last_run_timestamp": "2026-10-23T07:22:37.672Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.7311111666666666, "accuracy_std_dev": 0.03022542058387644, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05052855666666667, "avg_duration_per_task_ms": 436577.48333333334, "avg_output_tokens_per_task": 9693.223333333333, "last_run_timestamp": "2026-10-24T20:04:11.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12928292, "avg_duration_per_task_ms": 109953.54, "avg_output_tokens_per_task": 5131.753333333333, "last_run_timestamp": "2026-10-02T23:08:37.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.7309522857142857, "accuracy_std_dev": 0.01941313112092864, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04753026285714287, "avg_duration_per_task_ms": 161306.4342857143, "avg_output_tokens_per_task": 28780.68857142857, "last_run_timestamp": "2026-10-24T12:41:08.949Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Nebius", "accuracy": 0.7306399292929292, "accuracy_std_dev": 0.010748499999999994, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.08618277777777777, "avg_duration_per_task_ms": 204214.55555555556, "avg_output_tokens_per_task": 8369.424242424242, "last_run_timestamp": "2026-10-24T12:28:29.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.7305555, "accuracy_std_dev": 0.026346395026960852, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06789053999999999, "avg_duration_per_task_ms": 79373.93333333333, "avg_output_tokens_per_task": 8526.28, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7300001666666667, "accuracy_std_dev": 0.03605552816336059, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11662036, "avg_duration_per_task_ms": 82058.93666666666, "avg_output_tokens_per_task": 9160.97, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Crusoe", "accuracy": 0.73, "accuracy_std_dev": 0.04333300000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.023320379999999998, "avg_duration_per_task_ms": 139023.54, "avg_output_tokens_per_task": 12073.4, "last_run_timestamp": "2026-10-24T12:04:17.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Novita", "accuracy": 0.73, "accuracy_std_dev": 0.023332999999999993, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0186985, "avg_duration_per_task_ms": 114277.39, "avg_output_tokens_per_task": 11475.52, "last_run_timestamp": "2026-10-24T11:36:27.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.73, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12856867000000002, "avg_duration_per_task_ms": 121477.86, "avg_output_tokens_per_task": 19799.68, "last_run_timestamp": "2026-10-01T19:05:00.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.73, "accuracy_std_dev": 0.04333300000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.24468226999999998, "avg_duration_per_task_ms": 158524.06, "avg_output_tokens_per_task": 22552.35, "last_run_timestamp": "2026-10-01T19:04:55.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cohere", "accuracy": 0.73, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02344918, "avg_duration_per_task_ms": 35583.1, "avg_output_tokens_per_task": 4177.41, "last_run_timestamp": "2026-10-24T11:37:33.416Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.729931893877551, "accuracy_std_dev": 0.03636312637494197, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.018753575510204082, "avg_duration_per_task_ms": 310156.1551020408, "avg_output_tokens_per_task": 6681.187755102041, "last_run_timestamp": "2026-10-09T02:01:31.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.7296969090909091, "accuracy_std_dev": 0.03079598974617119, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3506678872727273, "avg_duration_per_task_ms": 233835.19090909092, "avg_output_tokens_per_task": 10922.210909090909, "last_run_timestamp": "2026-10-24T11:45:41.975Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.7295238571428572, "accuracy_std_dev": 0.035920148907695515, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07603518571428572, "avg_duration_per_task_ms": 216305.50285714286, "avg_output_tokens_per_task": 11169.142857142857, "last_run_timestamp": "2026-10-24T19:35:24.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7292263868194842, "accuracy_std_dev": 0.037395839263812465, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.07898853868194843, "avg_duration_per_task_ms": 295613.8395415473, "avg_output_tokens_per_task": 13730.816618911174, "last_run_timestamp": "2026-09-26T08:12:00.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.729166625, "accuracy_std_dev": 0.029237916546573137, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09863991500000001, "avg_duration_per_task_ms": 220840.6075, "avg_output_tokens_per_task": 12590.2025, "last_run_timestamp": "2026-10-24T19:51:30.269Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "ModelRun", "accuracy": 0.7288890000000001, "accuracy_std_dev": 0.03142665378305492, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02386675333333333, "avg_duration_per_task_ms": 151228.52, "avg_output_tokens_per_task": 22554.02666666667, "last_run_timestamp": "2026-10-28T06:40:49.113Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.7285714285714285, "accuracy_std_dev": 0.026598516660344282, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04723099714285714, "avg_duration_per_task_ms": 142250.88571428572, "avg_output_tokens_per_task": 24794.32857142857, "last_run_timestamp": "2026-10-24T13:07:38.963Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.7285712857142858, "accuracy_std_dev": 0.024871537523352083, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03253488857142857, "avg_duration_per_task_ms": 92457.76285714285, "avg_output_tokens_per_task": 23099.80285714286, "last_run_timestamp": "2026-10-24T11:48:04.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.7284848181818182, "accuracy_std_dev": 0.03400415574554003, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3294088909090909, "avg_duration_per_task_ms": 208197.42363636364, "avg_output_tokens_per_task": 10397.776363636363, "last_run_timestamp": "2026-10-24T12:22:38.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.7284848181818182, "accuracy_std_dev": 0.02941747886057535, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08185737454545454, "avg_duration_per_task_ms": 229135.09818181818, "avg_output_tokens_per_task": 8804.125454545454, "last_run_timestamp": "2026-10-24T19:45:17.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimeIntellect", "accuracy": 0.72833325, "accuracy_std_dev": 0.02640480280531367, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02797866, "avg_duration_per_task_ms": 99430.655, "avg_output_tokens_per_task": 13207.28, "last_run_timestamp": "2026-10-29T23:09:25.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.72833325, "accuracy_std_dev": 0.03411588502717615, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.055064995000000005, "avg_duration_per_task_ms": 291890.35, "avg_output_tokens_per_task": 8526.425, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.7281586250000001, "accuracy_std_dev": 0.032945022045826604, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.10040358467741935, "avg_duration_per_task_ms": 94375.23588709677, "avg_output_tokens_per_task": 7576.909274193548, "last_run_timestamp": "2026-09-26T08:26:47.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.7278787272727273, "accuracy_std_dev": 0.02618002940164649, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08620709454545455, "avg_duration_per_task_ms": 694656.5654545454, "avg_output_tokens_per_task": 9307.56, "last_run_timestamp": "2026-10-24T19:18:30.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.7278287449541284, "accuracy_std_dev": 0.03216567357904057, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.07681454862385322, "avg_duration_per_task_ms": 195964.3486238532, "avg_output_tokens_per_task": 8232.398165137614, "last_run_timestamp": "2026-10-28T03:37:10.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Makora", "accuracy": 0.7276631288659794, "accuracy_std_dev": 0.019224116824642454, "run_count": 4, "total_questions": 194, "avg_cost_per_task_usd": 0.07521926804123712, "avg_duration_per_task_ms": 73859.24742268042, "avg_output_tokens_per_task": 8316.994845360825, "last_run_timestamp": "2026-10-24T11:42:23.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7275001249999999, "accuracy_std_dev": 0.04047472891335252, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.10270093000000001, "avg_duration_per_task_ms": 213296.5425, "avg_output_tokens_per_task": 11976.1525, "last_run_timestamp": "2026-10-24T20:41:03.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7275, "accuracy_std_dev": 0.022220636056377873, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06640188, "avg_duration_per_task_ms": 199646.51, "avg_output_tokens_per_task": 9529.36, "last_run_timestamp": "2026-10-24T19:34:44.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.7274752368672397, "accuracy_std_dev": 0.02732433470092314, "run_count": 21, "total_questions": 1047, "avg_cost_per_task_usd": 0.4330738662846227, "avg_duration_per_task_ms": 236263.65329512893, "avg_output_tokens_per_task": 10771.178605539637, "last_run_timestamp": "2026-10-24T12:16:44.774Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7273502461538462, "accuracy_std_dev": 0.043866981805083166, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.07634109230769232, "avg_duration_per_task_ms": 620581.2615384615, "avg_output_tokens_per_task": 7602.702564102564, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7272047264573992, "accuracy_std_dev": 0.08169417811313268, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.009946286995515695, "avg_duration_per_task_ms": 180161.19058295965, "avg_output_tokens_per_task": 4234.017937219731, "last_run_timestamp": "2026-10-24T12:25:45.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.7271428571428571, "accuracy_std_dev": 0.01855313018202104, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.31224118, "avg_duration_per_task_ms": 126481.89142857143, "avg_output_tokens_per_task": 5238.86, "last_run_timestamp": "2026-10-01T19:05:16.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.21341358, "avg_duration_per_task_ms": 101630.84, "avg_output_tokens_per_task": 8827.48, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 2.91364046, "avg_duration_per_task_ms": 331183.04, "avg_output_tokens_per_task": 12422.4, "last_run_timestamp": "2026-10-01T08:02:52.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05448868, "avg_duration_per_task_ms": 350679.6, "avg_output_tokens_per_task": 17629, "last_run_timestamp": "2026-10-01T08:04:38.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.047453712340436424, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.011438460000000001, "avg_duration_per_task_ms": 86937.99333333333, "avg_output_tokens_per_task": 10682.4, "last_run_timestamp": "2026-10-24T11:51:31.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Nebius", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02419578, "avg_duration_per_task_ms": 133541.80666666667, "avg_output_tokens_per_task": 11460.326666666666, "last_run_timestamp": "2026-10-24T12:24:12.065Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.30249026, "avg_duration_per_task_ms": 183822.13333333333, "avg_output_tokens_per_task": 10747.233333333334, "last_run_timestamp": "2026-10-02T23:13:07.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013071513333333333, "avg_duration_per_task_ms": 97597.57333333333, "avg_output_tokens_per_task": 4935.306666666666, "last_run_timestamp": "2026-10-02T23:07:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Phala", "accuracy": 0.7266665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01900955, "avg_duration_per_task_ms": 214869.47, "avg_output_tokens_per_task": 11727.86, "last_run_timestamp": "2026-10-24T12:08:01.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Modal", "accuracy": 0.7266665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018476000000000003, "avg_duration_per_task_ms": 104934.72, "avg_output_tokens_per_task": 10917.83, "last_run_timestamp": "2026-10-24T11:48:43.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7266665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.43167194999999997, "avg_duration_per_task_ms": 195075.98, "avg_output_tokens_per_task": 31329.14, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Io Net", "accuracy": 0.7266665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06725601, "avg_duration_per_task_ms": 640810.38, "avg_output_tokens_per_task": 15599.02, "last_run_timestamp": "2026-10-27T14:13:59.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7266665, "accuracy_std_dev": 0.036055327855256024, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09256564750000001, "avg_duration_per_task_ms": 228748.9725, "avg_output_tokens_per_task": 10105.375, "last_run_timestamp": "2026-10-24T20:31:27.967Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Nebius", "accuracy": 0.7263158421052631, "accuracy_std_dev": 0.007406999999999997, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.013109768421052631, "avg_duration_per_task_ms": 155953.90526315788, "avg_output_tokens_per_task": 8164.357894736842, "last_run_timestamp": "2026-10-24T13:07:13.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.7262383654618474, "accuracy_std_dev": 0.017933902883644697, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.17520732931726907, "avg_duration_per_task_ms": 546387.9477911646, "avg_output_tokens_per_task": 8780.429718875503, "last_run_timestamp": "2026-10-24T21:19:54.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7253334, "accuracy_std_dev": 0.039305065549366525, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.00982467, "avg_duration_per_task_ms": 372894.758, "avg_output_tokens_per_task": 14244.68, "last_run_timestamp": "2026-10-17T13:13:33.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.7253334, "accuracy_std_dev": 0.037214139488103176, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.08614405199999998, "avg_duration_per_task_ms": 329579.82, "avg_output_tokens_per_task": 9375.922, "last_run_timestamp": "2026-10-24T19:37:38.817Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.7253333, "accuracy_std_dev": 0.02809501972610092, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.028140498, "avg_duration_per_task_ms": 228402.532, "avg_output_tokens_per_task": 9725.512, "last_run_timestamp": "2026-10-24T19:14:23.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.7244443333333334, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12152117999999999, "avg_duration_per_task_ms": 471173.63333333336, "avg_output_tokens_per_task": 54771.27333333333, "last_run_timestamp": "2026-10-02T23:27:09.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.7242855714285714, "accuracy_std_dev": 0.024734242792748124, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04387056285714286, "avg_duration_per_task_ms": 165104.78285714285, "avg_output_tokens_per_task": 29616.194285714286, "last_run_timestamp": "2026-10-24T12:35:19.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02063872, "avg_duration_per_task_ms": 638594.61, "avg_output_tokens_per_task": 13867.61, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7233335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11131322, "avg_duration_per_task_ms": 83286.18, "avg_output_tokens_per_task": 3736.91, "last_run_timestamp": "2026-10-01T19:04:40.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Particle AI", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0109795, "avg_duration_per_task_ms": 64854.76, "avg_output_tokens_per_task": 4155.27, "last_run_timestamp": "2026-10-28T23:08:11.828Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.72333325, "accuracy_std_dev": 0.024267113092705673, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07263319833333333, "avg_duration_per_task_ms": 224600.79833333334, "avg_output_tokens_per_task": 8546.035, "last_run_timestamp": "2026-10-24T19:52:00.596Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33860155, "avg_duration_per_task_ms": 113561.71, "avg_output_tokens_per_task": 10711.4, "last_run_timestamp": "2026-10-01T19:04:58.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7232361605839416, "accuracy_std_dev": 0.08188802109119138, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.04157626824817519, "avg_duration_per_task_ms": 270001.6879562044, "avg_output_tokens_per_task": 23794.242700729927, "last_run_timestamp": "2026-10-24T19:21:10.232Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.722963, "accuracy_std_dev": 0.024567656927124495, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.2753419711111111, "avg_duration_per_task_ms": 183403.79555555555, "avg_output_tokens_per_task": 11487.52, "last_run_timestamp": "2026-10-24T12:33:54.727Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.7229628888888889, "accuracy_std_dev": 0.0711188714906161, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.009601128888888888, "avg_duration_per_task_ms": 242853.24666666667, "avg_output_tokens_per_task": 7420.495555555555, "last_run_timestamp": "2026-10-27T05:56:39.952Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7228571428571429, "accuracy_std_dev": 0.019471637611287507, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10722213142857143, "avg_duration_per_task_ms": 326175.72, "avg_output_tokens_per_task": 14461.362857142858, "last_run_timestamp": "2026-10-17T19:16:53.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.7223148931552588, "accuracy_std_dev": 0.03960298431691171, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.025822385642737894, "avg_duration_per_task_ms": 283083.48080133554, "avg_output_tokens_per_task": 7842.495826377295, "last_run_timestamp": "2026-10-24T12:07:12.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.011330936570096713, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018792733333333332, "avg_duration_per_task_ms": 301192.17333333334, "avg_output_tokens_per_task": 19692.3, "last_run_timestamp": "2026-10-01T16:00:06.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Reka", "accuracy": 0.72166675, "accuracy_std_dev": 0.012801898733683979, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01389351, "avg_duration_per_task_ms": 148770.74, "avg_output_tokens_per_task": 13265.9, "last_run_timestamp": "2026-10-24T12:22:53.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.721666625, "accuracy_std_dev": 0.038405766711971455, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.074211325, "avg_duration_per_task_ms": 267437.405, "avg_output_tokens_per_task": 27915.47, "last_run_timestamp": "2026-10-24T18:56:04.577Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.721666625, "accuracy_std_dev": 0.02466430450538541, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.12928377000000002, "avg_duration_per_task_ms": 250878.7025, "avg_output_tokens_per_task": 10320.3, "last_run_timestamp": "2026-10-24T22:56:39.524Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.7210882448979592, "accuracy_std_dev": 0.0076780598851532815, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.009489710204081632, "avg_duration_per_task_ms": 145499.9755102041, "avg_output_tokens_per_task": 16260.81224489796, "last_run_timestamp": "2026-09-26T08:06:43.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.720606, "accuracy_std_dev": 0.021168874530996952, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08199455272727274, "avg_duration_per_task_ms": 233558.22181818183, "avg_output_tokens_per_task": 9247.22, "last_run_timestamp": "2026-10-24T19:37:32.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.720606, "accuracy_std_dev": 0.033928552923084485, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08656796545454544, "avg_duration_per_task_ms": 136294.9909090909, "avg_output_tokens_per_task": 9196.996363636363, "last_run_timestamp": "2026-10-24T18:58:26.138Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Sail Research", "accuracy": 0.7200002499999999, "accuracy_std_dev": 0.016329931619192377, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.005028675, "avg_duration_per_task_ms": 140463.585, "avg_output_tokens_per_task": 11883.17, "last_run_timestamp": "2026-10-24T12:43:45.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.72, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.026063740000000002, "avg_duration_per_task_ms": 127597.92, "avg_output_tokens_per_task": 12968.38, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Io Net", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12671924, "avg_duration_per_task_ms": 566714.78, "avg_output_tokens_per_task": 16714.02, "last_run_timestamp": "2026-09-28T18:00:05.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Venice", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.027683759999999998, "avg_duration_per_task_ms": 68973.72, "avg_output_tokens_per_task": 10080.22, "last_run_timestamp": "2026-10-29T00:22:34.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StreamLake", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0210898, "avg_duration_per_task_ms": 122128, "avg_output_tokens_per_task": 11448.46, "last_run_timestamp": "2026-10-26T03:28:50.420Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00317077, "avg_duration_per_task_ms": 128719.23, "avg_output_tokens_per_task": 15118.03, "last_run_timestamp": "2026-10-01T08:00:27.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "GMICloud", "accuracy": 0.72, "accuracy_std_dev": 0.016329931618554536, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01825590666666667, "avg_duration_per_task_ms": 233637.58, "avg_output_tokens_per_task": 15649.073333333334, "last_run_timestamp": "2026-10-27T01:30:02.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15154258, "avg_duration_per_task_ms": 105755.46, "avg_output_tokens_per_task": 13133.32, "last_run_timestamp": "2026-08-04T03:12:01.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.402901, "avg_duration_per_task_ms": 196016.4, "avg_output_tokens_per_task": 16256.52, "last_run_timestamp": "2026-09-27T00:00:03.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.028463379999999996, "avg_duration_per_task_ms": 583244.64, "avg_output_tokens_per_task": 43866.54, "last_run_timestamp": "2026-08-08T08:27:52.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.72, "accuracy_std_dev": 0.014142017775055995, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05195973499999999, "avg_duration_per_task_ms": 385642.415, "avg_output_tokens_per_task": 9384.3275, "last_run_timestamp": "2026-10-24T20:09:28.564Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": 0.020816740060585853, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.058833675, "avg_duration_per_task_ms": 335844.4, "avg_output_tokens_per_task": 10101.9275, "last_run_timestamp": "2026-10-24T20:09:59.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.7199998461538462, "accuracy_std_dev": 0.060595320121208315, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.28468824153846156, "avg_duration_per_task_ms": 424519.81384615385, "avg_output_tokens_per_task": 10050.18923076923, "last_run_timestamp": "2026-10-24T12:30:07.991Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "BaseTen", "accuracy": 0.719858, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.0657194255319149, "avg_duration_per_task_ms": 49852.255319148935, "avg_output_tokens_per_task": 5715.191489361702, "last_run_timestamp": "2026-10-24T12:16:29.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.719770172413793, "accuracy_std_dev": 0.03281128869666338, "run_count": 29, "total_questions": 1450, "avg_cost_per_task_usd": 0.11837045241379313, "avg_duration_per_task_ms": 130253.44965517242, "avg_output_tokens_per_task": 8577.781379310345, "last_run_timestamp": "2026-10-28T04:07:30.740Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.7193332, "accuracy_std_dev": 0.03143937799893632, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045152332, "avg_duration_per_task_ms": 326196.928, "avg_output_tokens_per_task": 13427.836, "last_run_timestamp": "2026-10-04T00:00:34.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.7192592777777778, "accuracy_std_dev": 0.027923028635887925, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.06975042666666668, "avg_duration_per_task_ms": 117431.64666666667, "avg_output_tokens_per_task": 8825.745555555555, "last_run_timestamp": "2026-10-24T19:51:31.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.7191665, "accuracy_std_dev": 0.032393007648256436, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05382420250000001, "avg_duration_per_task_ms": 373003.49, "avg_output_tokens_per_task": 11827.6175, "last_run_timestamp": "2026-10-24T20:11:31.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.7187878181818182, "accuracy_std_dev": 0.02571315171028306, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.04521527272727272, "avg_duration_per_task_ms": 179484.99272727274, "avg_output_tokens_per_task": 9729.592727272728, "last_run_timestamp": "2026-10-24T19:18:28.440Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Darkbloom", "accuracy": 0.7187496874999999, "accuracy_std_dev": 0.035822999999999994, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.015383489583333333, "avg_duration_per_task_ms": 816064.3020833334, "avg_output_tokens_per_task": 9819.75, "last_run_timestamp": "2026-10-30T05:52:34.884Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7185183333333334, "accuracy_std_dev": 0.033815486586933986, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.24051653333333337, "avg_duration_per_task_ms": 300854.7, "avg_output_tokens_per_task": 21246.093333333334, "last_run_timestamp": "2026-10-24T12:24:27.575Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7180951428571429, "accuracy_std_dev": 0.022737622035664484, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09258931428571428, "avg_duration_per_task_ms": 898101.5942857143, "avg_output_tokens_per_task": 9610.791428571429, "last_run_timestamp": "2026-10-24T20:25:57.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.7179166875, "accuracy_std_dev": 0.01925252734291899, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.038559541249999996, "avg_duration_per_task_ms": 195166.50625, "avg_output_tokens_per_task": 9795.555, "last_run_timestamp": "2026-10-24T19:52:24.184Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7177776666666666, "accuracy_std_dev": 0.017497908167041637, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.057615893333333335, "avg_duration_per_task_ms": 509994.74666666664, "avg_output_tokens_per_task": 13843.966666666667, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.7177776666666666, "accuracy_std_dev": 0.013698571612479267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10180220666666666, "avg_duration_per_task_ms": 128107.32666666666, "avg_output_tokens_per_task": 16602.8, "last_run_timestamp": "2026-10-24T12:39:45.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.717499875, "accuracy_std_dev": 0.03196146008256467, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09209893, "avg_duration_per_task_ms": 157130.905, "avg_output_tokens_per_task": 48565.055, "last_run_timestamp": "2026-10-01T19:06:49.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7167414529147983, "accuracy_std_dev": 0.04866792401901878, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.034667679372197314, "avg_duration_per_task_ms": 1113044.8408071748, "avg_output_tokens_per_task": 50703.41479820628, "last_run_timestamp": "2026-10-17T19:19:46.010Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "SambaNova", "accuracy": 0.7166669999999999, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04331858, "avg_duration_per_task_ms": 119875.46, "avg_output_tokens_per_task": 9185.81, "last_run_timestamp": "2026-10-24T12:22:21.794Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7166669999999999, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1280142, "avg_duration_per_task_ms": 531647.04, "avg_output_tokens_per_task": 56970.52, "last_run_timestamp": "2026-10-01T19:04:42.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.71669186, "avg_duration_per_task_ms": 277432.68, "avg_output_tokens_per_task": 22919.22, "last_run_timestamp": "2026-10-02T23:08:29.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.716250125, "accuracy_std_dev": 0.028550118578727003, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2090713325, "avg_duration_per_task_ms": 1406671.1525, "avg_output_tokens_per_task": 13817.695, "last_run_timestamp": "2026-10-24T19:32:17.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7155556666666666, "accuracy_std_dev": 0.03651479659596153, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3018651377777778, "avg_duration_per_task_ms": 498139.0266666667, "avg_output_tokens_per_task": 9517.055555555555, "last_run_timestamp": "2026-10-24T22:09:08.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7155553333333334, "accuracy_std_dev": 0.029979285029648195, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13937678, "avg_duration_per_task_ms": 263819.3466666667, "avg_output_tokens_per_task": 10311.9, "last_run_timestamp": "2026-10-03T16:18:50.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.715278, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.009582875, "avg_duration_per_task_ms": 164124.5, "avg_output_tokens_per_task": 18445.333333333332, "last_run_timestamp": "2026-10-10T11:44:02.317Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.7144445, "accuracy_std_dev": 0.03942493727431069, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08227690833333334, "avg_duration_per_task_ms": 100916.12166666667, "avg_output_tokens_per_task": 8631.778333333334, "last_run_timestamp": "2026-10-24T23:37:48.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.7139393636363637, "accuracy_std_dev": 0.03259241406516308, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.04342848545454545, "avg_duration_per_task_ms": 118501.67636363636, "avg_output_tokens_per_task": 8446.745454545455, "last_run_timestamp": "2026-10-24T19:43:58.683Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.71333325, "accuracy_std_dev": 0.016996797086142445, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01124519, "avg_duration_per_task_ms": 291598.365, "avg_output_tokens_per_task": 13318.715, "last_run_timestamp": "2026-10-24T13:02:30.014Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.71333325, "accuracy_std_dev": 0.023570237826282113, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0612510275, "avg_duration_per_task_ms": 315642.135, "avg_output_tokens_per_task": 9841.58, "last_run_timestamp": "2026-10-24T20:27:01.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7133330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12082516, "avg_duration_per_task_ms": 865226.27, "avg_output_tokens_per_task": 27649.46, "last_run_timestamp": "2026-08-15T08:15:24.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Crusoe", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01217508, "avg_duration_per_task_ms": 145478.36, "avg_output_tokens_per_task": 13792.14, "last_run_timestamp": "2026-10-26T04:07:05.258Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Crusoe", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14050459999999998, "avg_duration_per_task_ms": 414099.52, "avg_output_tokens_per_task": 22497.98, "last_run_timestamp": "2026-10-26T05:10:12.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05452906, "avg_duration_per_task_ms": 272872.18, "avg_output_tokens_per_task": 39719.44, "last_run_timestamp": "2026-10-13T20:01:29.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08007529, "avg_duration_per_task_ms": 140906.22, "avg_output_tokens_per_task": 42659.23, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.006318460000000001, "avg_duration_per_task_ms": 234030.92, "avg_output_tokens_per_task": 14773.36, "last_run_timestamp": "2026-10-01T08:05:57.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.713333, "accuracy_std_dev": 0.024494897427831803, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.046480935, "avg_duration_per_task_ms": 164672.7, "avg_output_tokens_per_task": 8116.895, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Nebius", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07153616, "avg_duration_per_task_ms": 149810.8, "avg_output_tokens_per_task": 15322.1, "last_run_timestamp": "2026-10-27T00:56:39.531Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01665106, "avg_duration_per_task_ms": 327560.6, "avg_output_tokens_per_task": 17227.24, "last_run_timestamp": "2026-10-01T08:00:41.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.712766, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.21022357446808512, "avg_duration_per_task_ms": 261220.25531914894, "avg_output_tokens_per_task": 27609.17021276596, "last_run_timestamp": "2026-10-01T08:02:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.7125925555555556, "accuracy_std_dev": 0.02047446802516685, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.023023279999999997, "avg_duration_per_task_ms": 256611.78222222222, "avg_output_tokens_per_task": 9809.96, "last_run_timestamp": "2026-10-27T04:17:23.321Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.712381, "accuracy_std_dev": 0.016106167044245584, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11192505714285715, "avg_duration_per_task_ms": 96202.06, "avg_output_tokens_per_task": 3694.017142857143, "last_run_timestamp": "2026-10-01T19:04:52.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.71166675, "accuracy_std_dev": 0.03411566518752785, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.025674655, "avg_duration_per_task_ms": 206861.205, "avg_output_tokens_per_task": 9942.77, "last_run_timestamp": "2026-10-24T19:49:21.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.71166675, "accuracy_std_dev": 0.03460084697211182, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.072092705, "avg_duration_per_task_ms": 320824.8375, "avg_output_tokens_per_task": 11065.2975, "last_run_timestamp": "2026-10-24T22:01:31.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7114285238095238, "accuracy_std_dev": 0.02603554463314856, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.1029916761904762, "avg_duration_per_task_ms": 108696.98, "avg_output_tokens_per_task": 9711.373333333333, "last_run_timestamp": "2026-10-29T23:00:08.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.7113332, "accuracy_std_dev": 0.017587783639788142, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.28712958000000005, "avg_duration_per_task_ms": 155529.044, "avg_output_tokens_per_task": 10850.46, "last_run_timestamp": "2026-10-24T19:05:28.397Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7108432289156627, "accuracy_std_dev": 0.016820726607373437, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.04982926104417671, "avg_duration_per_task_ms": 412025.6064257028, "avg_output_tokens_per_task": 8776.425702811244, "last_run_timestamp": "2026-10-24T21:22:48.118Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7104167499999999, "accuracy_std_dev": 0.11449707753448339, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.22292825749999998, "avg_duration_per_task_ms": 1269690.4575, "avg_output_tokens_per_task": 60617.9, "last_run_timestamp": "2026-10-10T19:16:51.412Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7103030909090909, "accuracy_std_dev": 0.03170096156687463, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.12382668181818182, "avg_duration_per_task_ms": 164491.53272727272, "avg_output_tokens_per_task": 11816.025454545455, "last_run_timestamp": "2026-10-24T19:52:24.144Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.710303, "accuracy_std_dev": 0.045426193987410164, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1190600290909091, "avg_duration_per_task_ms": 130010.17272727273, "avg_output_tokens_per_task": 9680.261818181818, "last_run_timestamp": "2026-10-24T19:35:31.077Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Novita", "accuracy": 0.71, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01784487, "avg_duration_per_task_ms": 239614.14, "avg_output_tokens_per_task": 16662.97, "last_run_timestamp": "2026-10-24T11:51:42.268Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.71, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12724396999999998, "avg_duration_per_task_ms": 73218.95, "avg_output_tokens_per_task": 3826.45, "last_run_timestamp": "2026-10-01T19:04:38.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.71, "accuracy_std_dev": 0.046187895240528104, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0660468475, "avg_duration_per_task_ms": 246966.18, "avg_output_tokens_per_task": 8220.67, "last_run_timestamp": "2026-10-24T19:39:33.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7099999166666667, "accuracy_std_dev": 0.03061105767381654, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03565966833333333, "avg_duration_per_task_ms": 184659.72166666668, "avg_output_tokens_per_task": 9078.276666666667, "last_run_timestamp": "2026-10-24T19:49:36.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.70990758186398, "accuracy_std_dev": 0.05927908384168378, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.04869609571788413, "avg_duration_per_task_ms": 986154.4534005037, "avg_output_tokens_per_task": 53874.76574307305, "last_run_timestamp": "2026-10-24T12:28:11.805Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.709166625, "accuracy_std_dev": 0.01198972650790563, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02624361, "avg_duration_per_task_ms": 203778.2175, "avg_output_tokens_per_task": 10089.3, "last_run_timestamp": "2026-10-24T19:34:31.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7090847895791583, "accuracy_std_dev": 0.018714036978695964, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.0960923246492986, "avg_duration_per_task_ms": 181290.61923847697, "avg_output_tokens_per_task": 10798.256513026052, "last_run_timestamp": "2026-10-17T19:53:39.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7084848181818182, "accuracy_std_dev": 0.037721935609784936, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.10331338545454545, "avg_duration_per_task_ms": 98153.38727272727, "avg_output_tokens_per_task": 9101.983636363637, "last_run_timestamp": "2026-10-24T19:12:36.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7083335, "accuracy_std_dev": 0.041466338423473054, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.156516845, "avg_duration_per_task_ms": 345707.575, "avg_output_tokens_per_task": 10875.9, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.7077778333333333, "accuracy_std_dev": 0.033091888529853064, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.04299519499999999, "avg_duration_per_task_ms": 138463.85833333334, "avg_output_tokens_per_task": 8481.851666666667, "last_run_timestamp": "2026-10-24T19:47:13.308Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7076191428571428, "accuracy_std_dev": 0.061867939965308524, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10378842857142857, "avg_duration_per_task_ms": 217807.51714285713, "avg_output_tokens_per_task": 11023.12, "last_run_timestamp": "2026-10-17T19:18:15.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.7075, "accuracy_std_dev": 0.038935452283234105, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06791916749999999, "avg_duration_per_task_ms": 69428.1925, "avg_output_tokens_per_task": 9662.645, "last_run_timestamp": "2026-10-24T19:30:31.496Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.707483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.004842469387755102, "avg_duration_per_task_ms": 161558.55102040817, "avg_output_tokens_per_task": 17024.020408163266, "last_run_timestamp": "2026-10-01T08:01:49.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7073334, "accuracy_std_dev": 0.025681872553223187, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.024487520000000002, "avg_duration_per_task_ms": 117939.624, "avg_output_tokens_per_task": 30855.352, "last_run_timestamp": "2026-10-24T19:30:58.294Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.3541182, "avg_duration_per_task_ms": 1646927.84, "avg_output_tokens_per_task": 24534.74, "last_run_timestamp": "2026-09-10T00:00:02.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cohere", "accuracy": 0.706667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07705153, "avg_duration_per_task_ms": 159916.36, "avg_output_tokens_per_task": 34851.38, "last_run_timestamp": "2026-10-24T12:16:31.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.706667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.44632813, "avg_duration_per_task_ms": 132811.5, "avg_output_tokens_per_task": 20668.24, "last_run_timestamp": "2026-10-01T19:04:52.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.71176192, "avg_duration_per_task_ms": 285827.28, "avg_output_tokens_per_task": 22895.76, "last_run_timestamp": "2026-10-01T16:00:34.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1838298, "avg_duration_per_task_ms": 329811.9, "avg_output_tokens_per_task": 9540.84, "last_run_timestamp": "2026-08-15T00:16:18.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 3.04321209, "avg_duration_per_task_ms": 317897.29, "avg_output_tokens_per_task": 13272.53, "last_run_timestamp": "2026-10-02T23:13:16.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1042244, "avg_duration_per_task_ms": 226024.67, "avg_output_tokens_per_task": 3543.23, "last_run_timestamp": "2026-10-30T00:17:12.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.70666675, "accuracy_std_dev": 0.027080004924067123, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04331178, "avg_duration_per_task_ms": 178565.92, "avg_output_tokens_per_task": 8366.805, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.5994524266666668, "avg_duration_per_task_ms": 636541.5466666666, "avg_output_tokens_per_task": 54354.86, "last_run_timestamp": "2026-10-02T23:22:41.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.01962587106064057, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08575882, "avg_duration_per_task_ms": 316120.8933333333, "avg_output_tokens_per_task": 32977.22666666667, "last_run_timestamp": "2026-10-02T23:12:53.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Makora", "accuracy": 0.7066665000000001, "accuracy_std_dev": 0.027888567949071897, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021449605000000004, "avg_duration_per_task_ms": 108412.625, "avg_output_tokens_per_task": 11247.685, "last_run_timestamp": "2026-10-24T11:28:47.983Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.7060606363636364, "accuracy_std_dev": 0.15377933291705106, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06115002181818183, "avg_duration_per_task_ms": 147923.08545454545, "avg_output_tokens_per_task": 17075.432727272728, "last_run_timestamp": "2026-10-24T18:32:59.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7057145714285713, "accuracy_std_dev": 0.030327119046712443, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.012995502857142857, "avg_duration_per_task_ms": 102610.46857142857, "avg_output_tokens_per_task": 4898.9228571428575, "last_run_timestamp": "2026-10-01T19:05:16.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.7050002499999999, "accuracy_std_dev": 0.029580300314356156, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050410445, "avg_duration_per_task_ms": 854714.71, "avg_output_tokens_per_task": 8739.19, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.705000125, "accuracy_std_dev": 0.054746931675294576, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.031620825, "avg_duration_per_task_ms": 323083.205, "avg_output_tokens_per_task": 10027.1975, "last_run_timestamp": "2026-10-24T19:49:45.253Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.70499975, "accuracy_std_dev": 0.011902485735656231, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.13260128, "avg_duration_per_task_ms": 477326.39, "avg_output_tokens_per_task": 55745.89, "last_run_timestamp": "2026-10-01T19:05:16.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.7040814081632653, "accuracy_std_dev": 0.029861000000000026, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.12006706122448979, "avg_duration_per_task_ms": 975720.112244898, "avg_output_tokens_per_task": 30678.173469387755, "last_run_timestamp": "2026-08-15T14:15:18.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DigitalOcean", "accuracy": 0.7033335, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01719149, "avg_duration_per_task_ms": 73722.59, "avg_output_tokens_per_task": 12133.38, "last_run_timestamp": "2026-10-24T12:37:04.711Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7033335, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08534018, "avg_duration_per_task_ms": 347658.09, "avg_output_tokens_per_task": 32908.85, "last_run_timestamp": "2026-10-01T16:00:47.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Parasail", "accuracy": 0.703333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.012750089999999999, "avg_duration_per_task_ms": 89612.54, "avg_output_tokens_per_task": 11420.52, "last_run_timestamp": "2026-10-24T12:33:10.739Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.703333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05425128, "avg_duration_per_task_ms": 396980.46, "avg_output_tokens_per_task": 20673.9, "last_run_timestamp": "2026-10-01T08:05:20.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.7030284301606922, "accuracy_std_dev": 0.030250110911983902, "run_count": 33, "total_questions": 1618, "avg_cost_per_task_usd": 0.10101714894932014, "avg_duration_per_task_ms": 71347.47466007416, "avg_output_tokens_per_task": 8063.588998763906, "last_run_timestamp": "2026-10-24T19:37:14.677Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.7027272727272728, "accuracy_std_dev": 0.026469593069959645, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11361526181818182, "avg_duration_per_task_ms": 93548.7109090909, "avg_output_tokens_per_task": 9867.885454545454, "last_run_timestamp": "2026-10-24T19:14:21.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.7022816510851418, "accuracy_std_dev": 0.032548814933197716, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.07496608848080134, "avg_duration_per_task_ms": 173732.56427378964, "avg_output_tokens_per_task": 9603.57429048414, "last_run_timestamp": "2026-10-24T19:09:54.424Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "ModelRun", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.013698571612479225, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013673380000000002, "avg_duration_per_task_ms": 132826.08666666667, "avg_output_tokens_per_task": 12739.9, "last_run_timestamp": "2026-10-28T05:04:05.678Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.021999169459070246, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10818771333333332, "avg_duration_per_task_ms": 79804.25333333333, "avg_output_tokens_per_task": 3542.24, "last_run_timestamp": "2026-10-02T23:10:36.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13357544, "avg_duration_per_task_ms": 105586.74666666667, "avg_output_tokens_per_task": 5260.826666666667, "last_run_timestamp": "2026-10-02T23:07:17.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.7018970975609756, "accuracy_std_dev": 0.11960704720475296, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.0732234268292683, "avg_duration_per_task_ms": 155630.15853658537, "avg_output_tokens_per_task": 15582.174796747968, "last_run_timestamp": "2026-10-03T16:16:30.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.70166675, "accuracy_std_dev": 0.02939198209014661, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.046523760000000004, "avg_duration_per_task_ms": 539638.46, "avg_output_tokens_per_task": 9099.4325, "last_run_timestamp": "2026-10-24T20:03:06.027Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.7015283553008596, "accuracy_std_dev": 0.030155364434940605, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.02074343553008596, "avg_duration_per_task_ms": 613410.9627507164, "avg_output_tokens_per_task": 9252.200573065902, "last_run_timestamp": "2026-10-24T19:39:11.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.701111, "accuracy_std_dev": 0.005665795148667743, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05215248, "avg_duration_per_task_ms": 222421.00666666665, "avg_output_tokens_per_task": 6776.133333333333, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.701111, "accuracy_std_dev": 0.02276020351988678, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.14173139733333334, "avg_duration_per_task_ms": 80397.728, "avg_output_tokens_per_task": 10481.090666666667, "last_run_timestamp": "2026-10-24T19:06:47.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.7010051055276382, "accuracy_std_dev": 0.03189651765083454, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.016903984924623115, "avg_duration_per_task_ms": 263739.4221105528, "avg_output_tokens_per_task": 8905.97487437186, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29769207999999997, "avg_duration_per_task_ms": 128375.6, "avg_output_tokens_per_task": 8500.86, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00357748, "avg_duration_per_task_ms": 48793.08, "avg_output_tokens_per_task": 1567.76, "last_run_timestamp": "2026-10-01T23:50:45.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0030046400000000003, "avg_duration_per_task_ms": 98848.12, "avg_output_tokens_per_task": 17113.74, "last_run_timestamp": "2026-10-01T08:00:19.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12726106, "avg_duration_per_task_ms": 181226.44, "avg_output_tokens_per_task": 9838.56, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07376152, "avg_duration_per_task_ms": 750952.84, "avg_output_tokens_per_task": 11282.28, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": 0.004714280910170736, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.634261665, "avg_duration_per_task_ms": 673734.145, "avg_output_tokens_per_task": 52923.67, "last_run_timestamp": "2026-10-01T16:01:02.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1012818, "avg_duration_per_task_ms": 144103.72, "avg_output_tokens_per_task": 3638.14, "last_run_timestamp": "2026-10-01T08:03:00.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "OpenAI", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05778106, "avg_duration_per_task_ms": 103456.94, "avg_output_tokens_per_task": 4693.4, "last_run_timestamp": "2026-10-28T00:06:02.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.7, "accuracy_std_dev": 0.04203161508614677, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06961394250000001, "avg_duration_per_task_ms": 353777.6075, "avg_output_tokens_per_task": 9996.6825, "last_run_timestamp": "2026-10-24T20:13:56.072Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.69916675, "accuracy_std_dev": 0.10463910531530503, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09495143499999999, "avg_duration_per_task_ms": 230681.74, "avg_output_tokens_per_task": 21996.935, "last_run_timestamp": "2026-10-24T19:19:49.639Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.6987950281124499, "accuracy_std_dev": 0.0269759350636822, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.02643562248995984, "avg_duration_per_task_ms": 223792.88353413655, "avg_output_tokens_per_task": 9633.240963855422, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.698333375, "accuracy_std_dev": 0.030867217869195378, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.051756159999999996, "avg_duration_per_task_ms": 379051.165, "avg_output_tokens_per_task": 9248.245, "last_run_timestamp": "2026-10-24T19:50:05.039Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.6977777333333334, "accuracy_std_dev": 0.035311727846834236, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.12898300933333331, "avg_duration_per_task_ms": 132102.74533333333, "avg_output_tokens_per_task": 9039.205333333333, "last_run_timestamp": "2026-10-24T19:19:29.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.6977776666666667, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.14221055333333335, "avg_duration_per_task_ms": 130884.03333333334, "avg_output_tokens_per_task": 4996.426666666666, "last_run_timestamp": "2026-10-30T00:14:00.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6970368888888889, "accuracy_std_dev": 0.01221661375563299, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021826557777777776, "avg_duration_per_task_ms": 527673.02, "avg_output_tokens_per_task": 18763.36, "last_run_timestamp": "2026-10-24T19:03:53.319Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.696667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01321477, "avg_duration_per_task_ms": 116452.62, "avg_output_tokens_per_task": 5139.02, "last_run_timestamp": "2026-10-01T19:04:36.191Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.62488186, "avg_duration_per_task_ms": 152081.62, "avg_output_tokens_per_task": 3927.45, "last_run_timestamp": "2026-10-01T08:05:41.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.6958335, "accuracy_std_dev": 0.03781341986980283, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.132227115, "avg_duration_per_task_ms": 205681.995, "avg_output_tokens_per_task": 9407.695, "last_run_timestamp": "2026-10-01T22:50:26.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.6948196351351352, "accuracy_std_dev": 0.0038228551575432913, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.005936054054054054, "avg_duration_per_task_ms": 331167.7297297297, "avg_output_tokens_per_task": 9197.358108108108, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.6947621428571428, "accuracy_std_dev": 0.08007667414141911, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.051026225714285715, "avg_duration_per_task_ms": 483301.40285714285, "avg_output_tokens_per_task": 16016.288571428571, "last_run_timestamp": "2026-10-24T19:51:41.428Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Together", "accuracy": 0.6933334999999999, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01276862, "avg_duration_per_task_ms": 66785.21, "avg_output_tokens_per_task": 8630.75, "last_run_timestamp": "2026-10-24T12:58:47.706Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6933334999999999, "accuracy_std_dev": 0.018257236010689022, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08853248000000001, "avg_duration_per_task_ms": 338024.125, "avg_output_tokens_per_task": 32557.16, "last_run_timestamp": "2026-10-01T16:01:04.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.014401903168987411, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03953364, "avg_duration_per_task_ms": 309829.13333333336, "avg_output_tokens_per_task": 32261.04, "last_run_timestamp": "2026-10-02T23:10:33.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cohere", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01518678, "avg_duration_per_task_ms": 46885.36, "avg_output_tokens_per_task": 12120.66, "last_run_timestamp": "2026-10-28T21:15:52.006Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05167764, "avg_duration_per_task_ms": 69227.88, "avg_output_tokens_per_task": 9318.98, "last_run_timestamp": "2026-08-08T08:31:07.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "BaseTen", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03237594, "avg_duration_per_task_ms": 69982.76, "avg_output_tokens_per_task": 15576.66, "last_run_timestamp": "2026-10-20T06:20:53.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12999802000000002, "avg_duration_per_task_ms": 96250.54, "avg_output_tokens_per_task": 4982.06, "last_run_timestamp": "2026-10-01T08:05:35.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02754302, "avg_duration_per_task_ms": 209408.8, "avg_output_tokens_per_task": 14914.54, "last_run_timestamp": "2026-10-24T12:03:24.771Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6920511153846154, "accuracy_std_dev": 0.02287235899731531, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.10611875923076924, "avg_duration_per_task_ms": 75613.00384615385, "avg_output_tokens_per_task": 9658.539230769231, "last_run_timestamp": "2026-10-30T16:20:14.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.69, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10882898999999999, "avg_duration_per_task_ms": 135482.83, "avg_output_tokens_per_task": 51335.48, "last_run_timestamp": "2026-10-01T19:05:16.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 2.99530482, "avg_duration_per_task_ms": 320001.13, "avg_output_tokens_per_task": 12475.35, "last_run_timestamp": "2026-10-01T08:05:20.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.69, "accuracy_std_dev": 0.04333300000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07366, "avg_duration_per_task_ms": 63569.01, "avg_output_tokens_per_task": 3649.5, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.6897523648648648, "accuracy_std_dev": 0.024522787726702033, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.02191278716216216, "avg_duration_per_task_ms": 348901.27364864864, "avg_output_tokens_per_task": 16679.054054054053, "last_run_timestamp": "2026-10-27T15:35:36.855Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.6886632385786803, "accuracy_std_dev": 0.0900995905873051, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.1329306802030457, "avg_duration_per_task_ms": 1780497.6497461929, "avg_output_tokens_per_task": 98848.87817258883, "last_run_timestamp": "2026-10-24T19:18:22.830Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.6877776666666667, "accuracy_std_dev": 0.0470485263023426, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09563632666666667, "avg_duration_per_task_ms": 192574.105, "avg_output_tokens_per_task": 7480.165, "last_run_timestamp": "2026-10-27T01:50:56.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6874997500000001, "accuracy_std_dev": 0.045023109806936926, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0177425775, "avg_duration_per_task_ms": 384968.0575, "avg_output_tokens_per_task": 20103.015, "last_run_timestamp": "2026-10-24T21:02:58.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00219582, "avg_duration_per_task_ms": 78121.92, "avg_output_tokens_per_task": 11659.46, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Azure", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06838028, "avg_duration_per_task_ms": 78879.18, "avg_output_tokens_per_task": 4783.1, "last_run_timestamp": "2026-10-28T00:05:23.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Tenstorrent", "accuracy": 0.6866665000000001, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01639777, "avg_duration_per_task_ms": 134478.47, "avg_output_tokens_per_task": 16372.87, "last_run_timestamp": "2026-10-30T18:45:46.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Wafer", "accuracy": 0.6866665000000001, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01561806, "avg_duration_per_task_ms": 117250.14, "avg_output_tokens_per_task": 14519.81, "last_run_timestamp": "2026-10-24T12:06:50.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.6861111666666666, "accuracy_std_dev": 0.0669138081126675, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11072275, "avg_duration_per_task_ms": 76500.85166666667, "avg_output_tokens_per_task": 9058.545, "last_run_timestamp": "2026-10-28T11:20:49.843Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.6854528208092485, "accuracy_std_dev": 0.016921495753349107, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.03335441329479769, "avg_duration_per_task_ms": 533340.3294797688, "avg_output_tokens_per_task": 18704.07225433526, "last_run_timestamp": "2026-10-17T19:21:01.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6850001666666666, "accuracy_std_dev": 0.0143694957278334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04138883, "avg_duration_per_task_ms": 242662.93333333332, "avg_output_tokens_per_task": 30866.096666666668, "last_run_timestamp": "2026-10-01T08:05:57.699Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.685000125, "accuracy_std_dev": 0.03693083437805562, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0358575875, "avg_duration_per_task_ms": 157897.08, "avg_output_tokens_per_task": 9891.505, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.6837050185873607, "accuracy_std_dev": 0.11487260701179351, "run_count": 11, "total_questions": 538, "avg_cost_per_task_usd": 0.32785563568773235, "avg_duration_per_task_ms": 1690067.0799256505, "avg_output_tokens_per_task": 54783.62267657993, "last_run_timestamp": "2026-10-24T12:39:42.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.6834170150753769, "accuracy_std_dev": 0.019760471659034357, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.006376396984924624, "avg_duration_per_task_ms": 294466.5879396985, "avg_output_tokens_per_task": 9614.21608040201, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.07779388246014092, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00909372, "avg_duration_per_task_ms": 253755.2, "avg_output_tokens_per_task": 15941.146666666667, "last_run_timestamp": "2026-10-08T22:38:42.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.012619766977070334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10710260999999997, "avg_duration_per_task_ms": 135398.91, "avg_output_tokens_per_task": 4027.3166666666666, "last_run_timestamp": "2026-10-01T08:06:09.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.683333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00634196, "avg_duration_per_task_ms": 404260.22, "avg_output_tokens_per_task": 17374, "last_run_timestamp": "2026-10-17T11:31:13.423Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.6826667, "accuracy_std_dev": 0.03427992350647823, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.01766094, "avg_duration_per_task_ms": 313229.622, "avg_output_tokens_per_task": 20074.712, "last_run_timestamp": "2026-10-30T16:21:36.179Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.021998563367234276, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.014046139999999999, "avg_duration_per_task_ms": 71689.16, "avg_output_tokens_per_task": 5022.28, "last_run_timestamp": "2026-10-30T00:15:40.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09746202, "avg_duration_per_task_ms": 132162.47333333333, "avg_output_tokens_per_task": 3507.6066666666666, "last_run_timestamp": "2026-10-02T23:10:34.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.6822220000000001, "accuracy_std_dev": 0.020607975203789406, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.015206260000000001, "avg_duration_per_task_ms": 252877.44, "avg_output_tokens_per_task": 17489.156666666666, "last_run_timestamp": "2026-10-24T20:33:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.6813334, "accuracy_std_dev": 0.0026667999999999913, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08704178, "avg_duration_per_task_ms": 82752.556, "avg_output_tokens_per_task": 3576.016, "last_run_timestamp": "2026-10-01T16:01:04.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.6811111666666666, "accuracy_std_dev": 0.008088865998326882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00903582, "avg_duration_per_task_ms": 105469.92666666667, "avg_output_tokens_per_task": 8459.866666666667, "last_run_timestamp": "2026-10-01T08:05:22.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Minimax", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01807364, "avg_duration_per_task_ms": 208774.72, "avg_output_tokens_per_task": 16251.38, "last_run_timestamp": "2026-10-24T12:17:54.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.58024224, "avg_duration_per_task_ms": 141487.98, "avg_output_tokens_per_task": 3731.58, "last_run_timestamp": "2026-10-01T08:05:19.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.68, "accuracy_std_dev": 0.03810292105162893, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00883898, "avg_duration_per_task_ms": 114396.12, "avg_output_tokens_per_task": 8841.28, "last_run_timestamp": "2026-10-02T23:08:44.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "PrimeIntellect", "accuracy": 0.68, "accuracy_std_dev": 0.06, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.049401919999999995, "avg_duration_per_task_ms": 283175.04, "avg_output_tokens_per_task": 24143.94, "last_run_timestamp": "2026-10-29T23:29:22.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.6785714285714286, "accuracy_std_dev": 0.020303791411297286, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.017469171428571427, "avg_duration_per_task_ms": 119261.06857142856, "avg_output_tokens_per_task": 21365.42, "last_run_timestamp": "2026-10-24T11:42:22.586Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.676667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01545023, "avg_duration_per_task_ms": 128578.81, "avg_output_tokens_per_task": 15653.52, "last_run_timestamp": "2026-10-24T12:17:01.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.67666675, "accuracy_std_dev": 0.028867484592314224, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.008505605000000001, "avg_duration_per_task_ms": 317641.53, "avg_output_tokens_per_task": 13432.655, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5967202500000001, "avg_duration_per_task_ms": 137724.15, "avg_output_tokens_per_task": 4015.19, "last_run_timestamp": "2026-10-02T23:06:01.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.676, "accuracy_std_dev": 0.19009238689542513, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011841847999999999, "avg_duration_per_task_ms": 391925.292, "avg_output_tokens_per_task": 10879.944, "last_run_timestamp": "2026-10-24T13:02:10.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6757320878661087, "accuracy_std_dev": 0.02326584314053542, "run_count": 5, "total_questions": 239, "avg_cost_per_task_usd": 0.14251672803347282, "avg_duration_per_task_ms": 407673.5857740586, "avg_output_tokens_per_task": 10868.456066945606, "last_run_timestamp": "2026-10-17T19:51:53.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.675675972972973, "accuracy_std_dev": 0.02284471405428887, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04042408108108108, "avg_duration_per_task_ms": 518660.3783783784, "avg_output_tokens_per_task": 20990.527027027027, "last_run_timestamp": "2026-09-26T08:11:26.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.63842362, "avg_duration_per_task_ms": 88201.35333333333, "avg_output_tokens_per_task": 3898.48, "last_run_timestamp": "2026-10-30T00:17:00.667Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07120372000000001, "avg_duration_per_task_ms": 144319.38666666666, "avg_output_tokens_per_task": 2561.846666666667, "last_run_timestamp": "2026-10-08T22:25:52.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.675, "accuracy_std_dev": 0.011667000000000038, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1079227, "avg_duration_per_task_ms": 145751.42, "avg_output_tokens_per_task": 18113.26, "last_run_timestamp": "2026-10-04T05:20:00.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.6741405246636771, "accuracy_std_dev": 0.030345316137851863, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.022004533632286993, "avg_duration_per_task_ms": 284802.9730941704, "avg_output_tokens_per_task": 16879.719730941702, "last_run_timestamp": "2026-10-24T19:52:46.621Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09937637, "avg_duration_per_task_ms": 637553.26, "avg_output_tokens_per_task": 9917.08, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.6292479799999999, "avg_duration_per_task_ms": 729264.79, "avg_output_tokens_per_task": 54931.75, "last_run_timestamp": "2026-10-01T16:01:02.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.6733330000000001, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.5819717300000002, "avg_duration_per_task_ms": 314318.37, "avg_output_tokens_per_task": 23725.55, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00928508, "avg_duration_per_task_ms": 121838.98, "avg_output_tokens_per_task": 8898.44, "last_run_timestamp": "2026-10-01T08:00:22.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.672, "accuracy_std_dev": 0.028409779224766934, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07962264, "avg_duration_per_task_ms": 80557.024, "avg_output_tokens_per_task": 3531.236, "last_run_timestamp": "2026-10-01T16:00:45.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.6719998, "accuracy_std_dev": 0.012927091279943813, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.015563263999999999, "avg_duration_per_task_ms": 230227.66, "avg_output_tokens_per_task": 10531.424, "last_run_timestamp": "2026-10-13T22:01:15.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.6706668, "accuracy_std_dev": 0.015548970672041286, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.142799016, "avg_duration_per_task_ms": 115124.684, "avg_output_tokens_per_task": 5068.812, "last_run_timestamp": "2026-10-01T08:04:38.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.669999875, "accuracy_std_dev": 0.10765177786669097, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.017295960000000003, "avg_duration_per_task_ms": 256892.225, "avg_output_tokens_per_task": 10151.75, "last_run_timestamp": "2026-10-27T02:33:21.663Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.6686666, "accuracy_std_dev": 0.08499931398452577, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.05116424000000001, "avg_duration_per_task_ms": 341233.54, "avg_output_tokens_per_task": 7834.636, "last_run_timestamp": "2026-10-03T16:13:53.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.66833325, "accuracy_std_dev": 0.005527532106419614, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.027697885000000002, "avg_duration_per_task_ms": 410531.75, "avg_output_tokens_per_task": 10158.81, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6670706666666666, "accuracy_std_dev": 0.03017076900885787, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.12485057212121214, "avg_duration_per_task_ms": 121821.06121212122, "avg_output_tokens_per_task": 12714.262424242424, "last_run_timestamp": "2026-10-24T13:04:43.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.038762819999999996, "avg_duration_per_task_ms": 314594.08, "avg_output_tokens_per_task": 31952.48, "last_run_timestamp": "2026-10-01T08:00:24.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.66666675, "accuracy_std_dev": 0.016996797086142445, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.008543925, "avg_duration_per_task_ms": 288167.26, "avg_output_tokens_per_task": 9621.62, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.6666667, "accuracy_std_dev": 0.1539986724332713, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.12421460999999999, "avg_duration_per_task_ms": 361200.082, "avg_output_tokens_per_task": 8728.562, "last_run_timestamp": "2026-10-27T01:27:10.502Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.6666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08889954, "avg_duration_per_task_ms": 108433.06, "avg_output_tokens_per_task": 12861.73, "last_run_timestamp": "2026-10-01T19:04:42.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.6651852222222222, "accuracy_std_dev": 0.07567163233159697, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09013567777777777, "avg_duration_per_task_ms": 120445.85555555555, "avg_output_tokens_per_task": 8441.84, "last_run_timestamp": "2026-10-28T11:09:01.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.6649998333333333, "accuracy_std_dev": 0.024626931209664655, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008708743333333335, "avg_duration_per_task_ms": 613076.7966666666, "avg_output_tokens_per_task": 17440.79, "last_run_timestamp": "2026-10-24T19:39:37.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Io Net", "accuracy": 0.66493075, "accuracy_std_dev": 0.03659399999999996, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.0157241875, "avg_duration_per_task_ms": 279786.6875, "avg_output_tokens_per_task": 12241.416666666666, "last_run_timestamp": "2026-10-27T13:23:22.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.6632998585858586, "accuracy_std_dev": 0.0034015000000000017, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.07516567676767677, "avg_duration_per_task_ms": 78894.82828282828, "avg_output_tokens_per_task": 13736.282828282829, "last_run_timestamp": "2026-08-15T14:12:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.662963, "accuracy_std_dev": 0.06555468465673864, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05622624222222223, "avg_duration_per_task_ms": 819076.5222222222, "avg_output_tokens_per_task": 18202.83777777778, "last_run_timestamp": "2026-10-05T20:43:11.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.6618180909090909, "accuracy_std_dev": 0.03062851381683575, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11597669454545456, "avg_duration_per_task_ms": 125484.61818181818, "avg_output_tokens_per_task": 12643.88, "last_run_timestamp": "2026-10-24T12:09:26.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6613334000000001, "accuracy_std_dev": 0.01758814752724117, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10186603200000001, "avg_duration_per_task_ms": 156255.32, "avg_output_tokens_per_task": 13208.728, "last_run_timestamp": "2026-10-01T19:04:42.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.6612121818181818, "accuracy_std_dev": 0.027386941316879746, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11405556727272728, "avg_duration_per_task_ms": 127222.07090909091, "avg_output_tokens_per_task": 12352.41090909091, "last_run_timestamp": "2026-10-24T12:22:57.423Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "PrimeIntellect", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03137252, "avg_duration_per_task_ms": 226181.32, "avg_output_tokens_per_task": 15432.64, "last_run_timestamp": "2026-10-29T23:53:36.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "auto-routing", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.013329200000000001, "avg_duration_per_task_ms": 278453.68, "avg_output_tokens_per_task": 15070.16, "last_run_timestamp": "2026-10-24T12:29:49.995Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.6590195882352939, "accuracy_std_dev": 0.028566314925427196, "run_count": 34, "total_questions": 1700, "avg_cost_per_task_usd": 0.12125589529411765, "avg_duration_per_task_ms": 121695.54823529412, "avg_output_tokens_per_task": 12641.248823529411, "last_run_timestamp": "2026-10-24T12:56:50.982Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.6588886666666667, "accuracy_std_dev": 0.03965918385723817, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.006265106666666667, "avg_duration_per_task_ms": 362004.0733333333, "avg_output_tokens_per_task": 19036.246666666666, "last_run_timestamp": "2026-10-24T19:34:04.376Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6585712857142858, "accuracy_std_dev": 0.014125931834894325, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.023240128571428573, "avg_duration_per_task_ms": 158768.29142857142, "avg_output_tokens_per_task": 17172.022857142856, "last_run_timestamp": "2026-10-24T19:17:27.524Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.6583334999999999, "accuracy_std_dev": 0.028037782049406136, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02298765, "avg_duration_per_task_ms": 355829.865, "avg_output_tokens_per_task": 10214.775, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.65666675, "accuracy_std_dev": 0.0074535971978300555, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02864462, "avg_duration_per_task_ms": 223453.205, "avg_output_tokens_per_task": 10372.695, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6566381031518624, "accuracy_std_dev": 0.020222279406836508, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.01698823782234957, "avg_duration_per_task_ms": 325791.0286532951, "avg_output_tokens_per_task": 22916.045845272205, "last_run_timestamp": "2026-10-24T12:47:29.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.6558186153846154, "accuracy_std_dev": 0.022915354823914075, "run_count": 17, "total_questions": 845, "avg_cost_per_task_usd": 0.010244520710059171, "avg_duration_per_task_ms": 311624.14082840236, "avg_output_tokens_per_task": 17072.48402366864, "last_run_timestamp": "2026-10-30T00:56:40.041Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6557823102040816, "accuracy_std_dev": 0.009318408855593308, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.014797057142857142, "avg_duration_per_task_ms": 277907.2775510204, "avg_output_tokens_per_task": 20245.579591836733, "last_run_timestamp": "2026-10-10T11:18:54.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.6555556666666666, "accuracy_std_dev": 0.012570472951413636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.030086579999999998, "avg_duration_per_task_ms": 125366.48, "avg_output_tokens_per_task": 16133.046666666667, "last_run_timestamp": "2026-10-02T23:09:54.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.654618485943775, "accuracy_std_dev": 0.11865847802618236, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.04366364257028112, "avg_duration_per_task_ms": 1933703.5220883535, "avg_output_tokens_per_task": 71251.46586345382, "last_run_timestamp": "2026-10-24T12:24:29.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.654279400900901, "accuracy_std_dev": 0.11858723742179567, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.025987085585585587, "avg_duration_per_task_ms": 1029787.9324324324, "avg_output_tokens_per_task": 34152.32432432433, "last_run_timestamp": "2026-10-17T21:28:32.796Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.65416675, "accuracy_std_dev": 0.03373725891262509, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.013133230000000001, "avg_duration_per_task_ms": 415006.955, "avg_output_tokens_per_task": 18075.7325, "last_run_timestamp": "2026-10-24T19:51:37.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.6533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00488981, "avg_duration_per_task_ms": 111736.2, "avg_output_tokens_per_task": 17377.29, "last_run_timestamp": "2026-10-01T08:04:38.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6533333333333333, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.2856746933333333, "avg_duration_per_task_ms": 183725.1, "avg_output_tokens_per_task": 8628.813333333334, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00923216, "avg_duration_per_task_ms": 145753.66, "avg_output_tokens_per_task": 8192.38, "last_run_timestamp": "2026-09-23T06:00:08.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.65166675, "accuracy_std_dev": 0.038657908491504044, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021260025000000002, "avg_duration_per_task_ms": 249515.925, "avg_output_tokens_per_task": 26907.185, "last_run_timestamp": "2026-10-24T12:38:08.218Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.65166675, "accuracy_std_dev": 0.04506177549972369, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04474220500000001, "avg_duration_per_task_ms": 212693.005, "avg_output_tokens_per_task": 3236.785, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.6515152727272727, "accuracy_std_dev": 0.024959225520075408, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11565049454545455, "avg_duration_per_task_ms": 139062.33454545453, "avg_output_tokens_per_task": 12670.038181818181, "last_run_timestamp": "2026-10-24T11:52:54.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.6513452645739911, "accuracy_std_dev": 0.03893599144686849, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.012713934977578476, "avg_duration_per_task_ms": 951818.3878923766, "avg_output_tokens_per_task": 23060.636771300447, "last_run_timestamp": "2026-10-27T16:29:30.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.6499997499999999, "accuracy_std_dev": 0.0378593669609979, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015033419999999999, "avg_duration_per_task_ms": 89205.15, "avg_output_tokens_per_task": 19468.825, "last_run_timestamp": "2026-10-24T19:10:30.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.6495835, "accuracy_std_dev": 0.03429607808846371, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.034037475, "avg_duration_per_task_ms": 61916.2125, "avg_output_tokens_per_task": 14847.6825, "last_run_timestamp": "2026-10-24T18:54:16.139Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6491228070175439, "accuracy_std_dev": 0.025681291234756373, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.01387597493734336, "avg_duration_per_task_ms": 805180.8822055138, "avg_output_tokens_per_task": 25965.736842105263, "last_run_timestamp": "2026-10-30T03:01:20.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.6490788090452262, "accuracy_std_dev": 0.06101351971284725, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.02417512814070352, "avg_duration_per_task_ms": 427158.12060301506, "avg_output_tokens_per_task": 15670.065326633166, "last_run_timestamp": "2026-10-24T19:06:03.144Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.6488889999999999, "accuracy_std_dev": 0.019115994210782406, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08501352666666667, "avg_duration_per_task_ms": 97755.09333333334, "avg_output_tokens_per_task": 12128.533333333333, "last_run_timestamp": "2026-10-02T23:10:27.235Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.64833325, "accuracy_std_dev": 0.021794475600195134, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.022383950000000007, "avg_duration_per_task_ms": 245046.31, "avg_output_tokens_per_task": 9447.4, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.647177495967742, "accuracy_std_dev": 0.07108568891563195, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.025456885080645163, "avg_duration_per_task_ms": 277301.2076612903, "avg_output_tokens_per_task": 25224.120967741936, "last_run_timestamp": "2026-10-17T19:25:29.604Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.6466665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0295618, "avg_duration_per_task_ms": 121397.75, "avg_output_tokens_per_task": 15953.93, "last_run_timestamp": "2026-10-01T19:04:28.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.6455556666666666, "accuracy_std_dev": 0.09964144484545016, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11177372666666668, "avg_duration_per_task_ms": 710817.72, "avg_output_tokens_per_task": 87845.5, "last_run_timestamp": "2026-09-29T19:45:03.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.6416665, "accuracy_std_dev": 0.02833350000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0220575, "avg_duration_per_task_ms": 236336.73, "avg_output_tokens_per_task": 9832.45, "last_run_timestamp": "2026-08-15T14:15:19.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.6413336000000001, "accuracy_std_dev": 0.014847377621654282, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.03101342, "avg_duration_per_task_ms": 232783.988, "avg_output_tokens_per_task": 15818.512, "last_run_timestamp": "2026-10-01T19:05:16.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "AkashML", "accuracy": 0.64, "accuracy_std_dev": 0.023726622023934812, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.016758953333333333, "avg_duration_per_task_ms": 469306.68666666665, "avg_output_tokens_per_task": 27813.52, "last_run_timestamp": "2026-10-30T01:16:49.428Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.64, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01684376, "avg_duration_per_task_ms": 1178898.4, "avg_output_tokens_per_task": 43776.78, "last_run_timestamp": "2026-10-09T00:39:20.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.6393825126903554, "accuracy_std_dev": 0.042421817536747466, "run_count": 16, "total_questions": 788, "avg_cost_per_task_usd": 0.01991223350253807, "avg_duration_per_task_ms": 319706.6497461929, "avg_output_tokens_per_task": 16793.3730964467, "last_run_timestamp": "2026-10-24T19:19:43.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.638298, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.020538510638297872, "avg_duration_per_task_ms": 686659.4680851063, "avg_output_tokens_per_task": 47094.31914893617, "last_run_timestamp": "2026-10-09T00:23:16.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.6377776666666667, "accuracy_std_dev": 0.032810732962790655, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.20948366666666668, "avg_duration_per_task_ms": 978205.8133333334, "avg_output_tokens_per_task": 27155.76, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.6366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.008583809999999999, "avg_duration_per_task_ms": 1454771.53, "avg_output_tokens_per_task": 17935.13, "last_run_timestamp": "2026-10-24T20:43:26.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6353466979865772, "accuracy_std_dev": 0.036572065936844325, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.033807637583892616, "avg_duration_per_task_ms": 262695.20805369125, "avg_output_tokens_per_task": 17180.040268456374, "last_run_timestamp": "2026-10-24T18:33:47.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.6333332500000001, "accuracy_std_dev": 0.12901321207123517, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05091860250000001, "avg_duration_per_task_ms": 844509.295, "avg_output_tokens_per_task": 27160.0425, "last_run_timestamp": "2026-10-24T19:37:12.172Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.6333332500000001, "accuracy_std_dev": 0.016996502910525478, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.024200355, "avg_duration_per_task_ms": 140351.54, "avg_output_tokens_per_task": 10016.095, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.6333332500000001, "accuracy_std_dev": 0.03299835012826401, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021382605000000002, "avg_duration_per_task_ms": 229847.125, "avg_output_tokens_per_task": 9805.265, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.6307017947368421, "accuracy_std_dev": 0.04099517109977711, "run_count": 4, "total_questions": 190, "avg_cost_per_task_usd": 0.02501245263157895, "avg_duration_per_task_ms": 66232.84736842105, "avg_output_tokens_per_task": 14146.842105263158, "last_run_timestamp": "2026-10-24T18:40:18.531Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.6304712676767678, "accuracy_std_dev": 0.028764697857953853, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.041468260101010096, "avg_duration_per_task_ms": 81255.43686868687, "avg_output_tokens_per_task": 21574.416666666668, "last_run_timestamp": "2026-10-24T19:23:06.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.6283335, "accuracy_std_dev": 0.18833350000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.15963215, "avg_duration_per_task_ms": 1567590.01, "avg_output_tokens_per_task": 36431.88, "last_run_timestamp": "2026-10-17T19:19:25.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.6283332500000001, "accuracy_std_dev": 0.023746444179866183, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.028451149999999998, "avg_duration_per_task_ms": 210873.285, "avg_output_tokens_per_task": 9700.365, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.6266666666666667, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12269806, "avg_duration_per_task_ms": 610304.8666666667, "avg_output_tokens_per_task": 27046.16, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.621666625, "accuracy_std_dev": 0.025331066237613754, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06826625, "avg_duration_per_task_ms": 32002.2325, "avg_output_tokens_per_task": 21494.85, "last_run_timestamp": "2026-10-24T18:29:40.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.6155556666666666, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12338818000000001, "avg_duration_per_task_ms": 626122.66, "avg_output_tokens_per_task": 27741.31333333333, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.615, "accuracy_std_dev": 0.03869391999397322, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0947713525, "avg_duration_per_task_ms": 55138.08, "avg_output_tokens_per_task": 3608.635, "last_run_timestamp": "2026-10-24T19:00:38.447Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6076188571428572, "accuracy_std_dev": 0.03274303997331158, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.046068719999999994, "avg_duration_per_task_ms": 191059.44, "avg_output_tokens_per_task": 3405.154285714286, "last_run_timestamp": "2026-10-08T22:20:40.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.606667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03212956, "avg_duration_per_task_ms": 335001.52, "avg_output_tokens_per_task": 8851.18, "last_run_timestamp": "2026-10-01T08:03:01.921Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6066668000000001, "accuracy_std_dev": 0.026666399999999996, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046513716, "avg_duration_per_task_ms": 209556.196, "avg_output_tokens_per_task": 3673.824, "last_run_timestamp": "2026-10-01T16:00:32.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.6049998333333334, "accuracy_std_dev": 0.01979055610989468, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04771943666666667, "avg_duration_per_task_ms": 93553.64666666667, "avg_output_tokens_per_task": 4645.643333333333, "last_run_timestamp": "2026-10-03T16:14:40.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.6026666, "accuracy_std_dev": 0.023701878031919768, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045193232, "avg_duration_per_task_ms": 105920.528, "avg_output_tokens_per_task": 4778.932, "last_run_timestamp": "2026-10-03T16:07:50.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6024903390804598, "accuracy_std_dev": 0.08896202865084331, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.031245040229885054, "avg_duration_per_task_ms": 1940985.2241379311, "avg_output_tokens_per_task": 61478.612068965514, "last_run_timestamp": "2026-10-24T19:11:22.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.6000001666666667, "accuracy_std_dev": 0.03771239112642186, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.19008634000000002, "avg_duration_per_task_ms": 186975.22, "avg_output_tokens_per_task": 11452.75, "last_run_timestamp": "2026-10-01T19:04:55.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.6000001666666667, "accuracy_std_dev": 0.13727506854659452, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04416580999999999, "avg_duration_per_task_ms": 477149.00666666665, "avg_output_tokens_per_task": 8772.623333333333, "last_run_timestamp": "2026-10-10T19:10:33.654Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.593333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04552234, "avg_duration_per_task_ms": 32685.39, "avg_output_tokens_per_task": 3125.98, "last_run_timestamp": "2026-09-26T08:22:32.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.593333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01896666, "avg_duration_per_task_ms": 306972.98, "avg_output_tokens_per_task": 7436.64, "last_run_timestamp": "2026-10-01T08:02:59.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.591111, "accuracy_std_dev": 0.006285295405415193, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029060479999999996, "avg_duration_per_task_ms": 77755.87, "avg_output_tokens_per_task": 7763.98, "last_run_timestamp": "2026-10-01T19:05:46.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.588889, "accuracy_std_dev": 0.008314764498569197, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.14882857333333332, "avg_duration_per_task_ms": 490464.7866666667, "avg_output_tokens_per_task": 24032.806666666667, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.5873334, "accuracy_std_dev": 0.015691608516656286, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.020115355999999997, "avg_duration_per_task_ms": 100935.536, "avg_output_tokens_per_task": 4366.388, "last_run_timestamp": "2026-10-03T16:14:27.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.583333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015297799999999999, "avg_duration_per_task_ms": 833284.24, "avg_output_tokens_per_task": 38596.74, "last_run_timestamp": "2026-10-01T08:04:13.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.58, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20638665, "avg_duration_per_task_ms": 179102.02, "avg_output_tokens_per_task": 12109.41, "last_run_timestamp": "2026-10-01T19:04:42.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5789978786127168, "accuracy_std_dev": 0.05180400838836742, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.18764681502890174, "avg_duration_per_task_ms": 2044873.4393063583, "avg_output_tokens_per_task": 41085.294797687864, "last_run_timestamp": "2026-10-17T19:42:10.129Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.57483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.017800938775510203, "avg_duration_per_task_ms": 142107.1224489796, "avg_output_tokens_per_task": 3993.1632653061224, "last_run_timestamp": "2026-08-08T16:03:49.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.5733883127572016, "accuracy_std_dev": 0.039571612153158485, "run_count": 5, "total_questions": 243, "avg_cost_per_task_usd": 0.08109196296296296, "avg_duration_per_task_ms": 99593.68724279835, "avg_output_tokens_per_task": 6510.271604938272, "last_run_timestamp": "2026-10-03T16:05:18.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.5666665000000001, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.028862480000000003, "avg_duration_per_task_ms": 75034.33, "avg_output_tokens_per_task": 7701.06, "last_run_timestamp": "2026-10-01T19:04:40.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.5666665, "accuracy_std_dev": 0.1466665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02785182, "avg_duration_per_task_ms": 139069.32, "avg_output_tokens_per_task": 11612.72, "last_run_timestamp": "2026-10-24T12:08:00.201Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.5648772953020134, "accuracy_std_dev": 0.0424873867714486, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1486826308724832, "avg_duration_per_task_ms": 18002.88590604027, "avg_output_tokens_per_task": 2546.3288590604025, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.5633334999999999, "accuracy_std_dev": 0.09666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07923399999999998, "avg_duration_per_task_ms": 1068653.19, "avg_output_tokens_per_task": 62040.81, "last_run_timestamp": "2026-08-15T14:08:35.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.560133587973274, "accuracy_std_dev": 0.2552451920126611, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.015204305122494433, "avg_duration_per_task_ms": 265953.0779510022, "avg_output_tokens_per_task": 7739.80846325167, "last_run_timestamp": "2026-10-17T11:22:12.284Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.557500125, "accuracy_std_dev": 0.07859452244501124, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.032340572500000005, "avg_duration_per_task_ms": 393952.61, "avg_output_tokens_per_task": 16997.9425, "last_run_timestamp": "2026-10-24T19:54:36.551Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.55, "accuracy_std_dev": 0.036667000000000005, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.2373302, "avg_duration_per_task_ms": 164429.2, "avg_output_tokens_per_task": 12508.34, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.546667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03543453, "avg_duration_per_task_ms": 61133.82, "avg_output_tokens_per_task": 6656.99, "last_run_timestamp": "2026-10-01T19:04:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.5458451776504298, "accuracy_std_dev": 0.0501062696215529, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.010405719197707736, "avg_duration_per_task_ms": 192494.59598853867, "avg_output_tokens_per_task": 16106.47564469914, "last_run_timestamp": "2026-10-24T19:30:50.871Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5370371414141414, "accuracy_std_dev": 0.020577999999999985, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.02047550505050505, "avg_duration_per_task_ms": 144559.1616161616, "avg_output_tokens_per_task": 38541.121212121216, "last_run_timestamp": "2026-10-01T19:05:00.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.535956005076142, "accuracy_std_dev": 0.0166977509787995, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.018755781725888324, "avg_duration_per_task_ms": 1279767.0253807106, "avg_output_tokens_per_task": 84685.69796954315, "last_run_timestamp": "2026-10-24T19:37:33.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.534014, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.01161230612244898, "avg_duration_per_task_ms": 94666.5306122449, "avg_output_tokens_per_task": 16582.326530612245, "last_run_timestamp": "2026-10-01T19:04:42.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.5336213227665707, "accuracy_std_dev": 0.029334415082017253, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.002658521613832853, "avg_duration_per_task_ms": 386644.1469740634, "avg_output_tokens_per_task": 6857.3314121037465, "last_run_timestamp": "2026-10-30T01:32:41.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.5333335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10219375, "avg_duration_per_task_ms": 375443.12, "avg_output_tokens_per_task": 17495.06, "last_run_timestamp": "2026-08-15T14:16:11.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.5283335, "accuracy_std_dev": 0.01993049032261877, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.066619805, "avg_duration_per_task_ms": 287868.2675, "avg_output_tokens_per_task": 4057.665, "last_run_timestamp": "2026-10-24T18:33:38.931Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.5258620229885057, "accuracy_std_dev": 0.09372402255384406, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.049525813218390814, "avg_duration_per_task_ms": 424260.5517241379, "avg_output_tokens_per_task": 7604.66091954023, "last_run_timestamp": "2026-10-24T18:46:40.201Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.522831205479452, "accuracy_std_dev": 0.027543964303555765, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.11947870547945207, "avg_duration_per_task_ms": 69921.19178082192, "avg_output_tokens_per_task": 2362.404109589041, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.519759618556701, "accuracy_std_dev": 0.11063848893355334, "run_count": 4, "total_questions": 194, "avg_cost_per_task_usd": 0.05320747422680413, "avg_duration_per_task_ms": 383087.36597938143, "avg_output_tokens_per_task": 23944.39175257732, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.517500125, "accuracy_std_dev": 0.024930426803594342, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0282700725, "avg_duration_per_task_ms": 444815, "avg_output_tokens_per_task": 58343.125, "last_run_timestamp": "2026-10-03T16:09:19.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.5173608125000001, "accuracy_std_dev": 0.025072499999999998, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.06539028125, "avg_duration_per_task_ms": 2881049.1770833335, "avg_output_tokens_per_task": 39867.583333333336, "last_run_timestamp": "2026-08-15T14:12:30.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.513889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.025741249999999997, "avg_duration_per_task_ms": 587337.8541666666, "avg_output_tokens_per_task": 10115.354166666666, "last_run_timestamp": "2026-08-08T16:03:43.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.5133333333333333, "accuracy_std_dev": 0.028803034820811636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12443134666666665, "avg_duration_per_task_ms": 52357.68, "avg_output_tokens_per_task": 2755.9866666666667, "last_run_timestamp": "2026-10-02T23:07:04.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.513333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03238348, "avg_duration_per_task_ms": 271942.88, "avg_output_tokens_per_task": 7084.96, "last_run_timestamp": "2026-10-01T08:05:50.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.512916625, "accuracy_std_dev": 0.03301683440662317, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.031192877499999997, "avg_duration_per_task_ms": 264931.19, "avg_output_tokens_per_task": 21003.91, "last_run_timestamp": "2026-10-24T19:23:13.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.5051021836734694, "accuracy_std_dev": 0.011159046957513885, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.019562938775510203, "avg_duration_per_task_ms": 1197227.9285714286, "avg_output_tokens_per_task": 100212.39285714286, "last_run_timestamp": "2026-08-29T17:16:15.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.4995211379310344, "accuracy_std_dev": 0.03521937973703148, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.16297116666666667, "avg_duration_per_task_ms": 2016445.1580459771, "avg_output_tokens_per_task": 38482.971264367814, "last_run_timestamp": "2026-10-17T19:35:49.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.4990475714285714, "accuracy_std_dev": 0.026047140242574162, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.025615628571428573, "avg_duration_per_task_ms": 596502.9742857143, "avg_output_tokens_per_task": 52103.73714285714, "last_run_timestamp": "2026-09-26T16:00:25.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.493333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14235776, "avg_duration_per_task_ms": 443941.74, "avg_output_tokens_per_task": 8833.66, "last_run_timestamp": "2026-08-06T00:00:02.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.48850570114942526, "accuracy_std_dev": 0.017439968302957548, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.031982068965517245, "avg_duration_per_task_ms": 642915.867816092, "avg_output_tokens_per_task": 59401.63505747126, "last_run_timestamp": "2026-10-23T09:26:25.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.4857621557788945, "accuracy_std_dev": 0.02560762584874622, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.028534012562814074, "avg_duration_per_task_ms": 352717.891959799, "avg_output_tokens_per_task": 60736.11306532663, "last_run_timestamp": "2026-10-03T16:14:03.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.48333349999999997, "accuracy_std_dev": 0.03666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04059527, "avg_duration_per_task_ms": 669013.45, "avg_output_tokens_per_task": 93900.79, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.48000024999999996, "accuracy_std_dev": 0.01414201777284628, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0673682, "avg_duration_per_task_ms": 287537.155, "avg_output_tokens_per_task": 4110.14, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011001320000000002, "avg_duration_per_task_ms": 80205.64, "avg_output_tokens_per_task": 3762.88, "last_run_timestamp": "2026-10-08T22:08:03.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.48, "accuracy_std_dev": 0.0312693389057076, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.050673248000000004, "avg_duration_per_task_ms": 91008.444, "avg_output_tokens_per_task": 2749.592, "last_run_timestamp": "2026-10-03T16:07:50.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07899766, "avg_duration_per_task_ms": 132704.5, "avg_output_tokens_per_task": 3161.38, "last_run_timestamp": "2026-10-04T03:00:16.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.47916700000000007, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.04362447916666667, "avg_duration_per_task_ms": 66953.94791666667, "avg_output_tokens_per_task": 2828.96875, "last_run_timestamp": "2026-08-15T14:08:14.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.47874697986577186, "accuracy_std_dev": 0.00975008705374242, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.16965920805369125, "avg_duration_per_task_ms": 2424316.9261744968, "avg_output_tokens_per_task": 45379.409395973154, "last_run_timestamp": "2026-10-17T19:38:30.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.47666650000000005, "accuracy_std_dev": 0.0166665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12559784000000002, "avg_duration_per_task_ms": 52928.96, "avg_output_tokens_per_task": 2668.55, "last_run_timestamp": "2026-10-01T19:04:26.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.4754382631578947, "accuracy_std_dev": 0.054999999999999966, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.01757382105263158, "avg_duration_per_task_ms": 167109.45263157896, "avg_output_tokens_per_task": 28512.663157894738, "last_run_timestamp": "2026-10-01T19:06:10.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.475, "accuracy_std_dev": 0.00986024566123988, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021282230000000003, "avg_duration_per_task_ms": 677068.455, "avg_output_tokens_per_task": 19681.38, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.47333349999999996, "accuracy_std_dev": 0.008164965814380372, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.024006935, "avg_duration_per_task_ms": 869541.065, "avg_output_tokens_per_task": 22816.51, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.47222181249999995, "accuracy_std_dev": 0.0011595000000000077, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.07886530208333332, "avg_duration_per_task_ms": 55709.895833333336, "avg_output_tokens_per_task": 3493.7291666666665, "last_run_timestamp": "2026-08-15T14:15:58.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.472, "accuracy_std_dev": 0.023999999999999994, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.034465088000000005, "avg_duration_per_task_ms": 43484.756, "avg_output_tokens_per_task": 3636.224, "last_run_timestamp": "2026-10-03T16:17:11.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4706666, "accuracy_std_dev": 0.020483090046182005, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023913492, "avg_duration_per_task_ms": 139319.672, "avg_output_tokens_per_task": 3726.148, "last_run_timestamp": "2026-10-03T16:13:56.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.47, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03551402, "avg_duration_per_task_ms": 437816.58, "avg_output_tokens_per_task": 20922.04, "last_run_timestamp": "2026-10-08T23:09:09.727Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.46820784971098267, "accuracy_std_dev": 0.04097451831844482, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.027368598265895955, "avg_duration_per_task_ms": 2298741.832369942, "avg_output_tokens_per_task": 136175.1647398844, "last_run_timestamp": "2026-10-24T19:05:04.624Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.4680002, "accuracy_std_dev": 0.03798255037987838, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.019017924000000002, "avg_duration_per_task_ms": 96994.328, "avg_output_tokens_per_task": 3628.408, "last_run_timestamp": "2026-10-03T16:12:20.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4644446666666666, "accuracy_std_dev": 0.029979285029648195, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04325750666666667, "avg_duration_per_task_ms": 738509.7133333334, "avg_output_tokens_per_task": 100007.46, "last_run_timestamp": "2026-10-02T23:22:06.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.4640002, "accuracy_std_dev": 0.04144079793343752, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.024943207999999998, "avg_duration_per_task_ms": 59536.72, "avg_output_tokens_per_task": 3736.6, "last_run_timestamp": "2026-10-03T16:10:49.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.46166674999999996, "accuracy_std_dev": 0.04010416645047619, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04448468, "avg_duration_per_task_ms": 738816.635, "avg_output_tokens_per_task": 95390.315, "last_run_timestamp": "2026-10-01T19:06:01.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011152899999999999, "avg_duration_per_task_ms": 87357.46, "avg_output_tokens_per_task": 9087.9, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 158124.84, "avg_output_tokens_per_task": 4515, "last_run_timestamp": "2026-10-01T19:06:51.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.46, "accuracy_std_dev": 0.006667000000000006, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03575541, "avg_duration_per_task_ms": 402821.3, "avg_output_tokens_per_task": 3754.72, "last_run_timestamp": "2026-08-15T08:15:42.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4566668333333333, "accuracy_std_dev": 0.03590122251129556, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.021435916666666662, "avg_duration_per_task_ms": 222009.41666666666, "avg_output_tokens_per_task": 3511.05, "last_run_timestamp": "2026-10-08T22:47:27.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.4541387986577181, "accuracy_std_dev": 0.02966298114934956, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.023203442953020136, "avg_duration_per_task_ms": 2013894.1275167784, "avg_output_tokens_per_task": 14181.58389261745, "last_run_timestamp": "2026-10-30T21:07:22.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.45333349999999994, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0063000800000000004, "avg_duration_per_task_ms": 38540.88, "avg_output_tokens_per_task": 1215.72, "last_run_timestamp": "2026-08-15T08:16:13.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.4524997500000001, "accuracy_std_dev": 0.02520749723172652, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02146402, "avg_duration_per_task_ms": 650786.745, "avg_output_tokens_per_task": 19908.01, "last_run_timestamp": "2026-10-09T00:21:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.4521073362068965, "accuracy_std_dev": 0.045992926219636195, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.015361428160919543, "avg_duration_per_task_ms": 128943.27873563218, "avg_output_tokens_per_task": 3310.485632183908, "last_run_timestamp": "2026-10-24T19:23:50.567Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.45121959349593493, "accuracy_std_dev": 0.052473835908574476, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.05198310569105691, "avg_duration_per_task_ms": 74014.12195121951, "avg_output_tokens_per_task": 3228.670731707317, "last_run_timestamp": "2026-10-03T16:07:05.221Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.45111100000000004, "accuracy_std_dev": 0.01571332689152746, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023389446666666668, "avg_duration_per_task_ms": 54689.44666666666, "avg_output_tokens_per_task": 2398.8933333333334, "last_run_timestamp": "2026-10-02T23:07:02.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.45095228571428575, "accuracy_std_dev": 0.024605712517417503, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04007145714285715, "avg_duration_per_task_ms": 282805.1142857143, "avg_output_tokens_per_task": 3547.6828571428573, "last_run_timestamp": "2026-10-24T19:41:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.45, "accuracy_std_dev": 0.041518865477893636, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05302523142857142, "avg_duration_per_task_ms": 144621.62857142856, "avg_output_tokens_per_task": 3704.842857142857, "last_run_timestamp": "2026-10-17T19:11:29.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.45, "accuracy_std_dev": 0.0033330000000000026, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08659784, "avg_duration_per_task_ms": 80537.63, "avg_output_tokens_per_task": 4189.08, "last_run_timestamp": "2026-08-15T08:11:06.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.4492010616438356, "accuracy_std_dev": 0.023681154664627318, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.05459431849315068, "avg_duration_per_task_ms": 157164.11643835617, "avg_output_tokens_per_task": 75683.87671232877, "last_run_timestamp": "2026-10-17T18:58:49.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.4466668, "accuracy_std_dev": 0.08033274135344816, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.20203069199999998, "avg_duration_per_task_ms": 115064.372, "avg_output_tokens_per_task": 3550.864, "last_run_timestamp": "2026-10-03T16:19:14.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.44333300000000003, "accuracy_std_dev": 0.009999999999999981, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.023192820000000003, "avg_duration_per_task_ms": 58714.73, "avg_output_tokens_per_task": 2330.08, "last_run_timestamp": "2026-10-01T19:04:38.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.4416665, "accuracy_std_dev": 0.017873086533948188, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.104832345, "avg_duration_per_task_ms": 890298.59, "avg_output_tokens_per_task": 21334.155, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.44, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03130282, "avg_duration_per_task_ms": 680376.23, "avg_output_tokens_per_task": 16688.33, "last_run_timestamp": "2026-10-01T16:00:49.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.43999974999999997, "accuracy_std_dev": 0.047081383440671115, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050863820000000004, "avg_duration_per_task_ms": 37018.795, "avg_output_tokens_per_task": 2487.295, "last_run_timestamp": "2026-10-03T16:12:29.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4377778333333333, "accuracy_std_dev": 0.040490818956963014, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.053731056666666666, "avg_duration_per_task_ms": 129281.94, "avg_output_tokens_per_task": 3456.7733333333335, "last_run_timestamp": "2026-10-03T16:19:54.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.43631422764227645, "accuracy_std_dev": 0.03840736243170051, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.057368723577235776, "avg_duration_per_task_ms": 251181.19512195123, "avg_output_tokens_per_task": 2298.849593495935, "last_run_timestamp": "2026-10-05T20:09:54.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.4353741734693877, "accuracy_std_dev": 0.03760835138885378, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.04635305102040816, "avg_duration_per_task_ms": 63193.6581632653, "avg_output_tokens_per_task": 2275.091836734694, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.435, "accuracy_std_dev": 0.020749805890658335, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.074401825, "avg_duration_per_task_ms": 778012.545, "avg_output_tokens_per_task": 20313.29, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.4322221666666667, "accuracy_std_dev": 0.01651776073823432, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.21270556333333332, "avg_duration_per_task_ms": 46305.06, "avg_output_tokens_per_task": 2266.36, "last_run_timestamp": "2026-10-02T23:10:05.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.431111, "accuracy_std_dev": 0.00314238253559302, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.24174756666666666, "avg_duration_per_task_ms": 46576.24, "avg_output_tokens_per_task": 2473.9733333333334, "last_run_timestamp": "2026-10-02T23:09:02.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.4301802162162162, "accuracy_std_dev": 0.02412263108922131, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.06743897297297298, "avg_duration_per_task_ms": 360892.5135135135, "avg_output_tokens_per_task": 3318.9054054054054, "last_run_timestamp": "2026-10-08T22:54:33.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.42916675, "accuracy_std_dev": 0.025967332577442378, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.027418319999999996, "avg_duration_per_task_ms": 84447.825, "avg_output_tokens_per_task": 2784.11, "last_run_timestamp": "2026-10-03T16:12:20.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.4283335, "accuracy_std_dev": 0.02598098663927142, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.016813855, "avg_duration_per_task_ms": 118682.965, "avg_output_tokens_per_task": 2808.965, "last_run_timestamp": "2026-10-03T16:14:29.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.428, "accuracy_std_dev": 0.052391746594287164, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.49939273599999995, "avg_duration_per_task_ms": 111261.628, "avg_output_tokens_per_task": 3387.876, "last_run_timestamp": "2026-10-03T16:14:53.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.42799980000000004, "accuracy_std_dev": 0.012220332784339386, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.039161852, "avg_duration_per_task_ms": 146989.988, "avg_output_tokens_per_task": 3492.532, "last_run_timestamp": "2026-10-03T16:18:20.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.4276191428571428, "accuracy_std_dev": 0.03190820308513861, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.007563445714285715, "avg_duration_per_task_ms": 175165.4, "avg_output_tokens_per_task": 4466.028571428571, "last_run_timestamp": "2026-10-24T19:16:50.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.42666699999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0178204, "avg_duration_per_task_ms": 1378502.34, "avg_output_tokens_per_task": 126873.58, "last_run_timestamp": "2026-10-30T02:24:16.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.42666699999999996, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02972828, "avg_duration_per_task_ms": 273042.46, "avg_output_tokens_per_task": 3546.89, "last_run_timestamp": "2026-08-15T08:02:48.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.42666675, "accuracy_std_dev": 0.014142017772846282, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.050337635, "avg_duration_per_task_ms": 158009.49, "avg_output_tokens_per_task": 2226.425, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.42666650000000006, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.21223042999999997, "avg_duration_per_task_ms": 51685.1, "avg_output_tokens_per_task": 2189.48, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.42592599999999997, "accuracy_std_dev": 0.03253776997070738, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.024220277777777778, "avg_duration_per_task_ms": 407107.03555555554, "avg_output_tokens_per_task": 3588.162222222222, "last_run_timestamp": "2026-10-24T19:25:31.387Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.4256754864864865, "accuracy_std_dev": 0.024259060786069653, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.03375386486486487, "avg_duration_per_task_ms": 525936.777027027, "avg_output_tokens_per_task": 5145.756756756757, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.4255553333333334, "accuracy_std_dev": 0.034462310778910675, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029985173333333334, "avg_duration_per_task_ms": 822675.42, "avg_output_tokens_per_task": 18074.12, "last_run_timestamp": "2026-10-08T22:41:10.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.42400019999999994, "accuracy_std_dev": 0.014514259655938366, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.024904308, "avg_duration_per_task_ms": 223225.116, "avg_output_tokens_per_task": 3537.724, "last_run_timestamp": "2026-10-03T16:13:22.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.4236811784511785, "accuracy_std_dev": 0.029725846878880795, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.05290695286195286, "avg_duration_per_task_ms": 528585.1717171717, "avg_output_tokens_per_task": 22153.03367003367, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.4204932602040816, "accuracy_std_dev": 0.025044575822109165, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.03310804591836735, "avg_duration_per_task_ms": 672143.8137755102, "avg_output_tokens_per_task": 6731.755102040816, "last_run_timestamp": "2026-10-24T19:18:23.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41888899999999996, "accuracy_std_dev": 0.014614444065147779, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01463076, "avg_duration_per_task_ms": 76549.69333333333, "avg_output_tokens_per_task": 8814.213333333333, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.4175171377551021, "accuracy_std_dev": 0.02667947172423725, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.057072859693877556, "avg_duration_per_task_ms": 211166.0357142857, "avg_output_tokens_per_task": 4534.619897959184, "last_run_timestamp": "2026-10-03T08:12:06.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.41751264974619284, "accuracy_std_dev": 0.027965463072500252, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.004796167512690355, "avg_duration_per_task_ms": 327290.3121827411, "avg_output_tokens_per_task": 4759.779187817259, "last_run_timestamp": "2026-10-24T19:54:38.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.416667, "accuracy_std_dev": 0.03, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.19647352999999998, "avg_duration_per_task_ms": 53166.02, "avg_output_tokens_per_task": 2086.91, "last_run_timestamp": "2026-10-01T16:01:13.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.411667, "accuracy_std_dev": 0.049749371855331, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.055448005, "avg_duration_per_task_ms": 142634.75, "avg_output_tokens_per_task": 3380.275, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.40836190721649485, "accuracy_std_dev": 0.024997748289707123, "run_count": 6, "total_questions": 291, "avg_cost_per_task_usd": 0.011245838487972509, "avg_duration_per_task_ms": 117422.78006872852, "avg_output_tokens_per_task": 3161.364261168385, "last_run_timestamp": "2026-10-03T16:17:21.424Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.406667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06754938, "avg_duration_per_task_ms": 114529.78, "avg_output_tokens_per_task": 3789.7, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.4066664000000001, "accuracy_std_dev": 0.025647109432448717, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07750359200000001, "avg_duration_per_task_ms": 118177.072, "avg_output_tokens_per_task": 3117.652, "last_run_timestamp": "2026-10-03T16:12:07.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.40533320000000006, "accuracy_std_dev": 0.01808612973966516, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.06744982, "avg_duration_per_task_ms": 383181.876, "avg_output_tokens_per_task": 10145.868, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.40389273722627733, "accuracy_std_dev": 0.01580206672840261, "run_count": 3, "total_questions": 137, "avg_cost_per_task_usd": 0.008183423357664232, "avg_duration_per_task_ms": 89294.24087591241, "avg_output_tokens_per_task": 1500.5474452554745, "last_run_timestamp": "2026-10-03T16:15:51.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.40380291275167784, "accuracy_std_dev": 0.008840018715415086, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.07104265771812081, "avg_duration_per_task_ms": 235544.13422818793, "avg_output_tokens_per_task": 8668.979865771813, "last_run_timestamp": "2026-10-02T15:44:48.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.40333325000000003, "accuracy_std_dev": 0.01914841159698371, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.023807599999999998, "avg_duration_per_task_ms": 77145.375, "avg_output_tokens_per_task": 2775.135, "last_run_timestamp": "2026-10-01T19:05:00.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.4026668, "accuracy_std_dev": 0.027519660851107883, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.027686715999999997, "avg_duration_per_task_ms": 276072.088, "avg_output_tokens_per_task": 4225.576, "last_run_timestamp": "2026-10-03T16:13:56.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.4, "accuracy_std_dev": 0.034769053930183376, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.076876536, "avg_duration_per_task_ms": 340404.576, "avg_output_tokens_per_task": 15764.94, "last_run_timestamp": "2026-10-03T16:12:29.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.39999980000000007, "accuracy_std_dev": 0.008432845839928543, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.036944032, "avg_duration_per_task_ms": 191861.204, "avg_output_tokens_per_task": 3680.784, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.39833349999999995, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06903712000000001, "avg_duration_per_task_ms": 315593.59, "avg_output_tokens_per_task": 7308.28, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.39466680000000004, "accuracy_std_dev": 0.019505181079907976, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.0621838, "avg_duration_per_task_ms": 95587.668, "avg_output_tokens_per_task": 2912.132, "last_run_timestamp": "2026-10-03T16:12:33.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.39166674999999995, "accuracy_std_dev": 0.00866020592639114, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015426675, "avg_duration_per_task_ms": 99199.855, "avg_output_tokens_per_task": 615.135, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.39132881081081083, "accuracy_std_dev": 0.025676864524422683, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.058435516891891894, "avg_duration_per_task_ms": 376057.5371621622, "avg_output_tokens_per_task": 3843.489864864865, "last_run_timestamp": "2026-10-08T23:21:09.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.3911292096774194, "accuracy_std_dev": 0.3156298317748815, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.005442725806451613, "avg_duration_per_task_ms": 343542.48387096776, "avg_output_tokens_per_task": 10219.729838709678, "last_run_timestamp": "2026-10-24T19:32:27.020Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.387500125, "accuracy_std_dev": 0.12580578865203054, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0213678575, "avg_duration_per_task_ms": 206487.0225, "avg_output_tokens_per_task": 3317.835, "last_run_timestamp": "2026-10-24T19:36:27.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.014510460000000001, "avg_duration_per_task_ms": 64332.54, "avg_output_tokens_per_task": 4106.96, "last_run_timestamp": "2026-10-09T15:37:38.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0592903, "avg_duration_per_task_ms": 203536.16, "avg_output_tokens_per_task": 5170.96, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.3866665, "accuracy_std_dev": 0.006666499999999992, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020927889999999998, "avg_duration_per_task_ms": 183788.07, "avg_output_tokens_per_task": 3842.25, "last_run_timestamp": "2026-08-15T08:02:44.458Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.38333325, "accuracy_std_dev": 0.007453597197830037, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.028690270000000004, "avg_duration_per_task_ms": 83258.84, "avg_output_tokens_per_task": 4293.12, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.3826666, "accuracy_std_dev": 0.01083190350030869, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.026093336, "avg_duration_per_task_ms": 200624.988, "avg_output_tokens_per_task": 4302.312, "last_run_timestamp": "2026-10-03T16:03:16.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.3773334, "accuracy_std_dev": 0.04552913987590804, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.05450766399999999, "avg_duration_per_task_ms": 92940.624, "avg_output_tokens_per_task": 3847.676, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.37599980000000005, "accuracy_std_dev": 0.032276543943861154, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.055068684, "avg_duration_per_task_ms": 70747.116, "avg_output_tokens_per_task": 4411.06, "last_run_timestamp": "2026-10-03T16:05:34.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.3738005794743429, "accuracy_std_dev": 0.022831389807948622, "run_count": 16, "total_questions": 799, "avg_cost_per_task_usd": 0.013108423028785983, "avg_duration_per_task_ms": 39230.8648310388, "avg_output_tokens_per_task": 749.793491864831, "last_run_timestamp": "2026-10-30T00:47:05.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.3688886666666667, "accuracy_std_dev": 0.01791611454963257, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03368235333333333, "avg_duration_per_task_ms": 150468.61333333334, "avg_output_tokens_per_task": 4460.053333333333, "last_run_timestamp": "2026-10-23T08:14:36.871Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.36833325, "accuracy_std_dev": 0.17853565236861657, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015193219999999999, "avg_duration_per_task_ms": 127682.09, "avg_output_tokens_per_task": 2917.505, "last_run_timestamp": "2026-10-03T16:12:29.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.3681779823677581, "accuracy_std_dev": 0.14087050789486985, "run_count": 16, "total_questions": 794, "avg_cost_per_task_usd": 0.038621142317380355, "avg_duration_per_task_ms": 585118.7216624685, "avg_output_tokens_per_task": 145279.47858942064, "last_run_timestamp": "2026-10-24T19:41:04.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.364498756097561, "accuracy_std_dev": 0.08717599304625098, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.42022371138211384, "avg_duration_per_task_ms": 436503.325203252, "avg_output_tokens_per_task": 17041.613821138213, "last_run_timestamp": "2026-10-10T18:31:45.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.36, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.041000339999999996, "avg_duration_per_task_ms": 322345.3, "avg_output_tokens_per_task": 4212.74, "last_run_timestamp": "2026-10-04T03:00:10.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.35714285714285715, "accuracy_std_dev": 0.029979379745928993, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.07267795758928572, "avg_duration_per_task_ms": 400336.21875, "avg_output_tokens_per_task": 8020.354910714285, "last_run_timestamp": "2026-10-08T23:34:52.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.35333325000000004, "accuracy_std_dev": 0.03448008281874479, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08867438999999999, "avg_duration_per_task_ms": 27089.57, "avg_output_tokens_per_task": 1029.7925, "last_run_timestamp": "2026-10-24T18:19:37.955Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.35175891959798994, "accuracy_std_dev": 0.022258587996757, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.0788611256281407, "avg_duration_per_task_ms": 211842.61055276383, "avg_output_tokens_per_task": 7349.442211055276, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.333333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.051354699999999996, "avg_duration_per_task_ms": 358714.54, "avg_output_tokens_per_task": 69285.14, "last_run_timestamp": "2026-10-04T03:00:10.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.326667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02872976, "avg_duration_per_task_ms": 133695.86, "avg_output_tokens_per_task": 3408.56, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.32416675, "accuracy_std_dev": 0.025806488741932716, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07527415750000001, "avg_duration_per_task_ms": 136836.5875, "avg_output_tokens_per_task": 7681.4025, "last_run_timestamp": "2026-10-03T08:23:09.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.3240741212121212, "accuracy_std_dev": 0.04532665948699506, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.04404032070707071, "avg_duration_per_task_ms": 93583.61111111111, "avg_output_tokens_per_task": 1344.4166666666667, "last_run_timestamp": "2026-10-24T18:56:17.284Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.32207219594594594, "accuracy_std_dev": 0.32266469334989745, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.010810280405405406, "avg_duration_per_task_ms": 61078.11824324324, "avg_output_tokens_per_task": 11341.706081081082, "last_run_timestamp": "2026-10-24T19:09:36.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.313333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15186704, "avg_duration_per_task_ms": 32635.52, "avg_output_tokens_per_task": 1294.54, "last_run_timestamp": "2026-10-04T03:00:16.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.30927829896907216, "accuracy_std_dev": 0.041318422501349206, "run_count": 4, "total_questions": 194, "avg_cost_per_task_usd": 0.016999237113402062, "avg_duration_per_task_ms": 169063.57731958764, "avg_output_tokens_per_task": 1549.3092783505156, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.30666649999999995, "accuracy_std_dev": 0.006666499999999992, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01779966, "avg_duration_per_task_ms": 55498.41, "avg_output_tokens_per_task": 3406.81, "last_run_timestamp": "2026-10-01T19:04:58.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.2847618571428572, "accuracy_std_dev": 0.02910620321918911, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1710796942857143, "avg_duration_per_task_ms": 139494.6, "avg_output_tokens_per_task": 1877.2057142857143, "last_run_timestamp": "2026-10-17T19:19:08.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.2844446666666667, "accuracy_std_dev": 0.02997950740459592, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.025635820000000004, "avg_duration_per_task_ms": 67806.75333333333, "avg_output_tokens_per_task": 4107.166666666667, "last_run_timestamp": "2026-10-02T23:13:47.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.2733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07945186, "avg_duration_per_task_ms": 632916.72, "avg_output_tokens_per_task": 5086.04, "last_run_timestamp": "2026-10-01T16:00:45.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.24333325000000003, "accuracy_std_dev": 0.08749614912204708, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17476605, "avg_duration_per_task_ms": 545108.995, "avg_output_tokens_per_task": 11742.65, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.24133339999999998, "accuracy_std_dev": 0.06591572936591082, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.013381287999999998, "avg_duration_per_task_ms": 21869.236, "avg_output_tokens_per_task": 1091.82, "last_run_timestamp": "2026-10-03T16:13:42.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.24, "accuracy_std_dev": 0.033333, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07327475, "avg_duration_per_task_ms": 662655.63, "avg_output_tokens_per_task": 5322.46, "last_run_timestamp": "2026-10-01T16:00:49.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.23180078160919543, "accuracy_std_dev": 0.19038523699508184, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.19703842528735632, "avg_duration_per_task_ms": 1194250.8362068965, "avg_output_tokens_per_task": 7552.1781609195405, "last_run_timestamp": "2026-10-03T08:09:23.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.2296295555555556, "accuracy_std_dev": 0.2407828888462297, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07424728444444444, "avg_duration_per_task_ms": 161317.51555555555, "avg_output_tokens_per_task": 18679.246666666666, "last_run_timestamp": "2026-10-24T19:07:57.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.22244883673469384, "accuracy_std_dev": 0.060049660254159636, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.004490742857142857, "avg_duration_per_task_ms": 16706.061224489797, "avg_output_tokens_per_task": 1551.4979591836734, "last_run_timestamp": "2026-10-24T19:30:38.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.21868997687861272, "accuracy_std_dev": 0.24138887954206054, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.009219916184971097, "avg_duration_per_task_ms": 788394.760115607, "avg_output_tokens_per_task": 61580.92485549133, "last_run_timestamp": "2026-10-24T19:33:50.345Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.2116665, "accuracy_std_dev": 0.12643621800042107, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.018232810000000002, "avg_duration_per_task_ms": 80959.52, "avg_output_tokens_per_task": 1422.615, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.186667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01693508, "avg_duration_per_task_ms": 26941.4, "avg_output_tokens_per_task": 1041.58, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.17942213775510205, "accuracy_std_dev": 0.009739601557558703, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.010118459183673469, "avg_duration_per_task_ms": 155344.85714285713, "avg_output_tokens_per_task": 4996.790816326531, "last_run_timestamp": "2026-10-01T19:04:52.650Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.17133340000000002, "accuracy_std_dev": 0.261806292768986, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.006590812, "avg_duration_per_task_ms": 17587.43, "avg_output_tokens_per_task": 3014.246, "last_run_timestamp": "2026-10-17T19:48:24.252Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.14, "accuracy_std_dev": 0.014142135623730954, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0072234, "avg_duration_per_task_ms": 76455.195, "avg_output_tokens_per_task": 2322.375, "last_run_timestamp": "2026-10-01T19:04:58.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.111111, "accuracy_std_dev": 0.01571332689152746, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.007342933333333333, "avg_duration_per_task_ms": 57269.24, "avg_output_tokens_per_task": 2382.0866666666666, "last_run_timestamp": "2026-10-02T23:07:59.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.11, "accuracy_std_dev": 0.009999999999999995, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.006762440000000001, "avg_duration_per_task_ms": 64306.02, "avg_output_tokens_per_task": 2312.51, "last_run_timestamp": "2026-10-01T19:04:36.194Z" } ] }