{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-09-30", "retrieved_at": "2026-09-30T01:45:37.914Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's non-zero-score runs from the newest 90 days: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2004, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2003321212121212, "avg_duration_per_task_ms": 124514.04545454546, "avg_output_tokens_per_task": 16571.29797979798, "last_run_timestamp": "2026-09-01T17:50:10.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra", "model_slug": "sakana/fugu-ultra", "model_name": "Fugu Ultra", "provider_name": "auto-routing", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 1.148739303030303, "avg_duration_per_task_ms": 328779.10101010103, "avg_output_tokens_per_task": 33271.606060606064, "last_run_timestamp": "2026-09-15T01:07:47.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.116425398989899, "avg_duration_per_task_ms": 44572.09595959596, "avg_output_tokens_per_task": 2176.570707070707, "last_run_timestamp": "2026-09-06T03:17:58.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.945286, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2012271818181818, "avg_duration_per_task_ms": 134249.20707070708, "avg_output_tokens_per_task": 16644.886363636364, "last_run_timestamp": "2026-09-01T17:53:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02981064646464646, "avg_duration_per_task_ms": 28082.027777777777, "avg_output_tokens_per_task": 7803.272727272727, "last_run_timestamp": "2026-09-01T13:28:27.192Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.20185933333333334, "avg_duration_per_task_ms": 126427.48484848485, "avg_output_tokens_per_task": 16698.565656565657, "last_run_timestamp": "2026-09-01T17:51:04.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11967219191919193, "avg_duration_per_task_ms": 38197.17171717172, "avg_output_tokens_per_task": 2017.3383838383838, "last_run_timestamp": "2026-09-26T10:54:50.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9436415317919074, "accuracy_std_dev": 0.0020135000000000014, "run_count": 2, "total_questions": 346, "avg_cost_per_task_usd": 0.09933485549132949, "avg_duration_per_task_ms": 198116.95664739885, "avg_output_tokens_per_task": 1767.9046242774566, "last_run_timestamp": "2026-09-06T03:05:46.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.942761, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.030509156565656566, "avg_duration_per_task_ms": 68136.79292929293, "avg_output_tokens_per_task": 7993.409090909091, "last_run_timestamp": "2026-09-01T13:25:46.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "typesafe/jev-router", "model_slug": "typesafe/jev-router", "model_name": "TypeSafe: Jev Router", "provider_name": "auto-routing", "accuracy": 0.939394, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.015926459595959598, "avg_duration_per_task_ms": 24665.858585858587, "avg_output_tokens_per_task": 2219.030303030303, "last_run_timestamp": "2026-09-28T18:11:34.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.9382716666666666, "accuracy_std_dev": 0.006918527315998818, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0262555, "avg_duration_per_task_ms": 19027.23400673401, "avg_output_tokens_per_task": 2243.60101010101, "last_run_timestamp": "2026-09-29T12:03:13.309Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.9377104999999999, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.30815713636363634, "avg_duration_per_task_ms": 160641.4873737374, "avg_output_tokens_per_task": 10142.272727272728, "last_run_timestamp": "2026-09-01T13:20:32.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.93771, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.4446258484848485, "avg_duration_per_task_ms": 116119.66666666667, "avg_output_tokens_per_task": 11825.727272727272, "last_run_timestamp": "2026-08-01T08:45:01.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.9351849999999999, "accuracy_std_dev": 0.006162785766193721, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1365803898989899, "avg_duration_per_task_ms": 176670.50606060607, "avg_output_tokens_per_task": 8896.30606060606, "last_run_timestamp": "2026-09-26T09:17:14.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.9335082598425197, "accuracy_std_dev": 0.011397999999999964, "run_count": 2, "total_questions": 381, "avg_cost_per_task_usd": 0.0213603280839895, "avg_duration_per_task_ms": 53235.17585301837, "avg_output_tokens_per_task": 7688.425196850394, "last_run_timestamp": "2026-09-01T13:29:18.783Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra-v2", "model_slug": "sakana/fugu-ultra-v2", "model_name": "Fugu Ultra V2", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.5025571414141414, "avg_duration_per_task_ms": 194915.41919191918, "avg_output_tokens_per_task": 13806.227272727272, "last_run_timestamp": "2026-09-15T01:07:48.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.12205839393939394, "avg_duration_per_task_ms": 394338.13636363635, "avg_output_tokens_per_task": 20077.80808080808, "last_run_timestamp": "2026-09-01T13:22:57.685Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9324968387096775, "accuracy_std_dev": 0.005080239036370865, "run_count": 3, "total_questions": 558, "avg_cost_per_task_usd": 0.0715788476702509, "avg_duration_per_task_ms": 64320.11827956989, "avg_output_tokens_per_task": 11534.012544802867, "last_run_timestamp": "2026-09-01T17:51:06.380Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.930976, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025067358585858585, "avg_duration_per_task_ms": 20515.378787878788, "avg_output_tokens_per_task": 2267.0252525252527, "last_run_timestamp": "2026-09-29T12:03:17.610Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.930976, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.32996825757575754, "avg_duration_per_task_ms": 166479.3383838384, "avg_output_tokens_per_task": 10868.631313131313, "last_run_timestamp": "2026-09-01T13:22:14.451Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9299660000000001, "accuracy_std_dev": 0.009009433123121587, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.32134650707070705, "avg_duration_per_task_ms": 152235.298989899, "avg_output_tokens_per_task": 9994.927272727273, "last_run_timestamp": "2026-09-01T13:24:49.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.929885, "accuracy_std_dev": null, "run_count": 1, "total_questions": 145, "avg_cost_per_task_usd": 0.09349187586206896, "avg_duration_per_task_ms": 204934.02068965518, "avg_output_tokens_per_task": 6095.813793103448, "last_run_timestamp": "2026-09-29T21:23:16.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.9294545733558178, "accuracy_std_dev": 0.004334827511626685, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.02681522428330523, "avg_duration_per_task_ms": 26517.369308600337, "avg_output_tokens_per_task": 2356.256323777403, "last_run_timestamp": "2026-09-23T15:41:43.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9292930000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14407252272727272, "avg_duration_per_task_ms": 58692.59848484849, "avg_output_tokens_per_task": 15884.012626262625, "last_run_timestamp": "2026-09-01T17:55:40.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.9292929999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02512639898989899, "avg_duration_per_task_ms": 21072.075757575756, "avg_output_tokens_per_task": 2272.929292929293, "last_run_timestamp": "2026-09-29T12:03:27.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.92887175, "accuracy_std_dev": 0.005239941859171739, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06947331565656567, "avg_duration_per_task_ms": 67244.06060606061, "avg_output_tokens_per_task": 11635.200757575758, "last_run_timestamp": "2026-09-01T17:56:37.544Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06683859343434344, "avg_duration_per_task_ms": 59600.07575757576, "avg_output_tokens_per_task": 11674.472222222223, "last_run_timestamp": "2026-09-01T17:57:17.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9283648630849219, "accuracy_std_dev": 0.017830944261311557, "run_count": 4, "total_questions": 577, "avg_cost_per_task_usd": 0.09151112998266897, "avg_duration_per_task_ms": 252153.69497400348, "avg_output_tokens_per_task": 7044.975736568457, "last_run_timestamp": "2026-09-26T11:09:52.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.9279571938392187, "accuracy_std_dev": 0.00835201542708194, "run_count": 22, "total_questions": 3993, "avg_cost_per_task_usd": 0.15530522213874282, "avg_duration_per_task_ms": 142386.56699223642, "avg_output_tokens_per_task": 8728.15927873779, "last_run_timestamp": "2026-09-26T11:50:40.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.927609, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.026096929292929293, "avg_duration_per_task_ms": 19999.954545454544, "avg_output_tokens_per_task": 2369.09595959596, "last_run_timestamp": "2026-09-29T12:06:06.771Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.927609, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14183102777777779, "avg_duration_per_task_ms": 79740.97727272728, "avg_output_tokens_per_task": 15646.176767676769, "last_run_timestamp": "2026-09-01T18:06:42.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.92613625, "accuracy_std_dev": 0.007949293313716653, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.09945460101010102, "avg_duration_per_task_ms": 230595.33712121213, "avg_output_tokens_per_task": 9430.997474747475, "last_run_timestamp": "2026-09-26T08:53:13.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9259257500000001, "accuracy_std_dev": 0.004610545920766882, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.1489979318181818, "avg_duration_per_task_ms": 84816.23358585859, "avg_output_tokens_per_task": 15665.375, "last_run_timestamp": "2026-09-01T17:51:30.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.925347078125, "accuracy_std_dev": 0.005810499999999996, "run_count": 2, "total_questions": 384, "avg_cost_per_task_usd": 0.3411057265625, "avg_duration_per_task_ms": 143482.140625, "avg_output_tokens_per_task": 10207.984375, "last_run_timestamp": "2026-09-01T13:21:58.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9245122977099236, "accuracy_std_dev": 0.009051464284164208, "run_count": 4, "total_questions": 786, "avg_cost_per_task_usd": 0.12044641603053435, "avg_duration_per_task_ms": 355037.55470737914, "avg_output_tokens_per_task": 19323.84987277354, "last_run_timestamp": "2026-09-29T03:08:09.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.026419161616161615, "avg_duration_per_task_ms": 21367.934343434343, "avg_output_tokens_per_task": 2243.222222222222, "last_run_timestamp": "2026-09-23T15:27:41.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "unbiased/pareto", "model_slug": "unbiased/pareto", "model_name": "Pareto", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03377419696969697, "avg_duration_per_task_ms": 114763.54545454546, "avg_output_tokens_per_task": 4358.363636363636, "last_run_timestamp": "2026-09-29T03:11:33.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.9239580000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.0028846625000000002, "avg_duration_per_task_ms": 871679.5875, "avg_output_tokens_per_task": 9877.29375, "last_run_timestamp": "2026-09-27T19:46:05.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.922559, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.4976751313131313, "avg_duration_per_task_ms": 91843.67676767676, "avg_output_tokens_per_task": 12034.212121212122, "last_run_timestamp": "2026-08-01T08:41:32.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9224121127739175, "accuracy_std_dev": 0.007223723392406446, "run_count": 10, "total_questions": 1871, "avg_cost_per_task_usd": 0.15054908498129344, "avg_duration_per_task_ms": 122412.57936932122, "avg_output_tokens_per_task": 9889.19080705505, "last_run_timestamp": "2026-09-26T08:55:05.759Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9215687647058822, "accuracy_std_dev": 0.0038094199325123296, "run_count": 3, "total_questions": 561, "avg_cost_per_task_usd": 0.10569379500891266, "avg_duration_per_task_ms": 54609.55971479501, "avg_output_tokens_per_task": 3484.6114081996434, "last_run_timestamp": "2026-09-01T13:22:25.850Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9212118, "accuracy_std_dev": 0.008675973568424486, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1201016404040404, "avg_duration_per_task_ms": 61346.69292929293, "avg_output_tokens_per_task": 3579.7959595959596, "last_run_timestamp": "2026-09-01T13:28:53.908Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.920875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.4329253232323232, "avg_duration_per_task_ms": 113725.84848484848, "avg_output_tokens_per_task": 11493.873737373737, "last_run_timestamp": "2026-08-01T08:53:39.678Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.9205012222222222, "accuracy_std_dev": 0.00850227984821038, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1576766610549944, "avg_duration_per_task_ms": 236410.12177328844, "avg_output_tokens_per_task": 12595.254208754208, "last_run_timestamp": "2026-09-26T08:55:21.775Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.920034, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.12737118686868687, "avg_duration_per_task_ms": 194690.12121212122, "avg_output_tokens_per_task": 8293.338383838383, "last_run_timestamp": "2026-09-29T12:27:43.930Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.9200335, "accuracy_std_dev": 0.007575500000000013, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07599968055555556, "avg_duration_per_task_ms": 289463.85101010103, "avg_output_tokens_per_task": 24687.122474747473, "last_run_timestamp": "2026-09-26T09:34:25.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.919865, "accuracy_std_dev": 0.007939302840930074, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.16831854444444444, "avg_duration_per_task_ms": 345204.0070707071, "avg_output_tokens_per_task": 9582.316161616161, "last_run_timestamp": "2026-09-26T09:45:25.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.9192883359550563, "accuracy_std_dev": 0.004632004101898027, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.19511707022471914, "avg_duration_per_task_ms": 338842.3713483146, "avg_output_tokens_per_task": 44188.06966292135, "last_run_timestamp": "2026-09-26T08:56:52.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191915, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07126612626262625, "avg_duration_per_task_ms": 64525.257575757576, "avg_output_tokens_per_task": 3520.8156565656564, "last_run_timestamp": "2026-09-01T13:21:08.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9188843954372623, "accuracy_std_dev": 0.012617923846160294, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.13335884410646387, "avg_duration_per_task_ms": 335309.8307984791, "avg_output_tokens_per_task": 10503.390367553866, "last_run_timestamp": "2026-09-26T08:52:39.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.9186867999999999, "accuracy_std_dev": 0.0068919500984844775, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13311589696969697, "avg_duration_per_task_ms": 546514.5777777778, "avg_output_tokens_per_task": 36924.677777777775, "last_run_timestamp": "2026-09-26T09:34:49.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.9179851756329115, "accuracy_std_dev": 0.013506707135345767, "run_count": 10, "total_questions": 1896, "avg_cost_per_task_usd": 0.12114161814345992, "avg_duration_per_task_ms": 120275.13765822785, "avg_output_tokens_per_task": 8135.254219409283, "last_run_timestamp": "2026-09-26T09:59:38.434Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.9175826670976084, "accuracy_std_dev": 0.014195212289272039, "run_count": 8, "total_questions": 1547, "avg_cost_per_task_usd": 0.008065983839689723, "avg_duration_per_task_ms": 489574.17323852616, "avg_output_tokens_per_task": 20043.881060116353, "last_run_timestamp": "2026-09-26T09:23:12.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.917508, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025625949494949497, "avg_duration_per_task_ms": 37548.47474747475, "avg_output_tokens_per_task": 2404.0757575757575, "last_run_timestamp": "2026-09-23T15:32:55.695Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.9166664999999999, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026023550505050508, "avg_duration_per_task_ms": 20014.84595959596, "avg_output_tokens_per_task": 2239.1363636363635, "last_run_timestamp": "2026-09-29T12:03:59.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.9164982, "accuracy_std_dev": 0.005280767383629021, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.09286130606060607, "avg_duration_per_task_ms": 636505.3202020202, "avg_output_tokens_per_task": 40298.016161616164, "last_run_timestamp": "2026-09-26T09:33:50.566Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.914983, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07349445707070708, "avg_duration_per_task_ms": 64386.71464646464, "avg_output_tokens_per_task": 3569.0353535353534, "last_run_timestamp": "2026-09-01T13:22:37.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9147023333333334, "accuracy_std_dev": 0.011850896290529652, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1591313367003367, "avg_duration_per_task_ms": 538081.3703703703, "avg_output_tokens_per_task": 39867.37205387205, "last_run_timestamp": "2026-09-26T09:14:59.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.9139881818181818, "accuracy_std_dev": 0.01312802668698594, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.032382866391184574, "avg_duration_per_task_ms": 222132.83562901744, "avg_output_tokens_per_task": 26800.957759412304, "last_run_timestamp": "2026-09-26T16:05:15.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.9139752900807381, "accuracy_std_dev": 0.006586842457168369, "run_count": 9, "total_questions": 1734, "avg_cost_per_task_usd": 0.14062165570934257, "avg_duration_per_task_ms": 294457.8246828143, "avg_output_tokens_per_task": 9226.012687427912, "last_run_timestamp": "2026-09-26T09:18:47.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9127381666666666, "accuracy_std_dev": 0.0041711655212369175, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.27201272474747473, "avg_duration_per_task_ms": 357401.7491582492, "avg_output_tokens_per_task": 38560.36363636364, "last_run_timestamp": "2026-09-26T09:19:10.819Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.9124577500000001, "accuracy_std_dev": 0.006628136630117118, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0378646856060606, "avg_duration_per_task_ms": 130990.08964646465, "avg_output_tokens_per_task": 28457.30808080808, "last_run_timestamp": "2026-09-26T08:54:48.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/switchyard", "model_slug": "nvidia/switchyard", "model_name": "NVIDIA: Switchyard", "provider_name": "auto-routing", "accuracy": 0.9118049954065226, "accuracy_std_dev": 0.030664109703108882, "run_count": 11, "total_questions": 2177, "avg_cost_per_task_usd": 0.018931710151584753, "avg_duration_per_task_ms": 61807.37344970142, "avg_output_tokens_per_task": 3220.4230592558565, "last_run_timestamp": "2026-09-28T14:41:42.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.9107743333333334, "accuracy_std_dev": 0.004956316261992276, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03197664478114478, "avg_duration_per_task_ms": 194795.97138047137, "avg_output_tokens_per_task": 26470.969696969696, "last_run_timestamp": "2026-09-26T08:52:40.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.9104936666666668, "accuracy_std_dev": 0.05077924822427192, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.10505623232323233, "avg_duration_per_task_ms": 246993.5, "avg_output_tokens_per_task": 9288.036195286195, "last_run_timestamp": "2026-09-26T09:35:57.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9102129999999999, "accuracy_std_dev": 0.002861334420627323, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0912948888888889, "avg_duration_per_task_ms": 155548.53703703705, "avg_output_tokens_per_task": 27869.387205387204, "last_run_timestamp": "2026-09-01T13:31:50.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.9096159477085782, "accuracy_std_dev": 0.03256740651027582, "run_count": 18, "total_questions": 3404, "avg_cost_per_task_usd": 0.1496695376028202, "avg_duration_per_task_ms": 473661.7364864865, "avg_output_tokens_per_task": 8839.687720329024, "last_run_timestamp": "2026-09-26T11:50:25.798Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.9091866447534767, "accuracy_std_dev": 0.0045401682512876225, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.029416860935524654, "avg_duration_per_task_ms": 453238.0587863464, "avg_output_tokens_per_task": 30418.960809102402, "last_run_timestamp": "2026-09-26T16:21:44.172Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2221867676767677, "avg_duration_per_task_ms": 298883.5202020202, "avg_output_tokens_per_task": 29352.99494949495, "last_run_timestamp": "2026-09-27T00:06:19.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.9090566691616767, "accuracy_std_dev": 0.011036068624917007, "run_count": 7, "total_questions": 1336, "avg_cost_per_task_usd": 0.012388138473053893, "avg_duration_per_task_ms": 304706.8982035928, "avg_output_tokens_per_task": 16929.82260479042, "last_run_timestamp": "2026-09-26T08:52:35.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.9087729427112877, "accuracy_std_dev": 0.01605553275354013, "run_count": 9, "total_questions": 1763, "avg_cost_per_task_usd": 0.188728044242768, "avg_duration_per_task_ms": 517068.7197958026, "avg_output_tokens_per_task": 12546.453204764606, "last_run_timestamp": "2026-09-26T08:17:01.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.9086699999999999, "accuracy_std_dev": 0.0024899934738870384, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.035269609848484845, "avg_duration_per_task_ms": 184713.31186868687, "avg_output_tokens_per_task": 29215.651515151516, "last_run_timestamp": "2026-09-26T09:27:25.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.908249, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1657261111111111, "avg_duration_per_task_ms": 467040.702020202, "avg_output_tokens_per_task": 43784.85353535353, "last_run_timestamp": "2026-09-28T06:30:03.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.9074072500000001, "accuracy_std_dev": 0.004610545920766882, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.030798316919191916, "avg_duration_per_task_ms": 142600.18308080808, "avg_output_tokens_per_task": 28269.325757575756, "last_run_timestamp": "2026-09-26T09:35:41.692Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9073550816901409, "accuracy_std_dev": 0.02204460352471743, "run_count": 11, "total_questions": 2130, "avg_cost_per_task_usd": 0.03663173568075117, "avg_duration_per_task_ms": 263398.2685446009, "avg_output_tokens_per_task": 30725.946948356806, "last_run_timestamp": "2026-09-26T16:06:12.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.9069262857142857, "accuracy_std_dev": 0.01966827796016641, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.13709101803751803, "avg_duration_per_task_ms": 571292.8715728716, "avg_output_tokens_per_task": 38969.107503607505, "last_run_timestamp": "2026-09-26T08:52:35.549Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.905724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.054357919191919186, "avg_duration_per_task_ms": 31472.4797979798, "avg_output_tokens_per_task": 2509.6414141414143, "last_run_timestamp": "2026-09-26T08:54:32.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.9057234999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03042029292929293, "avg_duration_per_task_ms": 203300.4116161616, "avg_output_tokens_per_task": 29610.87373737374, "last_run_timestamp": "2026-09-26T10:10:49.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.9054903294117648, "accuracy_std_dev": 0.002950830881105991, "run_count": 3, "total_questions": 425, "avg_cost_per_task_usd": 0.0986494188235294, "avg_duration_per_task_ms": 590399.5082352941, "avg_output_tokens_per_task": 26203.308235294116, "last_run_timestamp": "2026-09-05T08:31:24.281Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9053029638429752, "accuracy_std_dev": 0.01776711826408548, "run_count": 10, "total_questions": 1936, "avg_cost_per_task_usd": 0.03572506508264463, "avg_duration_per_task_ms": 311302.8930785124, "avg_output_tokens_per_task": 29769.41993801653, "last_run_timestamp": "2026-09-26T16:32:47.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.9051626666666666, "accuracy_std_dev": 0.00649574344389378, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.028482462962962966, "avg_duration_per_task_ms": 244979.07912457912, "avg_output_tokens_per_task": 29233.41919191919, "last_run_timestamp": "2026-09-26T08:52:19.672Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.9041170761421321, "accuracy_std_dev": 0.007233580779177685, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.013811796954314722, "avg_duration_per_task_ms": 592278.3874788494, "avg_output_tokens_per_task": 28886.412859560067, "last_run_timestamp": "2026-09-26T09:15:49.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.9040400000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02929719696969697, "avg_duration_per_task_ms": 146549.648989899, "avg_output_tokens_per_task": 27342.593434343435, "last_run_timestamp": "2026-09-01T13:36:23.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9037085815423515, "accuracy_std_dev": 0.00877916433950291, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.033949044247787614, "avg_duration_per_task_ms": 139050.61061946902, "avg_output_tokens_per_task": 28840.025284450065, "last_run_timestamp": "2026-09-26T08:51:27.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.903601419310345, "accuracy_std_dev": 0.02045915087022377, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.03446190114942529, "avg_duration_per_task_ms": 316983.98942528735, "avg_output_tokens_per_task": 28739.95632183908, "last_run_timestamp": "2026-09-26T16:08:07.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.9031985, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14676221212121213, "avg_duration_per_task_ms": 106397.32070707071, "avg_output_tokens_per_task": 9655.194444444445, "last_run_timestamp": "2026-09-01T17:54:59.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.9031985, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.043649184343434344, "avg_duration_per_task_ms": 141329.08080808082, "avg_output_tokens_per_task": 28078.323232323233, "last_run_timestamp": "2026-09-01T13:21:17.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.902848790174002, "accuracy_std_dev": 0.012183292931305548, "run_count": 10, "total_questions": 1954, "avg_cost_per_task_usd": 0.030339729273285567, "avg_duration_per_task_ms": 232833.84339815762, "avg_output_tokens_per_task": 25157.37871033777, "last_run_timestamp": "2026-09-26T16:18:02.009Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.90256725, "accuracy_std_dev": 0.031891435832171314, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.042016968434343424, "avg_duration_per_task_ms": 260202.41792929292, "avg_output_tokens_per_task": 31470.176136363636, "last_run_timestamp": "2026-09-26T16:00:24.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9024125477178423, "accuracy_std_dev": 0.026053497227534046, "run_count": 11, "total_questions": 2169, "avg_cost_per_task_usd": 0.04220562056247118, "avg_duration_per_task_ms": 254266.19502074688, "avg_output_tokens_per_task": 31038.156754264637, "last_run_timestamp": "2026-09-26T16:06:21.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.9023565, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.053309800505050506, "avg_duration_per_task_ms": 26495.010101010103, "avg_output_tokens_per_task": 2451.373737373737, "last_run_timestamp": "2026-09-26T08:59:34.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.9013467199999999, "accuracy_std_dev": 0.005345045073860445, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.21570517737373737, "avg_duration_per_task_ms": 94985.5094949495, "avg_output_tokens_per_task": 7862.312121212121, "last_run_timestamp": "2026-09-26T08:55:06.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.901199375, "accuracy_std_dev": 0.005700202593713214, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.2040564772727273, "avg_duration_per_task_ms": 98886.75126262626, "avg_output_tokens_per_task": 7886.399621212121, "last_run_timestamp": "2026-09-26T09:36:04.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.9006730000000001, "accuracy_std_dev": 0.007008969420202443, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1542171708754209, "avg_duration_per_task_ms": 127676.21296296296, "avg_output_tokens_per_task": 9507.647306397306, "last_run_timestamp": "2026-09-01T17:57:22.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.900673, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03341542424242424, "avg_duration_per_task_ms": 161489.9797979798, "avg_output_tokens_per_task": 27673.237373737375, "last_run_timestamp": "2026-09-28T18:09:11.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.9006724572127138, "accuracy_std_dev": 0.009841934649752557, "run_count": 10, "total_questions": 1636, "avg_cost_per_task_usd": 0.02167028850855746, "avg_duration_per_task_ms": 209667.61246943765, "avg_output_tokens_per_task": 17928.619193154034, "last_run_timestamp": "2026-09-26T17:07:03.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.9006104712571067, "accuracy_std_dev": 0.058973902356121696, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.14550911939355657, "avg_duration_per_task_ms": 301854.2046746684, "avg_output_tokens_per_task": 10187.655716993051, "last_run_timestamp": "2026-09-26T10:02:09.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.9004862222222222, "accuracy_std_dev": 0.044346110676956096, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.14017589562289562, "avg_duration_per_task_ms": 217013.79966329967, "avg_output_tokens_per_task": 9267.047699214367, "last_run_timestamp": "2026-09-26T09:14:25.270Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.9003924999999999, "accuracy_std_dev": 0.01706955011855908, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02751066750841751, "avg_duration_per_task_ms": 211685.4351851852, "avg_output_tokens_per_task": 25746.074915824916, "last_run_timestamp": "2026-09-26T08:50:52.330Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.900252375, "accuracy_std_dev": 0.024011889023656058, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.15187610795454545, "avg_duration_per_task_ms": 607070.2234848485, "avg_output_tokens_per_task": 38443.12626262626, "last_run_timestamp": "2026-09-26T09:23:06.229Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.8999515714285714, "accuracy_std_dev": 0.004917020144563225, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.23012572438672438, "avg_duration_per_task_ms": 111331.89033189033, "avg_output_tokens_per_task": 9001.132756132756, "last_run_timestamp": "2026-09-26T08:54:49.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.8998315, "accuracy_std_dev": 0.007575499999999957, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13947029797979799, "avg_duration_per_task_ms": 103780.42929292929, "avg_output_tokens_per_task": 9170.424242424242, "last_run_timestamp": "2026-09-01T17:56:45.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.8996633999999999, "accuracy_std_dev": 0.005792805834826492, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007343192929292929, "avg_duration_per_task_ms": 219337.0404040404, "avg_output_tokens_per_task": 25809.762626262625, "last_run_timestamp": "2026-08-08T09:30:52.456Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8995508333333332, "accuracy_std_dev": 0.004524373673657892, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.031289763468013466, "avg_duration_per_task_ms": 215945.1616161616, "avg_output_tokens_per_task": 26542.147306397306, "last_run_timestamp": "2026-09-26T09:19:06.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.8993638888888887, "accuracy_std_dev": 0.005599097039194869, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.009368053310886646, "avg_duration_per_task_ms": 186510.51515151514, "avg_output_tokens_per_task": 26068.07014590348, "last_run_timestamp": "2026-09-26T08:51:16.648Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.8993387243502051, "accuracy_std_dev": 0.026386926357751014, "run_count": 8, "total_questions": 1462, "avg_cost_per_task_usd": 0.027840691518467853, "avg_duration_per_task_ms": 740625.9062927497, "avg_output_tokens_per_task": 24883.40560875513, "last_run_timestamp": "2026-09-12T16:17:57.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.8989897499999999, "accuracy_std_dev": 0.007241239789393777, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.12451013636363638, "avg_duration_per_task_ms": 29637.002525252527, "avg_output_tokens_per_task": 2284.373737373737, "last_run_timestamp": "2026-09-26T08:53:11.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8987091666666667, "accuracy_std_dev": 0.006255284576970791, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05856475252525253, "avg_duration_per_task_ms": 31561.38973063973, "avg_output_tokens_per_task": 2534.496632996633, "last_run_timestamp": "2026-09-26T09:18:55.607Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.8986660000000002, "accuracy_std_dev": 0.006701794386580333, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.22345247746697744, "avg_duration_per_task_ms": 95310.8921911422, "avg_output_tokens_per_task": 8135.330031080031, "last_run_timestamp": "2026-09-26T09:16:27.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.8978795217391303, "accuracy_std_dev": 0.02199333682676437, "run_count": 7, "total_questions": 1242, "avg_cost_per_task_usd": 0.15500312399355878, "avg_duration_per_task_ms": 574496.192431562, "avg_output_tokens_per_task": 37507.40821256039, "last_run_timestamp": "2026-09-26T09:25:58.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.8978675, "accuracy_std_dev": 0.004085377899697062, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.057460788720538725, "avg_duration_per_task_ms": 25029.281986531987, "avg_output_tokens_per_task": 2489.7180134680134, "last_run_timestamp": "2026-09-26T09:20:45.216Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.897348959390863, "accuracy_std_dev": 0.009933136206097672, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.021320504230118442, "avg_duration_per_task_ms": 601175.5499153977, "avg_output_tokens_per_task": 17639.832487309646, "last_run_timestamp": "2026-09-26T09:15:29.044Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8973064285714285, "accuracy_std_dev": 0.009132616193107635, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.19710293722943723, "avg_duration_per_task_ms": 95577.20057720058, "avg_output_tokens_per_task": 7683.466089466089, "last_run_timestamp": "2026-09-26T09:05:17.040Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.8973061999999999, "accuracy_std_dev": 0.012997001706547546, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03622809090909091, "avg_duration_per_task_ms": 162801.12121212122, "avg_output_tokens_per_task": 27180.768686868687, "last_run_timestamp": "2026-09-26T08:53:11.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.8970957500000001, "accuracy_std_dev": 0.013079356862533413, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.20200678535353536, "avg_duration_per_task_ms": 97703.5946969697, "avg_output_tokens_per_task": 7866.200757575758, "last_run_timestamp": "2026-09-26T08:52:41.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.8967507774936062, "accuracy_std_dev": 0.0073637026710600875, "run_count": 18, "total_questions": 3519, "avg_cost_per_task_usd": 0.12523853253765274, "avg_duration_per_task_ms": 58154.57402671214, "avg_output_tokens_per_task": 4750.331628303496, "last_run_timestamp": "2026-09-26T08:53:17.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.8964645000000001, "accuracy_std_dev": 0.007575499999999957, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03375818181818182, "avg_duration_per_task_ms": 56405.70202020202, "avg_output_tokens_per_task": 3599.8333333333335, "last_run_timestamp": "2026-09-01T13:22:46.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.8961193926247291, "accuracy_std_dev": 0.016375418175062377, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.11571196312364425, "avg_duration_per_task_ms": 621953.1062906724, "avg_output_tokens_per_task": 36359.131597975414, "last_run_timestamp": "2026-09-26T09:20:04.760Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.8960958972972972, "accuracy_std_dev": 0.005877557334660528, "run_count": 3, "total_questions": 555, "avg_cost_per_task_usd": 0.1555473855855856, "avg_duration_per_task_ms": 83534.15135135135, "avg_output_tokens_per_task": 9299.446846846848, "last_run_timestamp": "2026-09-01T17:56:10.791Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.8959085241877256, "accuracy_std_dev": 0.035618009581258374, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.11392027942238267, "avg_duration_per_task_ms": 563860.525631769, "avg_output_tokens_per_task": 38531.1083032491, "last_run_timestamp": "2026-09-26T09:10:17.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.895623, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.016944118686868685, "avg_duration_per_task_ms": 304656.29545454547, "avg_output_tokens_per_task": 21473.98232323232, "last_run_timestamp": "2026-09-26T08:26:07.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.89562275, "accuracy_std_dev": 0.0051888143816771525, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.057555659090909095, "avg_duration_per_task_ms": 24846.121212121212, "avg_output_tokens_per_task": 2521.655303030303, "last_run_timestamp": "2026-09-26T09:36:25.776Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.8954697272727271, "accuracy_std_dev": 0.006025463303060488, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.20493744031221303, "avg_duration_per_task_ms": 100289.02800734619, "avg_output_tokens_per_task": 7987.1299357208445, "last_run_timestamp": "2026-09-26T08:51:26.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.8953789065420559, "accuracy_std_dev": 0.010355017014034617, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.012003740654205609, "avg_duration_per_task_ms": 284653.5062305296, "avg_output_tokens_per_task": 27098.440809968848, "last_run_timestamp": "2026-09-26T08:56:10.606Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8953486936299293, "accuracy_std_dev": 0.019178869929169434, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.03527696764408493, "avg_duration_per_task_ms": 158313.13549039434, "avg_output_tokens_per_task": 26466.745197168857, "last_run_timestamp": "2026-09-26T09:36:01.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.8950614444444446, "accuracy_std_dev": 0.007975755054623836, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.12023811111111112, "avg_duration_per_task_ms": 57861.70202020202, "avg_output_tokens_per_task": 4595.748597081931, "last_run_timestamp": "2026-09-26T08:56:21.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.8949365645569619, "accuracy_std_dev": 0.017816727566741866, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.021203940084388185, "avg_duration_per_task_ms": 226768.21476793248, "avg_output_tokens_per_task": 26685.461603375526, "last_run_timestamp": "2026-09-26T08:50:36.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.8946420862998922, "accuracy_std_dev": 0.007735019467331668, "run_count": 5, "total_questions": 927, "avg_cost_per_task_usd": 0.006440504854368933, "avg_duration_per_task_ms": 428399.7303128371, "avg_output_tokens_per_task": 13363.50377562028, "last_run_timestamp": "2026-09-26T08:57:06.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8939478279932548, "accuracy_std_dev": 0.009294310315828362, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.22570986565486229, "avg_duration_per_task_ms": 108477.89263631254, "avg_output_tokens_per_task": 8804.630691399663, "last_run_timestamp": "2026-09-26T08:50:43.725Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.8934859242520689, "accuracy_std_dev": 0.016678015536459194, "run_count": 8, "total_questions": 1571, "avg_cost_per_task_usd": 0.11211757733927435, "avg_duration_per_task_ms": 393101.5945257798, "avg_output_tokens_per_task": 7662.472310630172, "last_run_timestamp": "2026-09-26T09:51:40.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.8931287407407407, "accuracy_std_dev": 0.007059243819500409, "run_count": 27, "total_questions": 5346, "avg_cost_per_task_usd": 0.2046095759446315, "avg_duration_per_task_ms": 71087.34867190423, "avg_output_tokens_per_task": 7454.418069584736, "last_run_timestamp": "2026-09-26T09:18:54.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8930976666666668, "accuracy_std_dev": 0.010182556173290759, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05509197895622895, "avg_duration_per_task_ms": 49604.97474747475, "avg_output_tokens_per_task": 3808.635521885522, "last_run_timestamp": "2026-09-01T13:23:50.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8930975, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.054297787878787876, "avg_duration_per_task_ms": 31463.27525252525, "avg_output_tokens_per_task": 2503.689393939394, "last_run_timestamp": "2026-09-26T08:53:18.330Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.892567903763697, "accuracy_std_dev": 0.02022831814050264, "run_count": 11, "total_questions": 2099, "avg_cost_per_task_usd": 0.15797696188661267, "avg_duration_per_task_ms": 338559.5883754169, "avg_output_tokens_per_task": 39638.616484040016, "last_run_timestamp": "2026-09-19T08:51:08.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.8920512903225806, "accuracy_std_dev": 0.019911726970242915, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.13115581593927894, "avg_duration_per_task_ms": 517762.9652118912, "avg_output_tokens_per_task": 37601.20493358634, "last_run_timestamp": "2026-09-26T09:00:46.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8918440425531915, "accuracy_std_dev": 0.0082731599766957, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.050205347517730495, "avg_duration_per_task_ms": 44870.67021276596, "avg_output_tokens_per_task": 3654.1524822695037, "last_run_timestamp": "2026-09-01T13:21:31.062Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.8914139999999999, "accuracy_std_dev": 0.023312104420236266, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.086096711489899, "avg_duration_per_task_ms": 172501.05744949495, "avg_output_tokens_per_task": 29063.86047979798, "last_run_timestamp": "2026-09-26T16:08:44.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.8909932500000002, "accuracy_std_dev": 0.010197164639619184, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00789814393939394, "avg_duration_per_task_ms": 268244.18434343435, "avg_output_tokens_per_task": 27958.415404040403, "last_run_timestamp": "2026-08-08T10:06:09.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8905724999999999, "accuracy_std_dev": 0.011784499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.027024376262626264, "avg_duration_per_task_ms": 278293.2196969697, "avg_output_tokens_per_task": 26273.641414141413, "last_run_timestamp": "2026-09-19T09:00:31.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.890572, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.21863568181818183, "avg_duration_per_task_ms": 106364.56818181818, "avg_output_tokens_per_task": 8520.704545454546, "last_run_timestamp": "2026-09-01T13:30:25.730Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.890572, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1261595202020202, "avg_duration_per_task_ms": 738082.0303030303, "avg_output_tokens_per_task": 50200.232323232325, "last_run_timestamp": "2026-09-23T06:12:17.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.8905389900990098, "accuracy_std_dev": 0.007335499999999995, "run_count": 2, "total_questions": 303, "avg_cost_per_task_usd": 0.01631713201320132, "avg_duration_per_task_ms": 272632.9339933993, "avg_output_tokens_per_task": 22974.214521452144, "last_run_timestamp": "2026-09-26T09:25:09.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.8897369109414758, "accuracy_std_dev": 0.022583806568842405, "run_count": 6, "total_questions": 1179, "avg_cost_per_task_usd": 0.1931469830364716, "avg_duration_per_task_ms": 1259659.2798982188, "avg_output_tokens_per_task": 56717.179813401184, "last_run_timestamp": "2026-08-29T18:18:56.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.8897304999999999, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0318945303030303, "avg_duration_per_task_ms": 52439.469696969696, "avg_output_tokens_per_task": 3386.712121212121, "last_run_timestamp": "2026-09-01T13:38:25.421Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.8897192682926828, "accuracy_std_dev": 0.012292154908071387, "run_count": 13, "total_questions": 2542, "avg_cost_per_task_usd": 0.0036178985051140834, "avg_duration_per_task_ms": 277074.0971675846, "avg_output_tokens_per_task": 19808.066876475215, "last_run_timestamp": "2026-09-26T09:01:36.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.888889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00832349494949495, "avg_duration_per_task_ms": 86259.9494949495, "avg_output_tokens_per_task": 13087.217171717171, "last_run_timestamp": "2026-09-24T09:11:45.985Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.8888888430379747, "accuracy_std_dev": 0.009622693519820156, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.21254848649789027, "avg_duration_per_task_ms": 102777.66371308017, "avg_output_tokens_per_task": 8273.845147679325, "last_run_timestamp": "2026-09-26T09:15:06.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.8887592307692308, "accuracy_std_dev": 0.013361790161926698, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.012804753690753689, "avg_duration_per_task_ms": 213849.3986013986, "avg_output_tokens_per_task": 24454.132867132867, "last_run_timestamp": "2026-09-26T08:57:03.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.888552, "accuracy_std_dev": 0.03117712682079604, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1316316707070707, "avg_duration_per_task_ms": 39126.25959595959, "avg_output_tokens_per_task": 2423.617171717172, "last_run_timestamp": "2026-09-26T08:53:20.595Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.8884639560229446, "accuracy_std_dev": 0.012263269453369714, "run_count": 11, "total_questions": 2092, "avg_cost_per_task_usd": 0.09539477390057362, "avg_duration_per_task_ms": 534326.7394837476, "avg_output_tokens_per_task": 29086.82982791587, "last_run_timestamp": "2026-09-26T08:56:57.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8883868300395255, "accuracy_std_dev": 0.008563760488459709, "run_count": 27, "total_questions": 5313, "avg_cost_per_task_usd": 0.1336053077357425, "avg_duration_per_task_ms": 59533.9862601167, "avg_output_tokens_per_task": 4780.512516469038, "last_run_timestamp": "2026-09-26T09:19:13.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.8876862857142855, "accuracy_std_dev": 0.018697250017157118, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.13258516161616163, "avg_duration_per_task_ms": 538569.6168831169, "avg_output_tokens_per_task": 33498.70923520924, "last_run_timestamp": "2026-09-26T09:29:09.038Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.8875662777777777, "accuracy_std_dev": 0.03225232181080149, "run_count": 4, "total_questions": 756, "avg_cost_per_task_usd": 0.0045816917989418, "avg_duration_per_task_ms": 604811.5727513228, "avg_output_tokens_per_task": 27586.727513227514, "last_run_timestamp": "2026-09-26T08:58:10.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.8872987222222223, "accuracy_std_dev": 0.006285923036662124, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.24028202581369246, "avg_duration_per_task_ms": 84935.12850729517, "avg_output_tokens_per_task": 8938.721380471381, "last_run_timestamp": "2026-09-26T08:53:09.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.8872055000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20363688131313135, "avg_duration_per_task_ms": 99426.9191919192, "avg_output_tokens_per_task": 7920.752525252526, "last_run_timestamp": "2026-09-01T13:21:53.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.887153, "accuracy_std_dev": null, "run_count": 1, "total_questions": 192, "avg_cost_per_task_usd": 0.112435484375, "avg_duration_per_task_ms": 652216.9166666666, "avg_output_tokens_per_task": 32066.067708333332, "last_run_timestamp": "2026-09-29T02:34:51.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.8868242601351352, "accuracy_std_dev": 0.009599924258960477, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.20454106644144146, "avg_duration_per_task_ms": 100059.57263513513, "avg_output_tokens_per_task": 7955.086148648648, "last_run_timestamp": "2026-09-26T08:50:54.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.8866439999999999, "accuracy_std_dev": 0.012472582491208459, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008770978114478115, "avg_duration_per_task_ms": 61645.40740740741, "avg_output_tokens_per_task": 12897.501683501683, "last_run_timestamp": "2026-09-24T09:11:57.282Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8864601803278688, "accuracy_std_dev": 0.006211138077858382, "run_count": 3, "total_questions": 549, "avg_cost_per_task_usd": 0.009764420765027322, "avg_duration_per_task_ms": 50630.12750455373, "avg_output_tokens_per_task": 7246.790528233151, "last_run_timestamp": "2026-09-01T17:51:11.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.8864530691399664, "accuracy_std_dev": 0.01029973730112676, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.023206354974704888, "avg_duration_per_task_ms": 81414.45531197301, "avg_output_tokens_per_task": 7880.962900505902, "last_run_timestamp": "2026-09-01T17:55:01.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.8863635000000001, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0069983762626262625, "avg_duration_per_task_ms": 635651.6565656565, "avg_output_tokens_per_task": 21974.18181818182, "last_run_timestamp": "2026-09-26T10:42:27.487Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8862431428571429, "accuracy_std_dev": 0.008624548475610979, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.016605583694083693, "avg_duration_per_task_ms": 131101.75468975469, "avg_output_tokens_per_task": 25668.465367965367, "last_run_timestamp": "2026-09-26T08:53:15.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.8860826666666668, "accuracy_std_dev": 0.004199386807089298, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0278726632996633, "avg_duration_per_task_ms": 157091.45622895623, "avg_output_tokens_per_task": 23053.37037037037, "last_run_timestamp": "2026-09-26T09:47:15.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.8860007604485219, "accuracy_std_dev": 0.0277911335645022, "run_count": 5, "total_questions": 981, "avg_cost_per_task_usd": 0.04559343119266055, "avg_duration_per_task_ms": 698776.9031600407, "avg_output_tokens_per_task": 26722.172273190623, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-max", "model_slug": "sakana/fugu-max", "model_name": "Fugu Max", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07555469191919191, "avg_duration_per_task_ms": 225091.94949494948, "avg_output_tokens_per_task": 12279.550505050505, "last_run_timestamp": "2026-09-15T01:07:47.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.8849736038095237, "accuracy_std_dev": 0.03742100947662959, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.1364461853968254, "avg_duration_per_task_ms": 794770.7498412698, "avg_output_tokens_per_task": 34155.479365079365, "last_run_timestamp": "2026-09-26T10:09:19.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8849202417582418, "accuracy_std_dev": 0.01938538920596302, "run_count": 3, "total_questions": 546, "avg_cost_per_task_usd": 0.04324962454212454, "avg_duration_per_task_ms": 205056.15750915752, "avg_output_tokens_per_task": 33761.00366300366, "last_run_timestamp": "2026-09-26T10:02:16.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.88468, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009158123737373737, "avg_duration_per_task_ms": 142387.94696969696, "avg_output_tokens_per_task": 32237.108585858587, "last_run_timestamp": "2026-09-26T09:33:13.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.8845707161572053, "accuracy_std_dev": 0.02143221713073307, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.16085208820960697, "avg_duration_per_task_ms": 756655.7283842795, "avg_output_tokens_per_task": 40813.194759825325, "last_run_timestamp": "2026-09-16T00:37:42.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.8841129772439951, "accuracy_std_dev": 0.0033388888402431233, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.011443802781289506, "avg_duration_per_task_ms": 364670.14664981037, "avg_output_tokens_per_task": 22004.87104930468, "last_run_timestamp": "2026-09-26T12:21:31.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.8839537178510998, "accuracy_std_dev": 0.013099282603342133, "run_count": 12, "total_questions": 2364, "avg_cost_per_task_usd": 0.00725531429780034, "avg_duration_per_task_ms": 489249.17766497465, "avg_output_tokens_per_task": 25997.4179357022, "last_run_timestamp": "2026-09-26T08:56:41.992Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03956430303030303, "avg_duration_per_task_ms": 264270.07575757575, "avg_output_tokens_per_task": 34221.91414141414, "last_run_timestamp": "2026-09-28T18:58:44.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11161540404040404, "avg_duration_per_task_ms": 930549.2727272727, "avg_output_tokens_per_task": 44407.63636363636, "last_run_timestamp": "2026-09-23T15:38:01.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8835014000000001, "accuracy_std_dev": 0.025562678362018323, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.14118799797979797, "avg_duration_per_task_ms": 41635.47171717172, "avg_output_tokens_per_task": 2614.5828282828284, "last_run_timestamp": "2026-09-26T08:54:00.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8832771111111111, "accuracy_std_dev": 0.009589207202132645, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02110603142536476, "avg_duration_per_task_ms": 350549.90852974186, "avg_output_tokens_per_task": 24400.115039281707, "last_run_timestamp": "2026-09-26T08:58:18.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.8829965, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13198183333333333, "avg_duration_per_task_ms": 254155.68181818182, "avg_output_tokens_per_task": 43876.38636363636, "last_run_timestamp": "2026-09-01T17:57:36.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.882716, "accuracy_std_dev": 0.006496204173720757, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.027370449494949496, "avg_duration_per_task_ms": 130101.09932659933, "avg_output_tokens_per_task": 22617.942760942762, "last_run_timestamp": "2026-09-26T10:08:37.923Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8824913999999999, "accuracy_std_dev": 0.008205959897537906, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015855806060606062, "avg_duration_per_task_ms": 286632.0282828283, "avg_output_tokens_per_task": 24219.753535353535, "last_run_timestamp": "2026-09-26T09:30:12.511Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.8823229999999999, "accuracy_std_dev": 0.01560213662932099, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.11433169949494948, "avg_duration_per_task_ms": 635262.6303030303, "avg_output_tokens_per_task": 29892.36919191919, "last_run_timestamp": "2026-09-26T09:11:00.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8821944899391673, "accuracy_std_dev": 0.012966199129335096, "run_count": 22, "total_questions": 4274, "avg_cost_per_task_usd": 0.19237118413664017, "avg_duration_per_task_ms": 381859.63991576975, "avg_output_tokens_per_task": 30958.449695835283, "last_run_timestamp": "2026-09-26T08:58:33.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.8821546666666666, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02880540235690236, "avg_duration_per_task_ms": 99924.20707070707, "avg_output_tokens_per_task": 23871.781144781144, "last_run_timestamp": "2026-09-26T09:49:42.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8821545714285715, "accuracy_std_dev": 0.014256633562332045, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.18132904473304476, "avg_duration_per_task_ms": 297976.3398268398, "avg_output_tokens_per_task": 45509.72438672439, "last_run_timestamp": "2026-09-26T09:19:29.410Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8818613645976111, "accuracy_std_dev": 0.010002367123265743, "run_count": 27, "total_questions": 5107, "avg_cost_per_task_usd": 0.1356296686900333, "avg_duration_per_task_ms": 62377.717642451535, "avg_output_tokens_per_task": 4931.54278441355, "last_run_timestamp": "2026-09-26T08:58:07.474Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8809521930501931, "accuracy_std_dev": 0.01086493577983783, "run_count": 4, "total_questions": 777, "avg_cost_per_task_usd": 0.021439528957528958, "avg_duration_per_task_ms": 189160.3075933076, "avg_output_tokens_per_task": 23673.939510939512, "last_run_timestamp": "2026-09-29T03:31:47.451Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.8806582222222221, "accuracy_std_dev": 0.005109133929167741, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.005640715488215489, "avg_duration_per_task_ms": 362315.8209876543, "avg_output_tokens_per_task": 28732.147025813694, "last_run_timestamp": "2026-09-26T09:34:56.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.880471, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0032475000000000004, "avg_duration_per_task_ms": 58949.292929292926, "avg_output_tokens_per_task": 5746.025252525253, "last_run_timestamp": "2026-09-23T06:09:34.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8799936573033706, "accuracy_std_dev": 0.07692276680482653, "run_count": 12, "total_questions": 2136, "avg_cost_per_task_usd": 0.008731384363295881, "avg_duration_per_task_ms": 144796.46582397004, "avg_output_tokens_per_task": 23515.395599250936, "last_run_timestamp": "2026-09-26T08:54:04.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8798644568527919, "accuracy_std_dev": 0.004527248931623837, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.0029938477157360403, "avg_duration_per_task_ms": 316818.90016920475, "avg_output_tokens_per_task": 18321.88663282572, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto", "model_slug": "openrouter/auto", "model_name": "Auto Router", "provider_name": "auto-routing", "accuracy": 0.8795957999999999, "accuracy_std_dev": 0.026351833738091142, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08917041111111111, "avg_duration_per_task_ms": 196788.44343434344, "avg_output_tokens_per_task": 37445.42121212121, "last_run_timestamp": "2026-09-29T14:28:17.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.8793486666666668, "accuracy_std_dev": 0.0031744380430068155, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.07590014478114479, "avg_duration_per_task_ms": 323507.5353535354, "avg_output_tokens_per_task": 20962.156565656565, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.8792515918367347, "accuracy_std_dev": 0.00992611788026112, "run_count": 9, "total_questions": 1764, "avg_cost_per_task_usd": 0.004467375283446712, "avg_duration_per_task_ms": 383208.2947845805, "avg_output_tokens_per_task": 18476.153628117914, "last_run_timestamp": "2026-09-26T10:33:25.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.879189, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.020723571428571428, "avg_duration_per_task_ms": 141095.54497354498, "avg_output_tokens_per_task": 17734.21164021164, "last_run_timestamp": "2026-09-15T00:33:26.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8790546552363299, "accuracy_std_dev": 0.030844451113161732, "run_count": 6, "total_questions": 1079, "avg_cost_per_task_usd": 0.1346450278035218, "avg_duration_per_task_ms": 284974.0537534754, "avg_output_tokens_per_task": 31126.302131603337, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13332395959595958, "avg_duration_per_task_ms": 138478.24242424243, "avg_output_tokens_per_task": 9426.934343434343, "last_run_timestamp": "2026-08-01T08:50:28.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8787875, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0075271136363636365, "avg_duration_per_task_ms": 80896.84595959596, "avg_output_tokens_per_task": 8213.79797979798, "last_run_timestamp": "2026-09-01T17:51:13.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8787801424227066, "accuracy_std_dev": 0.016401176693323544, "run_count": 11, "total_questions": 1973, "avg_cost_per_task_usd": 0.011341015205271159, "avg_duration_per_task_ms": 261740.6791687785, "avg_output_tokens_per_task": 24571.48302078054, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.8787658893709328, "accuracy_std_dev": 0.007021239756217274, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.008512674620390456, "avg_duration_per_task_ms": 290385.6428054953, "avg_output_tokens_per_task": 30108.634851771512, "last_run_timestamp": "2026-08-29T19:12:25.584Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8785529767441861, "accuracy_std_dev": 0.101852, "run_count": 2, "total_questions": 258, "avg_cost_per_task_usd": 0.1083256472868217, "avg_duration_per_task_ms": 620596.2751937985, "avg_output_tokens_per_task": 16165.639534883721, "last_run_timestamp": "2026-09-01T16:19:34.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8784510000000001, "accuracy_std_dev": 0.011536704780828876, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.022268736363636365, "avg_duration_per_task_ms": 229673.4696969697, "avg_output_tokens_per_task": 24972.11919191919, "last_run_timestamp": "2026-09-26T09:19:28.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.8784135555555556, "accuracy_std_dev": 0.017145903852336865, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.023216065656565654, "avg_duration_per_task_ms": 214871.89786756455, "avg_output_tokens_per_task": 24869.92312008979, "last_run_timestamp": "2026-09-26T09:39:38.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.8783669000000001, "accuracy_std_dev": 0.03386573186408348, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.14244708787878788, "avg_duration_per_task_ms": 338071.06212121213, "avg_output_tokens_per_task": 33002.009090909094, "last_run_timestamp": "2026-09-26T08:53:37.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8783643322981367, "accuracy_std_dev": 0.020055520558938385, "run_count": 10, "total_questions": 1932, "avg_cost_per_task_usd": 0.1293156180124224, "avg_duration_per_task_ms": 63254.61180124224, "avg_output_tokens_per_task": 4948.346273291925, "last_run_timestamp": "2026-09-26T08:52:39.749Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.8779459999999999, "accuracy_std_dev": 0.014310000000000045, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13208357575757576, "avg_duration_per_task_ms": 257170.34595959596, "avg_output_tokens_per_task": 43907.46464646464, "last_run_timestamp": "2026-09-01T17:57:59.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.8776173653429603, "accuracy_std_dev": 0.01555889568113131, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.08483577111913358, "avg_duration_per_task_ms": 220187.85559566788, "avg_output_tokens_per_task": 21593.077978339352, "last_run_timestamp": "2026-09-26T10:06:10.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8771045000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14050905303030303, "avg_duration_per_task_ms": 185398.66919191918, "avg_output_tokens_per_task": 46713.32070707071, "last_run_timestamp": "2026-09-01T17:54:30.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8771039999999999, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007407005050505051, "avg_duration_per_task_ms": 78924.21717171717, "avg_output_tokens_per_task": 8106.714646464647, "last_run_timestamp": "2026-09-01T17:50:43.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.8770211787234045, "accuracy_std_dev": 0.03195742141593335, "run_count": 6, "total_questions": 1175, "avg_cost_per_task_usd": 0.0032351353191489365, "avg_duration_per_task_ms": 354647.9063829787, "avg_output_tokens_per_task": 26251.109787234043, "last_run_timestamp": "2026-09-19T08:54:45.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.8768792503888027, "accuracy_std_dev": 0.0068076105343578604, "run_count": 4, "total_questions": 643, "avg_cost_per_task_usd": 0.007664079315707621, "avg_duration_per_task_ms": 987610.4805598756, "avg_output_tokens_per_task": 30419.066874027994, "last_run_timestamp": "2026-09-26T09:19:44.251Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8765966933797908, "accuracy_std_dev": 0.020189124617118887, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.004125730836236933, "avg_duration_per_task_ms": 453615.28397212544, "avg_output_tokens_per_task": 16054.564459930314, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.8762625, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008190393939393938, "avg_duration_per_task_ms": 699109.8106060605, "avg_output_tokens_per_task": 28884.636363636364, "last_run_timestamp": "2026-07-18T08:31:23.912Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.8762625, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.15284478282828282, "avg_duration_per_task_ms": 339007.8914141414, "avg_output_tokens_per_task": 34511.28535353536, "last_run_timestamp": "2026-07-18T08:26:40.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8757013333333332, "accuracy_std_dev": 0.01234889446693734, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00532792760942761, "avg_duration_per_task_ms": 293763.77946127945, "avg_output_tokens_per_task": 28613.604377104377, "last_run_timestamp": "2026-08-29T18:22:03.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.8756311250000001, "accuracy_std_dev": 0.007549310638023532, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0497966994949495, "avg_duration_per_task_ms": 300230.047979798, "avg_output_tokens_per_task": 21153.447601010103, "last_run_timestamp": "2026-08-29T18:36:08.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.8754206666666666, "accuracy_std_dev": 0.00901385812081723, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.061844909090909096, "avg_duration_per_task_ms": 262173.4595959596, "avg_output_tokens_per_task": 22965.14478114478, "last_run_timestamp": "2026-09-26T09:19:06.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.8748596666666667, "accuracy_std_dev": 0.006198499675100621, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.003131114478114478, "avg_duration_per_task_ms": 32869.14646464647, "avg_output_tokens_per_task": 5711.222222222223, "last_run_timestamp": "2026-09-23T06:08:44.627Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.8745317312013828, "accuracy_std_dev": 0.011186512454241005, "run_count": 6, "total_questions": 1157, "avg_cost_per_task_usd": 0.0043274416594641305, "avg_duration_per_task_ms": 185028.52031114954, "avg_output_tokens_per_task": 21870.51080380294, "last_run_timestamp": "2026-09-26T08:53:59.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.8744972514285714, "accuracy_std_dev": 0.006555065674728234, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.12269054984126984, "avg_duration_per_task_ms": 985188.8444444444, "avg_output_tokens_per_task": 43152.0253968254, "last_run_timestamp": "2026-08-29T19:07:02.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8740740000000001, "accuracy_std_dev": 0.03472083287595504, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1182012676767677, "avg_duration_per_task_ms": 28405.98686868687, "avg_output_tokens_per_task": 2155.238383838384, "last_run_timestamp": "2026-09-26T08:52:17.920Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.874074, "accuracy_std_dev": 0.004178433821421624, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07029311111111111, "avg_duration_per_task_ms": 171365.84242424244, "avg_output_tokens_per_task": 11443.449494949495, "last_run_timestamp": "2026-09-26T09:10:25.930Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8737373, "accuracy_std_dev": 0.01570253542616605, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.11062107323232323, "avg_duration_per_task_ms": 229265.31616161615, "avg_output_tokens_per_task": 31458.501515151514, "last_run_timestamp": "2026-09-26T08:52:34.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.8737372500000001, "accuracy_std_dev": 0.019341996831700202, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12990520643939396, "avg_duration_per_task_ms": 63171.816287878784, "avg_output_tokens_per_task": 4978.773358585859, "last_run_timestamp": "2026-09-26T08:54:39.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.8727692590673575, "accuracy_std_dev": 0.006680537619749563, "run_count": 3, "total_questions": 579, "avg_cost_per_task_usd": 0.10616132815198619, "avg_duration_per_task_ms": 1070108.6269430052, "avg_output_tokens_per_task": 42555.205526770296, "last_run_timestamp": "2026-08-01T17:53:10.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.8726002303664923, "accuracy_std_dev": 0.013852891908270337, "run_count": 3, "total_questions": 573, "avg_cost_per_task_usd": 0.10812300000000001, "avg_duration_per_task_ms": 1136520.2198952879, "avg_output_tokens_per_task": 43220.56020942408, "last_run_timestamp": "2026-08-01T17:46:48.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.8724287134608445, "accuracy_std_dev": 0.032129376762123, "run_count": 14, "total_questions": 2771, "avg_cost_per_task_usd": 0.1523031241429087, "avg_duration_per_task_ms": 425564.390111873, "avg_output_tokens_per_task": 11498.582100324793, "last_run_timestamp": "2026-09-26T09:12:17.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.872292240506329, "accuracy_std_dev": 0.013194491052666223, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.00790528016877637, "avg_duration_per_task_ms": 421318.1687763713, "avg_output_tokens_per_task": 21316.46075949367, "last_run_timestamp": "2026-09-26T09:05:47.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8720540000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.22419905555555555, "avg_duration_per_task_ms": 276263.6666666667, "avg_output_tokens_per_task": 22325.227272727272, "last_run_timestamp": "2026-09-01T13:30:31.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8714926666666667, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03188070202020202, "avg_duration_per_task_ms": 240511.38888888888, "avg_output_tokens_per_task": 26418.183501683503, "last_run_timestamp": "2026-09-26T08:53:40.686Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.871212, "accuracy_std_dev": 0.010943000000000036, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.25165717424242423, "avg_duration_per_task_ms": 248967.9292929293, "avg_output_tokens_per_task": 23863.48232323232, "last_run_timestamp": "2026-09-01T13:53:50.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8711222842639593, "accuracy_std_dev": 0.0010748023074035804, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.04708872588832487, "avg_duration_per_task_ms": 168365.97292724196, "avg_output_tokens_per_task": 39058.98984771573, "last_run_timestamp": "2026-09-26T10:21:09.898Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.8710750330630069, "accuracy_std_dev": 0.028110456226890915, "run_count": 9, "total_questions": 1603, "avg_cost_per_task_usd": 0.09870775982532752, "avg_duration_per_task_ms": 221158.11790393014, "avg_output_tokens_per_task": 26829.437305053027, "last_run_timestamp": "2026-09-26T09:19:13.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8710435000000001, "accuracy_std_dev": 0.015469940324707153, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.007902119696969698, "avg_duration_per_task_ms": 203714.64696969697, "avg_output_tokens_per_task": 28019.556565656567, "last_run_timestamp": "2026-09-26T09:36:46.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.8710116425661915, "accuracy_std_dev": 0.01025225770257459, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.025059682281059068, "avg_duration_per_task_ms": 328330.6405295316, "avg_output_tokens_per_task": 20642.19348268839, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8709859808450705, "accuracy_std_dev": 0.00995822887470397, "run_count": 9, "total_questions": 1775, "avg_cost_per_task_usd": 0.007974392676056338, "avg_duration_per_task_ms": 311914.2202816901, "avg_output_tokens_per_task": 28209.84957746479, "last_run_timestamp": "2026-09-26T09:58:11.632Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.87037, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013891080808080807, "avg_duration_per_task_ms": 291446.38383838383, "avg_output_tokens_per_task": 27719.68181818182, "last_run_timestamp": "2026-09-29T02:39:06.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "WandB", "accuracy": 0.870085, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.11565565128205128, "avg_duration_per_task_ms": 2243159.2153846156, "avg_output_tokens_per_task": 32886.94358974359, "last_run_timestamp": "2026-07-11T08:42:46.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8699951448047163, "accuracy_std_dev": 0.03969212027542979, "run_count": 14, "total_questions": 2714, "avg_cost_per_task_usd": 0.00531859358879882, "avg_duration_per_task_ms": 114563.89867354458, "avg_output_tokens_per_task": 20095.400515843772, "last_run_timestamp": "2026-09-26T08:53:37.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.8699495, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00502325, "avg_duration_per_task_ms": 389434.8611111111, "avg_output_tokens_per_task": 17944.10101010101, "last_run_timestamp": "2026-08-08T10:05:07.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.8698091666666667, "accuracy_std_dev": 0.01883877588394628, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.07924426515151516, "avg_duration_per_task_ms": 527347.8063973064, "avg_output_tokens_per_task": 34630.43771043771, "last_run_timestamp": "2026-08-29T19:21:48.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.8697745034965034, "accuracy_std_dev": 0.09942524551683361, "run_count": 11, "total_questions": 2145, "avg_cost_per_task_usd": 0.026464635431235433, "avg_duration_per_task_ms": 212770.69836829836, "avg_output_tokens_per_task": 23008.030303030304, "last_run_timestamp": "2026-09-26T16:07:03.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8695795853658537, "accuracy_std_dev": 0.014776870412912198, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.010841884146341461, "avg_duration_per_task_ms": 398387.18699186994, "avg_output_tokens_per_task": 22967.41768292683, "last_run_timestamp": "2026-09-26T09:27:32.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10538721464646464, "avg_duration_per_task_ms": 581981.0101010101, "avg_output_tokens_per_task": 34695.7398989899, "last_run_timestamp": "2026-08-08T09:03:54.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.86938268, "accuracy_std_dev": 0.008067879753335774, "run_count": 7, "total_questions": 1350, "avg_cost_per_task_usd": 0.004681328148148148, "avg_duration_per_task_ms": 490155.47185185185, "avg_output_tokens_per_task": 28757.50888888889, "last_run_timestamp": "2026-08-29T19:32:50.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.869318125, "accuracy_std_dev": 0.03295688160474795, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10640666414141413, "avg_duration_per_task_ms": 513815.1142676768, "avg_output_tokens_per_task": 34567.331439393936, "last_run_timestamp": "2026-09-26T08:56:11.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.8693140356415479, "accuracy_std_dev": 0.004604825312647615, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.11175445315682281, "avg_duration_per_task_ms": 722023.3411405295, "avg_output_tokens_per_task": 31928.803462321794, "last_run_timestamp": "2026-08-29T18:36:08.021Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.8692809647058825, "accuracy_std_dev": 0.009104176033557385, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.0047775359477124175, "avg_duration_per_task_ms": 482494.045751634, "avg_output_tokens_per_task": 26353.58431372549, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.8691076666666667, "accuracy_std_dev": 0.07237401544446428, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.12576101936026934, "avg_duration_per_task_ms": 1221265.2171717172, "avg_output_tokens_per_task": 55812.33249158249, "last_run_timestamp": "2026-08-29T20:15:38.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.8689024817073171, "accuracy_std_dev": 0.04756926554684933, "run_count": 3, "total_questions": 328, "avg_cost_per_task_usd": 0.014946170731707318, "avg_duration_per_task_ms": 479610.80792682926, "avg_output_tokens_per_task": 32324.884146341465, "last_run_timestamp": "2026-09-26T08:55:13.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.868687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13888562626262627, "avg_duration_per_task_ms": 1028019.2020202021, "avg_output_tokens_per_task": 46184.969696969696, "last_run_timestamp": "2026-07-18T16:16:27.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.868687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0028392929292929293, "avg_duration_per_task_ms": 49008.020202020205, "avg_output_tokens_per_task": 5520.065656565656, "last_run_timestamp": "2026-09-23T06:08:55.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "ModelRun", "accuracy": 0.8685324351145038, "accuracy_std_dev": 0.008327000000000029, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.08006628498727736, "avg_duration_per_task_ms": 605980.251908397, "avg_output_tokens_per_task": 42020.10687022901, "last_run_timestamp": "2026-07-11T08:27:26.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.8684572837465565, "accuracy_std_dev": 0.006350273025626523, "run_count": 4, "total_questions": 726, "avg_cost_per_task_usd": 0.0059327286501377405, "avg_duration_per_task_ms": 153900.47520661156, "avg_output_tokens_per_task": 21060.526170798898, "last_run_timestamp": "2026-08-08T09:25:45.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.8683500000000001, "accuracy_std_dev": 0.014020687172888494, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08152287070707072, "avg_duration_per_task_ms": 190241.53434343435, "avg_output_tokens_per_task": 13202.136363636364, "last_run_timestamp": "2026-09-26T08:55:09.429Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.86835, "accuracy_std_dev": 0.01103428752570821, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07775004545454545, "avg_duration_per_task_ms": 107497.15858585859, "avg_output_tokens_per_task": 12580.208080808081, "last_run_timestamp": "2026-09-26T08:57:01.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.8681256666666667, "accuracy_std_dev": 0.02130941144710999, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02939427441077441, "avg_duration_per_task_ms": 110502.21548821549, "avg_output_tokens_per_task": 19394.752525252527, "last_run_timestamp": "2026-09-26T09:38:56.771Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.8681256666666665, "accuracy_std_dev": 0.00396809470093087, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008190090067340068, "avg_duration_per_task_ms": 450083.40993265994, "avg_output_tokens_per_task": 29007.585016835015, "last_run_timestamp": "2026-08-29T18:13:33.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.86798182245827, "accuracy_std_dev": 0.027206693677108224, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.11290408548305515, "avg_duration_per_task_ms": 768015.9382903388, "avg_output_tokens_per_task": 30078.80880121396, "last_run_timestamp": "2026-09-26T09:22:13.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.8676579999999999, "accuracy_std_dev": 0.007874153076151538, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.006459232323232324, "avg_duration_per_task_ms": 253155.9539842873, "avg_output_tokens_per_task": 18088.584736251403, "last_run_timestamp": "2026-09-26T08:54:07.885Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.8672838333333331, "accuracy_std_dev": 0.008551917044668319, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.2189638569023569, "avg_duration_per_task_ms": 1248558.3703703703, "avg_output_tokens_per_task": 55040.72895622896, "last_run_timestamp": "2026-08-29T18:24:25.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8670031666666665, "accuracy_std_dev": 0.004661340237051534, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.07032303872053872, "avg_duration_per_task_ms": 224493.27861952863, "avg_output_tokens_per_task": 11452.973905723906, "last_run_timestamp": "2026-09-26T09:37:35.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.8667788887015176, "accuracy_std_dev": 0.01707605497635669, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.08546577121978641, "avg_duration_per_task_ms": 213324.25801011804, "avg_output_tokens_per_task": 23606.13940415964, "last_run_timestamp": "2026-08-29T16:55:12.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.8667228333333332, "accuracy_std_dev": 0.006024375774486715, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0256143728956229, "avg_duration_per_task_ms": 228795.63636363635, "avg_output_tokens_per_task": 21105.421717171717, "last_run_timestamp": "2026-09-19T09:03:56.948Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8659932, "accuracy_std_dev": 0.008010237759267841, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07810327272727272, "avg_duration_per_task_ms": 87737.49797979798, "avg_output_tokens_per_task": 12770.258585858586, "last_run_timestamp": "2026-09-26T08:55:49.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.8658808333333333, "accuracy_std_dev": 0.0104532872886422, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.07000543097643097, "avg_duration_per_task_ms": 223270.64393939395, "avg_output_tokens_per_task": 11398.105218855218, "last_run_timestamp": "2026-09-26T08:58:06.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.865758552529183, "accuracy_std_dev": 0.008634405610694918, "run_count": 8, "total_questions": 1542, "avg_cost_per_task_usd": 0.008104218547341117, "avg_duration_per_task_ms": 281362.64915693906, "avg_output_tokens_per_task": 30092.439688715953, "last_run_timestamp": "2026-08-29T19:20:21.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.8656778723404256, "accuracy_std_dev": 0.011228325910392875, "run_count": 8, "total_questions": 1551, "avg_cost_per_task_usd": 0.008557784010315926, "avg_duration_per_task_ms": 288449.6731141199, "avg_output_tokens_per_task": 30252.780786589297, "last_run_timestamp": "2026-08-29T18:42:44.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.86532, "accuracy_std_dev": 0.008780061799326916, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02537659898989899, "avg_duration_per_task_ms": 218871.01414141414, "avg_output_tokens_per_task": 20892.29595959596, "last_run_timestamp": "2026-09-19T08:55:39.417Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.8653196666666666, "accuracy_std_dev": 0.010378007269649058, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.029422861952861954, "avg_duration_per_task_ms": 503984.98484848486, "avg_output_tokens_per_task": 34386.4898989899, "last_run_timestamp": "2026-08-08T09:24:32.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.8651804917197452, "accuracy_std_dev": 0.013354329136557154, "run_count": 4, "total_questions": 785, "avg_cost_per_task_usd": 0.060597563057324846, "avg_duration_per_task_ms": 205391.9694267516, "avg_output_tokens_per_task": 23879.336305732482, "last_run_timestamp": "2026-09-26T09:20:06.107Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.8650391666666667, "accuracy_std_dev": 0.011558638146665698, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03327992845117845, "avg_duration_per_task_ms": 670922.4377104377, "avg_output_tokens_per_task": 34339.50252525252, "last_run_timestamp": "2026-08-29T19:08:19.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.8649764115566037, "accuracy_std_dev": 0.05470934774460394, "run_count": 5, "total_questions": 848, "avg_cost_per_task_usd": 0.08730664976415095, "avg_duration_per_task_ms": 645412.391509434, "avg_output_tokens_per_task": 26303.754716981133, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.8649570000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.020998656410256413, "avg_duration_per_task_ms": 619238.4615384615, "avg_output_tokens_per_task": 23774.34358974359, "last_run_timestamp": "2026-09-26T01:32:20.726Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.8648248729281768, "accuracy_std_dev": 0.02631682070007696, "run_count": 5, "total_questions": 905, "avg_cost_per_task_usd": 0.010246941436464088, "avg_duration_per_task_ms": 355027.6861878453, "avg_output_tokens_per_task": 21157.914917127073, "last_run_timestamp": "2026-09-26T09:56:32.081Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.8648077338403042, "accuracy_std_dev": 0.0058285997192721924, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.03674314195183777, "avg_duration_per_task_ms": 350368.4626108999, "avg_output_tokens_per_task": 23592.097591888465, "last_run_timestamp": "2026-08-01T17:57:47.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.8646862059405941, "accuracy_std_dev": 0.024035029940114947, "run_count": 3, "total_questions": 505, "avg_cost_per_task_usd": 0.10486535445544554, "avg_duration_per_task_ms": 438191.62970297027, "avg_output_tokens_per_task": 30917.40594059406, "last_run_timestamp": "2026-07-18T08:30:34.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.8646021311475409, "accuracy_std_dev": 0.007616834002538237, "run_count": 3, "total_questions": 549, "avg_cost_per_task_usd": 0.1932159398907104, "avg_duration_per_task_ms": 529434.2877959927, "avg_output_tokens_per_task": 56744.85792349727, "last_run_timestamp": "2026-07-18T08:26:48.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.864345941772152, "accuracy_std_dev": 0.03040778172991867, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.12872155822784812, "avg_duration_per_task_ms": 705035.3753164557, "avg_output_tokens_per_task": 43078.59620253165, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.8639189175738725, "accuracy_std_dev": 0.02340391523335358, "run_count": 4, "total_questions": 643, "avg_cost_per_task_usd": 0.06298315552099534, "avg_duration_per_task_ms": 414941.54743390356, "avg_output_tokens_per_task": 21263.947122861588, "last_run_timestamp": "2026-08-15T08:47:51.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.863768366310743, "accuracy_std_dev": 0.013824474714642947, "run_count": 27, "total_questions": 4589, "avg_cost_per_task_usd": 0.16332582131183263, "avg_duration_per_task_ms": 245950.8069296143, "avg_output_tokens_per_task": 29748.09130529527, "last_run_timestamp": "2026-09-26T08:54:22.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8636842273684211, "accuracy_std_dev": 0.01763815223202249, "run_count": 5, "total_questions": 950, "avg_cost_per_task_usd": 0.08584259578947369, "avg_duration_per_task_ms": 395986.4378947368, "avg_output_tokens_per_task": 24656.30947368421, "last_run_timestamp": "2026-09-26T08:57:34.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.8636519239465571, "accuracy_std_dev": 0.028707041857903168, "run_count": 11, "total_questions": 1946, "avg_cost_per_task_usd": 0.010248814491264132, "avg_duration_per_task_ms": 324581.0220966084, "avg_output_tokens_per_task": 18963.51336073998, "last_run_timestamp": "2026-09-26T08:55:27.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.863636375, "accuracy_std_dev": 0.017230143926687758, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07376672032828283, "avg_duration_per_task_ms": 300093.57133838383, "avg_output_tokens_per_task": 20992.814393939392, "last_run_timestamp": "2026-08-29T18:39:14.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.8636360000000001, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13014110606060605, "avg_duration_per_task_ms": 586191.3131313132, "avg_output_tokens_per_task": 37366.416666666664, "last_run_timestamp": "2026-08-29T19:12:34.610Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0938164393939394, "avg_duration_per_task_ms": 667365.3939393939, "avg_output_tokens_per_task": 23446.50505050505, "last_run_timestamp": "2026-09-10T01:44:54.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.8633267645181878, "accuracy_std_dev": 0.013466913424667874, "run_count": 8, "total_questions": 1567, "avg_cost_per_task_usd": 0.07518679770261646, "avg_duration_per_task_ms": 298763.8781110402, "avg_output_tokens_per_task": 20760.19719208679, "last_run_timestamp": "2026-08-29T18:30:43.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.8631252833052278, "accuracy_std_dev": 0.03388611321929843, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.09767507644744237, "avg_duration_per_task_ms": 708453.107925801, "avg_output_tokens_per_task": 37470.564362001125, "last_run_timestamp": "2026-09-21T15:33:37.909Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8631102531752752, "accuracy_std_dev": 0.01428710704815903, "run_count": 6, "total_questions": 1181, "avg_cost_per_task_usd": 0.006545317527519051, "avg_duration_per_task_ms": 246133.70618120237, "avg_output_tokens_per_task": 12855.507197290432, "last_run_timestamp": "2026-09-26T09:18:13.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.8628256666666666, "accuracy_std_dev": 0.019902461007624172, "run_count": 5, "total_questions": 972, "avg_cost_per_task_usd": 0.08342864197530865, "avg_duration_per_task_ms": 454814.01543209876, "avg_output_tokens_per_task": 23652.057613168723, "last_run_timestamp": "2026-09-26T08:53:09.429Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.8627945, "accuracy_std_dev": 0.033022030634865585, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0750204577020202, "avg_duration_per_task_ms": 314213.95328282827, "avg_output_tokens_per_task": 21335.801767676767, "last_run_timestamp": "2026-08-29T20:00:02.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.8627943749999999, "accuracy_std_dev": 0.00929733078546604, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.855192389520202, "avg_duration_per_task_ms": 483915.25, "avg_output_tokens_per_task": 34017.0119949495, "last_run_timestamp": "2026-09-26T08:53:16.951Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.8626742857142858, "accuracy_std_dev": 0.017858490299448887, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.08792987914862915, "avg_duration_per_task_ms": 406224.28066378064, "avg_output_tokens_per_task": 39434.920634920636, "last_run_timestamp": "2026-09-26T08:56:57.800Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.862584125, "accuracy_std_dev": 0.009408632345318578, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.8442858333333334, "avg_duration_per_task_ms": 415897.4154040404, "avg_output_tokens_per_task": 33580.344696969696, "last_run_timestamp": "2026-09-26T08:53:36.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.8622473714739068, "accuracy_std_dev": 0.021847945469408447, "run_count": 29, "total_questions": 5672, "avg_cost_per_task_usd": 0.2541298106488012, "avg_duration_per_task_ms": 337436.6962270804, "avg_output_tokens_per_task": 46060.357369534555, "last_run_timestamp": "2026-09-26T11:00:26.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.8620605995542349, "accuracy_std_dev": 0.04244629212767299, "run_count": 7, "total_questions": 1346, "avg_cost_per_task_usd": 0.05972067161961367, "avg_duration_per_task_ms": 704231.3573551263, "avg_output_tokens_per_task": 21477.753343239227, "last_run_timestamp": "2026-09-12T09:19:57.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.8619530000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008309641414141413, "avg_duration_per_task_ms": 730613.3737373737, "avg_output_tokens_per_task": 29358.666666666668, "last_run_timestamp": "2026-07-31T04:09:58.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.8619525, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.223144601010101, "avg_duration_per_task_ms": 273332.86363636365, "avg_output_tokens_per_task": 22220.545454545456, "last_run_timestamp": "2026-09-01T13:23:04.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.86184890094552, "accuracy_std_dev": 0.17338949639000253, "run_count": 13, "total_questions": 2221, "avg_cost_per_task_usd": 0.14356039261593875, "avg_duration_per_task_ms": 275384.9054479964, "avg_output_tokens_per_task": 11018.233678523187, "last_run_timestamp": "2026-09-29T21:35:11.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8613916666666667, "accuracy_std_dev": 0.019552374609977406, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.15697531565656564, "avg_duration_per_task_ms": 1157244.2163299664, "avg_output_tokens_per_task": 59089.26262626263, "last_run_timestamp": "2026-08-29T19:01:27.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.04603966955336814, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004525512626262627, "avg_duration_per_task_ms": 565963.2584175084, "avg_output_tokens_per_task": 18978.082491582492, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.8612792, "accuracy_std_dev": 0.03136197207702347, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005120404040404041, "avg_duration_per_task_ms": 530889.794949495, "avg_output_tokens_per_task": 17542.636363636364, "last_run_timestamp": "2026-09-05T09:22:54.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.8611646092843325, "accuracy_std_dev": 0.009461303461336583, "run_count": 8, "total_questions": 1551, "avg_cost_per_task_usd": 0.05262209671179884, "avg_duration_per_task_ms": 438586.79239200515, "avg_output_tokens_per_task": 27289.920696324953, "last_run_timestamp": "2026-08-01T17:44:01.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "WandB", "accuracy": 0.8609557877094972, "accuracy_std_dev": 0.0034322869279178035, "run_count": 3, "total_questions": 537, "avg_cost_per_task_usd": 0.005924823091247672, "avg_duration_per_task_ms": 435495.54189944133, "avg_output_tokens_per_task": 16657.258845437616, "last_run_timestamp": "2026-07-18T08:04:02.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.8605498333333333, "accuracy_std_dev": 0.020281775936512293, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0053592163299663285, "avg_duration_per_task_ms": 748446.3863636364, "avg_output_tokens_per_task": 29643.154882154882, "last_run_timestamp": "2026-08-29T19:07:25.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8605498333333332, "accuracy_std_dev": 0.014065440607112956, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.16507093434343434, "avg_duration_per_task_ms": 1149871.6085858585, "avg_output_tokens_per_task": 53779.61784511784, "last_run_timestamp": "2026-08-29T18:43:04.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.8605496666666668, "accuracy_std_dev": 0.016734196123175105, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11623640740740741, "avg_duration_per_task_ms": 564457.3611111111, "avg_output_tokens_per_task": 34704.39057239057, "last_run_timestamp": "2026-08-29T18:39:19.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8603745, "accuracy_std_dev": 0.008427361219563313, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.8821099195075758, "avg_duration_per_task_ms": 416796.10890151514, "avg_output_tokens_per_task": 33415.74179292929, "last_run_timestamp": "2026-09-26T09:19:04.990Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.860269, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.20759804545454547, "avg_duration_per_task_ms": 271144.0252525252, "avg_output_tokens_per_task": 20665.12626262626, "last_run_timestamp": "2026-09-01T13:31:11.391Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8601822310030396, "accuracy_std_dev": 0.013547575585321513, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.007783161094224923, "avg_duration_per_task_ms": 526386.7112462006, "avg_output_tokens_per_task": 27793.246200607904, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.8599929535603715, "accuracy_std_dev": 0.009491817962856216, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.1207670278637771, "avg_duration_per_task_ms": 352384.2745098039, "avg_output_tokens_per_task": 30417.953560371516, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.8599885, "accuracy_std_dev": 0.01684674659738986, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.19415970033670032, "avg_duration_per_task_ms": 1192894.914983165, "avg_output_tokens_per_task": 57724.79882154882, "last_run_timestamp": "2026-08-29T20:16:17.828Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.8599289999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 188, "avg_cost_per_task_usd": 0.023036143617021276, "avg_duration_per_task_ms": 1599986.2287234042, "avg_output_tokens_per_task": 29232.622340425532, "last_run_timestamp": "2026-07-18T08:57:54.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Morph", "accuracy": 0.859829, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.07579302051282051, "avg_duration_per_task_ms": 503262.1435897436, "avg_output_tokens_per_task": 21548.302564102563, "last_run_timestamp": "2026-07-04T18:14:56.425Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.859638, "accuracy_std_dev": 0.008071103007024502, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.815575909090909, "avg_duration_per_task_ms": 465568.93876262626, "avg_output_tokens_per_task": 32451.6672979798, "last_run_timestamp": "2026-09-26T09:01:02.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8594275, "accuracy_std_dev": 0.013424092492604471, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0069255740740740744, "avg_duration_per_task_ms": 366969.99410774413, "avg_output_tokens_per_task": 23129.368686868685, "last_run_timestamp": "2026-09-26T09:34:13.855Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.8593380638297874, "accuracy_std_dev": 0.008570441036233525, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.25169353191489363, "avg_duration_per_task_ms": 300098.05673758866, "avg_output_tokens_per_task": 62735.48936170213, "last_run_timestamp": "2026-07-18T08:26:40.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.8593072857142857, "accuracy_std_dev": 0.014195632454227251, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.05442466955266955, "avg_duration_per_task_ms": 720332.65007215, "avg_output_tokens_per_task": 32949.00865800866, "last_run_timestamp": "2026-08-29T19:37:04.387Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.8591573843017328, "accuracy_std_dev": 0.02115249701193692, "run_count": 5, "total_questions": 981, "avg_cost_per_task_usd": 0.10401832415902142, "avg_duration_per_task_ms": 479500.45667686034, "avg_output_tokens_per_task": 30593.96636085627, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.8590881190053286, "accuracy_std_dev": 0.02052541308201858, "run_count": 4, "total_questions": 563, "avg_cost_per_task_usd": 0.08836215808170515, "avg_duration_per_task_ms": 517852.0088809947, "avg_output_tokens_per_task": 22138.96802841918, "last_run_timestamp": "2026-08-15T08:47:41.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8589315283806344, "accuracy_std_dev": 0.01816958311832932, "run_count": 7, "total_questions": 1198, "avg_cost_per_task_usd": 0.15454068447412356, "avg_duration_per_task_ms": 551935.939899833, "avg_output_tokens_per_task": 51245.50918196995, "last_run_timestamp": "2026-08-29T19:17:01.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.8587961250000001, "accuracy_std_dev": 0.026570942984948328, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0075656887626262635, "avg_duration_per_task_ms": 497586.5126262626, "avg_output_tokens_per_task": 24566.76767676768, "last_run_timestamp": "2026-09-26T08:53:09.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8580246666666665, "accuracy_std_dev": 0.007694244595078101, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06644984511784513, "avg_duration_per_task_ms": 213157.5361952862, "avg_output_tokens_per_task": 10806.75925925926, "last_run_timestamp": "2026-09-26T09:18:24.826Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.857954375, "accuracy_std_dev": 0.013038726633930733, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.7458450347222222, "avg_duration_per_task_ms": 493231.4678030303, "avg_output_tokens_per_task": 29674.352904040403, "last_run_timestamp": "2026-09-26T09:34:52.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.8577779119999999, "accuracy_std_dev": 0.006748318809155334, "run_count": 4, "total_questions": 750, "avg_cost_per_task_usd": 0.23260101733333335, "avg_duration_per_task_ms": 433991.3, "avg_output_tokens_per_task": 51935.09333333333, "last_run_timestamp": "2026-08-15T08:57:45.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.8577439999999998, "accuracy_std_dev": 0.005891999999999953, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008277098484848486, "avg_duration_per_task_ms": 710306.3914141414, "avg_output_tokens_per_task": 29437.88888888889, "last_run_timestamp": "2026-07-18T08:30:06.336Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.8577028343558282, "accuracy_std_dev": 0.011904490003355866, "run_count": 5, "total_questions": 978, "avg_cost_per_task_usd": 0.14577651738241312, "avg_duration_per_task_ms": 258434.56646216768, "avg_output_tokens_per_task": 28881.36707566462, "last_run_timestamp": "2026-09-26T09:21:12.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.857418, "accuracy_std_dev": null, "run_count": 1, "total_questions": 173, "avg_cost_per_task_usd": 0.08778269942196533, "avg_duration_per_task_ms": 317630.5202312139, "avg_output_tokens_per_task": 36575.57225433526, "last_run_timestamp": "2026-08-04T03:12:38.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.857191527607362, "accuracy_std_dev": 0.013500948849617942, "run_count": 5, "total_questions": 978, "avg_cost_per_task_usd": 0.1052768057259714, "avg_duration_per_task_ms": 186949.86605316974, "avg_output_tokens_per_task": 27345.398773006134, "last_run_timestamp": "2026-09-26T08:51:46.229Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.8571024481012658, "accuracy_std_dev": 0.017386228014021777, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.05838445316455696, "avg_duration_per_task_ms": 511596.7248945148, "avg_output_tokens_per_task": 33795.75696202532, "last_run_timestamp": "2026-08-29T19:11:30.664Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.8569513442340791, "accuracy_std_dev": 0.013334663992859697, "run_count": 9, "total_questions": 1743, "avg_cost_per_task_usd": 0.10127325989672976, "avg_duration_per_task_ms": 639996.5054503729, "avg_output_tokens_per_task": 35441.2759609868, "last_run_timestamp": "2026-09-19T08:55:10.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8569022000000001, "accuracy_std_dev": 0.011711955283384562, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005455330303030303, "avg_duration_per_task_ms": 645344.1272727273, "avg_output_tokens_per_task": 28522.870707070706, "last_run_timestamp": "2026-08-22T17:16:03.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.8569022, "accuracy_std_dev": 0.010212534228094395, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.8410627575757575, "avg_duration_per_task_ms": 450697.24494949495, "avg_output_tokens_per_task": 33467.82424242424, "last_run_timestamp": "2026-09-26T09:29:09.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8566216666666666, "accuracy_std_dev": 0.010073872139792559, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004407090909090909, "avg_duration_per_task_ms": 239041.3745791246, "avg_output_tokens_per_task": 27919.25, "last_run_timestamp": "2026-08-29T19:40:39.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8565655999999999, "accuracy_std_dev": 0.01503514097838793, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.09365999898989899, "avg_duration_per_task_ms": 671877.2212121212, "avg_output_tokens_per_task": 28686.372727272726, "last_run_timestamp": "2026-09-26T08:52:43.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.8565324139194139, "accuracy_std_dev": 0.018712538162953735, "run_count": 4, "total_questions": 546, "avg_cost_per_task_usd": 0.00399579304029304, "avg_duration_per_task_ms": 408743.60622710624, "avg_output_tokens_per_task": 7814.793040293041, "last_run_timestamp": "2026-09-28T08:34:14.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8564577918781726, "accuracy_std_dev": 0.028181512947320644, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.10150419204737732, "avg_duration_per_task_ms": 330925.20050761424, "avg_output_tokens_per_task": 22836.06937394247, "last_run_timestamp": "2026-09-26T09:19:52.896Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.8563503216174183, "accuracy_std_dev": 0.016757180106276432, "run_count": 17, "total_questions": 3215, "avg_cost_per_task_usd": 0.1523061856920684, "avg_duration_per_task_ms": 200594.1632970451, "avg_output_tokens_per_task": 27265.382270606533, "last_run_timestamp": "2026-09-26T09:17:01.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.8563168223350255, "accuracy_std_dev": 0.012593349140849445, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.01789011252115059, "avg_duration_per_task_ms": 343652.3730964467, "avg_output_tokens_per_task": 35569.74111675127, "last_run_timestamp": "2026-09-26T08:50:11.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.8562288, "accuracy_std_dev": 0.011497073373689477, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1913848090909091, "avg_duration_per_task_ms": 350329.27575757576, "avg_output_tokens_per_task": 55974.5, "last_run_timestamp": "2026-08-29T18:20:49.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.8560649439728354, "accuracy_std_dev": 0.042572315121043564, "run_count": 9, "total_questions": 1767, "avg_cost_per_task_usd": 0.08027640350877194, "avg_duration_per_task_ms": 491193.9134125637, "avg_output_tokens_per_task": 21784.452744765138, "last_run_timestamp": "2026-09-26T08:55:14.254Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8557868225629791, "accuracy_std_dev": 0.008479443730551645, "run_count": 14, "total_questions": 2739, "avg_cost_per_task_usd": 0.061858265790434463, "avg_duration_per_task_ms": 609941.4859437752, "avg_output_tokens_per_task": 34015.95436290617, "last_run_timestamp": "2026-08-29T19:16:17.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8555556, "accuracy_std_dev": 0.0176375124050984, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.011364495959595959, "avg_duration_per_task_ms": 652022.0676767677, "avg_output_tokens_per_task": 33080.43131313131, "last_run_timestamp": "2026-09-26T08:57:03.952Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.8550504, "accuracy_std_dev": 0.04995446213743072, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.07905246363636365, "avg_duration_per_task_ms": 503288.9419191919, "avg_output_tokens_per_task": 37401.228282828284, "last_run_timestamp": "2026-09-26T09:14:27.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.8549529134157944, "accuracy_std_dev": 0.0077363797752788375, "run_count": 16, "total_questions": 3153, "avg_cost_per_task_usd": 0.05448097875039644, "avg_duration_per_task_ms": 435614.29495718365, "avg_output_tokens_per_task": 27906.339993656835, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8547375714285715, "accuracy_std_dev": 0.04301245273458486, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1766843759018759, "avg_duration_per_task_ms": 45692.21067821068, "avg_output_tokens_per_task": 3336.1709956709956, "last_run_timestamp": "2026-09-26T09:24:09.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8546573333333333, "accuracy_std_dev": 0.015141104458467416, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007946281144781145, "avg_duration_per_task_ms": 637289.4747474748, "avg_output_tokens_per_task": 29143.16835016835, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.8543893879003559, "accuracy_std_dev": 0.010879822884587815, "run_count": 3, "total_questions": 562, "avg_cost_per_task_usd": 0.0749505462633452, "avg_duration_per_task_ms": 1305986.9822064056, "avg_output_tokens_per_task": 38419.777580071175, "last_run_timestamp": "2026-07-18T16:22:39.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8541664999999999, "accuracy_std_dev": 0.020077776601257404, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.8391522569444444, "avg_duration_per_task_ms": 453772.9583333333, "avg_output_tokens_per_task": 33398.183712121216, "last_run_timestamp": "2026-09-26T09:34:32.494Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8535522308340354, "accuracy_std_dev": 0.010038567733441294, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.01690193850042123, "avg_duration_per_task_ms": 472592.7733782645, "avg_output_tokens_per_task": 33567.315080033695, "last_run_timestamp": "2026-09-26T08:54:58.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.8529230909090909, "accuracy_std_dev": 0.020659392015407452, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.09120310835629016, "avg_duration_per_task_ms": 630647.5684113866, "avg_output_tokens_per_task": 41423.92011019284, "last_run_timestamp": "2026-09-26T08:59:56.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8526935, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20698726515151514, "avg_duration_per_task_ms": 257611.93181818182, "avg_output_tokens_per_task": 19625.295454545456, "last_run_timestamp": "2026-09-01T13:50:27.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.8522881577868852, "accuracy_std_dev": 0.009664339658766143, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.09954490881147542, "avg_duration_per_task_ms": 487265.8411885246, "avg_output_tokens_per_task": 30959.861680327867, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.16585097979797978, "avg_duration_per_task_ms": 389107.60101010103, "avg_output_tokens_per_task": 37536.631313131315, "last_run_timestamp": "2026-08-04T03:12:10.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.851851625, "accuracy_std_dev": 0.018151435989595288, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.7516495170454545, "avg_duration_per_task_ms": 500493.72222222225, "avg_output_tokens_per_task": 29958.29797979798, "last_run_timestamp": "2026-09-26T08:55:10.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.851679139925373, "accuracy_std_dev": 0.03687109443675745, "run_count": 28, "total_questions": 4824, "avg_cost_per_task_usd": 0.011843935737976785, "avg_duration_per_task_ms": 411338.9653814262, "avg_output_tokens_per_task": 16150.16169154229, "last_run_timestamp": "2026-09-26T08:52:37.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8516111428571428, "accuracy_std_dev": 0.032073947329028685, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.831446111111111, "avg_duration_per_task_ms": 427691.9718614719, "avg_output_tokens_per_task": 33200.131313131315, "last_run_timestamp": "2026-09-26T08:56:21.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.8512395454545454, "accuracy_std_dev": 0.008130117793663436, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.07382550229568412, "avg_duration_per_task_ms": 230546.61202938476, "avg_output_tokens_per_task": 22967.3406795225, "last_run_timestamp": "2026-09-26T08:53:37.648Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.8511034444444444, "accuracy_std_dev": 0.03586305000387234, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.17666739506172838, "avg_duration_per_task_ms": 44619.677328843994, "avg_output_tokens_per_task": 3328.8120089786757, "last_run_timestamp": "2026-09-26T08:53:27.502Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8510597058823527, "accuracy_std_dev": 0.02313439249450838, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.1586523764111705, "avg_duration_per_task_ms": 153997.80986333927, "avg_output_tokens_per_task": 15572.684789067142, "last_run_timestamp": "2026-09-26T09:21:26.479Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.8510099000000001, "accuracy_std_dev": 0.015100002764569275, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.7538109368686869, "avg_duration_per_task_ms": 501864.8787878788, "avg_output_tokens_per_task": 30068.36717171717, "last_run_timestamp": "2026-09-26T09:17:18.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.8508604502868068, "accuracy_std_dev": 0.02478573563844512, "run_count": 6, "total_questions": 1046, "avg_cost_per_task_usd": 0.010149271510516252, "avg_duration_per_task_ms": 344062.4971319312, "avg_output_tokens_per_task": 20091.341300191205, "last_run_timestamp": "2026-09-26T09:16:37.664Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.8508416000000001, "accuracy_std_dev": 0.01139804777319343, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.10227179090909091, "avg_duration_per_task_ms": 614412.1656565657, "avg_output_tokens_per_task": 32654.38888888889, "last_run_timestamp": "2026-08-29T19:18:11.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8506943125000002, "accuracy_std_dev": 0.0170931360994653, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.8004648680555555, "avg_duration_per_task_ms": 479005.81407828286, "avg_output_tokens_per_task": 30341.69191919192, "last_run_timestamp": "2026-09-26T09:19:41.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.850575, "accuracy_std_dev": null, "run_count": 1, "total_questions": 145, "avg_cost_per_task_usd": 0.015425489655172412, "avg_duration_per_task_ms": 62324.744827586204, "avg_output_tokens_per_task": 15913.096551724138, "last_run_timestamp": "2026-08-01T18:26:19.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8505651288135594, "accuracy_std_dev": 0.014603362238425309, "run_count": 6, "total_questions": 1180, "avg_cost_per_task_usd": 0.008425746610169493, "avg_duration_per_task_ms": 368412.6118644068, "avg_output_tokens_per_task": 24250.365254237287, "last_run_timestamp": "2026-09-26T09:22:08.527Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8504212593103447, "accuracy_std_dev": 0.008642258916137237, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.05126995816091954, "avg_duration_per_task_ms": 291139.3857471264, "avg_output_tokens_per_task": 45945.63908045977, "last_run_timestamp": "2026-09-26T16:05:38.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.8503937480314961, "accuracy_std_dev": 0.00023499999999998522, "run_count": 2, "total_questions": 381, "avg_cost_per_task_usd": 0.023863905511811022, "avg_duration_per_task_ms": 137111.656167979, "avg_output_tokens_per_task": 36528.412073490814, "last_run_timestamp": "2026-09-26T09:52:38.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.8503154279475983, "accuracy_std_dev": 0.03140494390885563, "run_count": 7, "total_questions": 1374, "avg_cost_per_task_usd": 0.09674545997088793, "avg_duration_per_task_ms": 1349515.0895196507, "avg_output_tokens_per_task": 43276.89374090247, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.8501715195876289, "accuracy_std_dev": 0.027779548602524146, "run_count": 5, "total_questions": 970, "avg_cost_per_task_usd": 0.13975330515463918, "avg_duration_per_task_ms": 383469.5556701031, "avg_output_tokens_per_task": 31875.5793814433, "last_run_timestamp": "2026-09-26T08:53:10.326Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8501683333333333, "accuracy_std_dev": 0.041790840030375614, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.17284996857463525, "avg_duration_per_task_ms": 43624.98316498317, "avg_output_tokens_per_task": 3253.4483726150393, "last_run_timestamp": "2026-09-26T08:54:57.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8500861979274612, "accuracy_std_dev": 0.01857196094762209, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.016829795854922277, "avg_duration_per_task_ms": 389325.4870466321, "avg_output_tokens_per_task": 22513.535751295338, "last_run_timestamp": "2026-08-29T18:07:13.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.849820223628692, "accuracy_std_dev": 0.010339301856316641, "run_count": 11, "total_questions": 2133, "avg_cost_per_task_usd": 0.027238617440225037, "avg_duration_per_task_ms": 658650.0131270511, "avg_output_tokens_per_task": 31066.365682137835, "last_run_timestamp": "2026-09-26T09:22:28.862Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.8498050435706695, "accuracy_std_dev": 0.03351477910773096, "run_count": 5, "total_questions": 941, "avg_cost_per_task_usd": 0.11714972582359193, "avg_duration_per_task_ms": 505045.9829968119, "avg_output_tokens_per_task": 33189.51115834219, "last_run_timestamp": "2026-09-26T08:54:17.442Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.8496956203301478, "accuracy_std_dev": 0.012943756573044104, "run_count": 6, "total_questions": 1151, "avg_cost_per_task_usd": 0.076488348392702, "avg_duration_per_task_ms": 490389.1841876629, "avg_output_tokens_per_task": 33522.794960903564, "last_run_timestamp": "2026-08-22T17:03:07.750Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8495983231939164, "accuracy_std_dev": 0.010674015891406563, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.09633560709759188, "avg_duration_per_task_ms": 314235.76045627374, "avg_output_tokens_per_task": 30867.523447401774, "last_run_timestamp": "2026-09-26T09:23:32.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.8493265000000001, "accuracy_std_dev": 0.01119888692013629, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.026091900252525255, "avg_duration_per_task_ms": 290649.8787878788, "avg_output_tokens_per_task": 21576.02525252525, "last_run_timestamp": "2026-08-01T17:57:21.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.8492335655172413, "accuracy_std_dev": 0.026170202058877375, "run_count": 9, "total_questions": 1740, "avg_cost_per_task_usd": 0.029375192528735626, "avg_duration_per_task_ms": 542113.9201149425, "avg_output_tokens_per_task": 28046.42528735632, "last_run_timestamp": "2026-09-26T08:57:03.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.8492264101265824, "accuracy_std_dev": 0.04905497536664576, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.1081006405063291, "avg_duration_per_task_ms": 612102.1265822785, "avg_output_tokens_per_task": 27354.124050632912, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.8490153107221006, "accuracy_std_dev": 0.02104665580299624, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.04150975711159738, "avg_duration_per_task_ms": 669606.0547045952, "avg_output_tokens_per_task": 34749.842450765864, "last_run_timestamp": "2026-08-29T19:20:22.044Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.8486918037542662, "accuracy_std_dev": 0.014893804491204496, "run_count": 6, "total_questions": 1172, "avg_cost_per_task_usd": 0.017789553754266208, "avg_duration_per_task_ms": 299607.09129692835, "avg_output_tokens_per_task": 35353.55375426621, "last_run_timestamp": "2026-09-26T08:54:42.351Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.20520298484848487, "avg_duration_per_task_ms": 278139.5202020202, "avg_output_tokens_per_task": 20422.565656565657, "last_run_timestamp": "2026-09-01T13:34:32.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8482611842239186, "accuracy_std_dev": 0.011728653128556577, "run_count": 10, "total_questions": 1965, "avg_cost_per_task_usd": 0.12833305750636134, "avg_duration_per_task_ms": 389662.5424936387, "avg_output_tokens_per_task": 33100.7262086514, "last_run_timestamp": "2026-09-26T08:55:48.508Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.8482548437025798, "accuracy_std_dev": 0.010506007044067693, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.029977341932220538, "avg_duration_per_task_ms": 657961.3520485584, "avg_output_tokens_per_task": 31303.588770864946, "last_run_timestamp": "2026-09-26T09:15:06.166Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "WandB", "accuracy": 0.8481760229007633, "accuracy_std_dev": 0.021743000000000012, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.025479938931297706, "avg_duration_per_task_ms": 435380.475826972, "avg_output_tokens_per_task": 21459.994910941477, "last_run_timestamp": "2026-07-18T16:18:27.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8480639, "accuracy_std_dev": 0.00907569944907828, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04004892272727273, "avg_duration_per_task_ms": 395495.8308080808, "avg_output_tokens_per_task": 40373.71717171717, "last_run_timestamp": "2026-09-26T16:12:42.605Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.8480638333333334, "accuracy_std_dev": 0.0534225021796891, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06181613468013468, "avg_duration_per_task_ms": 279974.0715488215, "avg_output_tokens_per_task": 20518.97474747475, "last_run_timestamp": "2026-08-15T15:55:02.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.847643, "accuracy_std_dev": 0.01430999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02130802777777778, "avg_duration_per_task_ms": 487650.29545454547, "avg_output_tokens_per_task": 42398.98737373737, "last_run_timestamp": "2026-09-26T09:15:51.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8475419999999999, "accuracy_std_dev": 0.02235490172109912, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.16931346101010097, "avg_duration_per_task_ms": 157193.26565656567, "avg_output_tokens_per_task": 15870.87696969697, "last_run_timestamp": "2026-09-26T09:21:17.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8473202765017668, "accuracy_std_dev": 0.011288322955111134, "run_count": 6, "total_questions": 1132, "avg_cost_per_task_usd": 0.016132280918727912, "avg_duration_per_task_ms": 230171.71908127208, "avg_output_tokens_per_task": 32052.750883392226, "last_run_timestamp": "2026-09-26T08:49:42.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.8472522501081783, "accuracy_std_dev": 0.022478090167458387, "run_count": 12, "total_questions": 2311, "avg_cost_per_task_usd": 0.153898237126785, "avg_duration_per_task_ms": 298616.1008221549, "avg_output_tokens_per_task": 40013.343574210296, "last_run_timestamp": "2026-09-26T08:55:26.766Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8468012500000001, "accuracy_std_dev": 0.031201721542047957, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.11716699116161616, "avg_duration_per_task_ms": 561187.7866161616, "avg_output_tokens_per_task": 31215.503787878788, "last_run_timestamp": "2026-08-29T18:26:17.965Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8466330000000001, "accuracy_std_dev": 0.02123421248363121, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.13107612676767677, "avg_duration_per_task_ms": 434259.98131313134, "avg_output_tokens_per_task": 34303.56616161616, "last_run_timestamp": "2026-09-26T10:36:35.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8462764299380979, "accuracy_std_dev": 0.015770782183096275, "run_count": 9, "total_questions": 1777, "avg_cost_per_task_usd": 0.08913706978052896, "avg_duration_per_task_ms": 416673.65053460887, "avg_output_tokens_per_task": 22175.449071468767, "last_run_timestamp": "2026-09-26T09:15:30.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.8461668398357288, "accuracy_std_dev": 0.06450182954877243, "run_count": 6, "total_questions": 974, "avg_cost_per_task_usd": 0.13265561088295688, "avg_duration_per_task_ms": 1458439.1293634498, "avg_output_tokens_per_task": 46081.89835728953, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.8460472371705964, "accuracy_std_dev": 0.020750268207603947, "run_count": 4, "total_questions": 721, "avg_cost_per_task_usd": 0.1417408987517337, "avg_duration_per_task_ms": 223845.73370319002, "avg_output_tokens_per_task": 32811.28294036061, "last_run_timestamp": "2026-09-26T09:01:35.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8458658888888889, "accuracy_std_dev": 0.03972899765618852, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.17558623344556676, "avg_duration_per_task_ms": 45322.60044893378, "avg_output_tokens_per_task": 3305.652076318743, "last_run_timestamp": "2026-09-26T08:54:44.118Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.8458657777777778, "accuracy_std_dev": 0.025409284277723633, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.15791502805836138, "avg_duration_per_task_ms": 153529.50112233445, "avg_output_tokens_per_task": 15577.850168350169, "last_run_timestamp": "2026-09-26T09:19:06.330Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8452106827586208, "accuracy_std_dev": 0.035124267859903004, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.054803951724137934, "avg_duration_per_task_ms": 227710.32091954022, "avg_output_tokens_per_task": 22733.708505747127, "last_run_timestamp": "2026-09-26T09:01:34.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.8451892474226804, "accuracy_std_dev": 0.006483228158872687, "run_count": 5, "total_questions": 970, "avg_cost_per_task_usd": 0.007323164948453608, "avg_duration_per_task_ms": 502972.8824742268, "avg_output_tokens_per_task": 18045.067010309278, "last_run_timestamp": "2026-08-29T18:17:30.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.8451776040609137, "accuracy_std_dev": 0.007926079064287278, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.020253781725888324, "avg_duration_per_task_ms": 578832.0761421319, "avg_output_tokens_per_task": 22875.73434856176, "last_run_timestamp": "2026-08-15T14:57:08.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8451178333333332, "accuracy_std_dev": 0.062380510090403164, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.10489853282828282, "avg_duration_per_task_ms": 490349.9595959596, "avg_output_tokens_per_task": 26821.65404040404, "last_run_timestamp": "2026-09-26T08:57:17.745Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.8451176666666667, "accuracy_std_dev": 0.014740528220597193, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1084811388888889, "avg_duration_per_task_ms": 330556.0925925926, "avg_output_tokens_per_task": 28808.55387205387, "last_run_timestamp": "2026-09-26T09:15:10.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.8450335999999999, "accuracy_std_dev": 0.011207895897089693, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.10169760353535354, "avg_duration_per_task_ms": 588854.2631313131, "avg_output_tokens_per_task": 32291.069191919192, "last_run_timestamp": "2026-09-26T10:00:58.971Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.8447463261390887, "accuracy_std_dev": 0.02515683625839815, "run_count": 19, "total_questions": 3753, "avg_cost_per_task_usd": 0.23025201278976826, "avg_duration_per_task_ms": 509807.78284039436, "avg_output_tokens_per_task": 43633.612310151875, "last_run_timestamp": "2026-09-26T08:55:16.971Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8446432684563759, "accuracy_std_dev": 0.0059905795195115885, "run_count": 7, "total_questions": 1341, "avg_cost_per_task_usd": 0.007240005219985085, "avg_duration_per_task_ms": 381452.95600298286, "avg_output_tokens_per_task": 17843.34004474273, "last_run_timestamp": "2026-08-29T18:39:38.845Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DekaLLM", "accuracy": 0.844444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 180, "avg_cost_per_task_usd": 0.08250182777777779, "avg_duration_per_task_ms": 453463.44444444444, "avg_output_tokens_per_task": 29087.933333333334, "last_run_timestamp": "2026-07-04T09:58:40.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.8443695555555555, "accuracy_std_dev": 0.00866138720107314, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.14565384680134677, "avg_duration_per_task_ms": 171228.07014590348, "avg_output_tokens_per_task": 32219.744668911335, "last_run_timestamp": "2026-09-26T09:17:52.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.8443149751647239, "accuracy_std_dev": 0.019658458576399102, "run_count": 10, "total_questions": 1973, "avg_cost_per_task_usd": 0.2023946046629498, "avg_duration_per_task_ms": 318245.6705524582, "avg_output_tokens_per_task": 45936.32032437912, "last_run_timestamp": "2026-09-26T09:23:36.793Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.844276, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.059444419191919194, "avg_duration_per_task_ms": 215120.31818181818, "avg_output_tokens_per_task": 34746.81060606061, "last_run_timestamp": "2026-09-26T10:57:32.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.844202597826087, "accuracy_std_dev": 0.014526749979936743, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.03675822753623188, "avg_duration_per_task_ms": 642776.863768116, "avg_output_tokens_per_task": 34394.93550724638, "last_run_timestamp": "2026-08-29T19:21:17.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.8440308723559445, "accuracy_std_dev": 0.010942287133606307, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.006902234865061998, "avg_duration_per_task_ms": 486147.99270605395, "avg_output_tokens_per_task": 17097.730123997084, "last_run_timestamp": "2026-08-29T19:18:32.746Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.8439305549132948, "accuracy_std_dev": 0.013499116239012633, "run_count": 8, "total_questions": 1557, "avg_cost_per_task_usd": 0.08022249261400129, "avg_duration_per_task_ms": 891006.2575465639, "avg_output_tokens_per_task": 40664.11753371869, "last_run_timestamp": "2026-08-30T00:19:02.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8436028, "accuracy_std_dev": 0.0247016256015672, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.06619451666666666, "avg_duration_per_task_ms": 657610.2742424243, "avg_output_tokens_per_task": 49788.666666666664, "last_run_timestamp": "2026-09-26T09:04:08.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.8431370045248868, "accuracy_std_dev": 0.004260933614280586, "run_count": 7, "total_questions": 1326, "avg_cost_per_task_usd": 0.039398003770739064, "avg_duration_per_task_ms": 50754.730015082954, "avg_output_tokens_per_task": 14781.932126696833, "last_run_timestamp": "2026-09-01T17:50:42.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.8431245977011494, "accuracy_std_dev": 0.011756871278107985, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.027553961047254147, "avg_duration_per_task_ms": 530629.3461047254, "avg_output_tokens_per_task": 22775.175606641125, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.8428064354527938, "accuracy_std_dev": 0.014271656998128247, "run_count": 6, "total_questions": 1038, "avg_cost_per_task_usd": 0.006940633911368016, "avg_duration_per_task_ms": 644934.6493256262, "avg_output_tokens_per_task": 17139.102119460502, "last_run_timestamp": "2026-08-29T18:15:38.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.842761, "accuracy_std_dev": 0.013366603308245514, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.11932980505050504, "avg_duration_per_task_ms": 317023.23434343433, "avg_output_tokens_per_task": 26929.522222222222, "last_run_timestamp": "2026-09-26T08:52:53.172Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8427608000000001, "accuracy_std_dev": 0.011595230077924271, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08432281212121212, "avg_duration_per_task_ms": 322252.27373737376, "avg_output_tokens_per_task": 20908.12626262626, "last_run_timestamp": "2026-09-26T08:55:48.606Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.8427092264988897, "accuracy_std_dev": 0.008653421154287425, "run_count": 7, "total_questions": 1351, "avg_cost_per_task_usd": 0.026629270910436716, "avg_duration_per_task_ms": 68698.27461139897, "avg_output_tokens_per_task": 18885.883049592892, "last_run_timestamp": "2026-09-29T21:03:10.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.8425925, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06301305808080808, "avg_duration_per_task_ms": 90351.56313131313, "avg_output_tokens_per_task": 13876.954545454546, "last_run_timestamp": "2026-09-01T17:51:50.923Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.8425925, "accuracy_std_dev": 0.02455548471828647, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.06700256376262627, "avg_duration_per_task_ms": 555614.4097222222, "avg_output_tokens_per_task": 21893.804924242424, "last_run_timestamp": "2026-08-29T19:19:13.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.8425925, "accuracy_std_dev": 0.008788056170166437, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.14231570202020202, "avg_duration_per_task_ms": 163901.45454545456, "avg_output_tokens_per_task": 32169.752525252527, "last_run_timestamp": "2026-09-26T09:14:46.960Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.8420405496688741, "accuracy_std_dev": 0.03337997922071716, "run_count": 7, "total_questions": 1359, "avg_cost_per_task_usd": 0.03366557321559971, "avg_duration_per_task_ms": 468305.63208241353, "avg_output_tokens_per_task": 31573.827814569537, "last_run_timestamp": "2026-08-29T19:03:08.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.8418721404174574, "accuracy_std_dev": 0.020746850670267405, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.025684514231499055, "avg_duration_per_task_ms": 264287.531941809, "avg_output_tokens_per_task": 21340.85009487666, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.8417577430497052, "accuracy_std_dev": 0.014641061103015125, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.13158069755686608, "avg_duration_per_task_ms": 164554.15669755687, "avg_output_tokens_per_task": 29682.097725358046, "last_run_timestamp": "2026-09-26T08:53:13.623Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.8417004740890687, "accuracy_std_dev": 0.014854076686525622, "run_count": 14, "total_questions": 2470, "avg_cost_per_task_usd": 0.01521865870445344, "avg_duration_per_task_ms": 573091.9230769231, "avg_output_tokens_per_task": 16302.382995951417, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.8415826000000001, "accuracy_std_dev": 0.01677086684223569, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.013069406060606059, "avg_duration_per_task_ms": 474851.301010101, "avg_output_tokens_per_task": 38641.41818181818, "last_run_timestamp": "2026-09-26T08:53:10.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8410445402985075, "accuracy_std_dev": 0.013784355106942281, "run_count": 7, "total_questions": 1340, "avg_cost_per_task_usd": 0.21741457462686567, "avg_duration_per_task_ms": 421390.1029850746, "avg_output_tokens_per_task": 49267.26119402985, "last_run_timestamp": "2026-08-29T19:00:02.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8410292857142857, "accuracy_std_dev": 0.03771800444544936, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.6859342640692642, "avg_duration_per_task_ms": 470200.2972582973, "avg_output_tokens_per_task": 27535.334054834057, "last_run_timestamp": "2026-09-26T08:58:47.240Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.8409129024016351, "accuracy_std_dev": 0.025338336264245934, "run_count": 10, "total_questions": 1957, "avg_cost_per_task_usd": 0.20147738068472154, "avg_duration_per_task_ms": 293735.5861011753, "avg_output_tokens_per_task": 45634.63157894737, "last_run_timestamp": "2026-09-26T08:54:40.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8408546584615385, "accuracy_std_dev": 0.004112945970955614, "run_count": 5, "total_questions": 975, "avg_cost_per_task_usd": 0.09125773538461539, "avg_duration_per_task_ms": 289113.2892307692, "avg_output_tokens_per_task": 22386.701538461537, "last_run_timestamp": "2026-09-19T08:59:39.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.840661254601227, "accuracy_std_dev": 0.04601154819825125, "run_count": 5, "total_questions": 978, "avg_cost_per_task_usd": 0.16520423415132923, "avg_duration_per_task_ms": 1050780.9069529653, "avg_output_tokens_per_task": 49997.56237218814, "last_run_timestamp": "2026-08-29T19:03:21.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.840404, "accuracy_std_dev": 0.008869764145680555, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07076540101010101, "avg_duration_per_task_ms": 297317.25757575757, "avg_output_tokens_per_task": 20594.954545454544, "last_run_timestamp": "2026-09-26T08:53:05.194Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.8394122720364743, "accuracy_std_dev": 0.0596591189780741, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.06978041185410336, "avg_duration_per_task_ms": 685532.4701114488, "avg_output_tokens_per_task": 44128.422998986825, "last_run_timestamp": "2026-09-26T09:41:13.780Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Together", "accuracy": 0.8393938763636365, "accuracy_std_dev": 0.012496125008266452, "run_count": 3, "total_questions": 550, "avg_cost_per_task_usd": 0.03367001454545455, "avg_duration_per_task_ms": 269728.67454545456, "avg_output_tokens_per_task": 28518.758181818182, "last_run_timestamp": "2026-07-18T08:29:12.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.8392968267419962, "accuracy_std_dev": 0.0037396706405778625, "run_count": 4, "total_questions": 531, "avg_cost_per_task_usd": 0.12392348210922788, "avg_duration_per_task_ms": 100276.54613935969, "avg_output_tokens_per_task": 27948.913370998118, "last_run_timestamp": "2026-09-26T09:05:37.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.8388888000000001, "accuracy_std_dev": 0.07596307646192327, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1830352515151515, "avg_duration_per_task_ms": 532300.757070707, "avg_output_tokens_per_task": 41655.45656565657, "last_run_timestamp": "2026-09-26T08:52:39.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.8388092148272959, "accuracy_std_dev": 0.039447312816872845, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.16212616849199665, "avg_duration_per_task_ms": 1421951.0867733783, "avg_output_tokens_per_task": 51701.65206402696, "last_run_timestamp": "2026-08-29T18:13:11.491Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.8387204, "accuracy_std_dev": 0.02117180060930105, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.020740932323232323, "avg_duration_per_task_ms": 437294.63434343436, "avg_output_tokens_per_task": 37324.31616161616, "last_run_timestamp": "2026-09-26T09:10:17.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.838579522693997, "accuracy_std_dev": 0.017426082982771814, "run_count": 7, "total_questions": 1366, "avg_cost_per_task_usd": 0.0794436691068814, "avg_duration_per_task_ms": 602046.7005856516, "avg_output_tokens_per_task": 29605.27232796486, "last_run_timestamp": "2026-09-26T09:19:10.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.8385383078393881, "accuracy_std_dev": 0.038297652145521575, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.10716400509878904, "avg_duration_per_task_ms": 1170088.323773104, "avg_output_tokens_per_task": 44102.23709369025, "last_run_timestamp": "2026-08-29T18:42:20.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.838284705882353, "accuracy_std_dev": 0.05170697825901454, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.18276167468805704, "avg_duration_per_task_ms": 40754.59982174688, "avg_output_tokens_per_task": 3290.764111705288, "last_run_timestamp": "2026-09-26T08:53:17.875Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8381625397574123, "accuracy_std_dev": 0.007901272276048144, "run_count": 9, "total_questions": 1484, "avg_cost_per_task_usd": 0.017624041105121297, "avg_duration_per_task_ms": 189126.76886792452, "avg_output_tokens_per_task": 35057.77156334232, "last_run_timestamp": "2026-09-26T09:29:08.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.8380671171597633, "accuracy_std_dev": 0.02773812208207325, "run_count": 5, "total_questions": 845, "avg_cost_per_task_usd": 0.07336154792899409, "avg_duration_per_task_ms": 417593.0852071006, "avg_output_tokens_per_task": 24105.62721893491, "last_run_timestamp": "2026-09-26T08:54:14.754Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.8380537102396514, "accuracy_std_dev": 0.018206946326350625, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.04750454974582426, "avg_duration_per_task_ms": 652669.2650689905, "avg_output_tokens_per_task": 37622.816993464054, "last_run_timestamp": "2026-08-29T19:06:57.766Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.8377145373291274, "accuracy_std_dev": 0.08364258942907019, "run_count": 5, "total_questions": 951, "avg_cost_per_task_usd": 0.09799871188222924, "avg_duration_per_task_ms": 610509.8033648791, "avg_output_tokens_per_task": 25161.19242902208, "last_run_timestamp": "2026-09-26T08:53:09.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8376334722439405, "accuracy_std_dev": 0.027219006084059767, "run_count": 7, "total_questions": 1279, "avg_cost_per_task_usd": 0.0889748506645817, "avg_duration_per_task_ms": 1571235.6520719312, "avg_output_tokens_per_task": 41303.13604378421, "last_run_timestamp": "2026-08-29T19:12:33.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8376065934065935, "accuracy_std_dev": 0.00571412028808169, "run_count": 3, "total_questions": 546, "avg_cost_per_task_usd": 0.036677783882783886, "avg_duration_per_task_ms": 47951.65201465201, "avg_output_tokens_per_task": 14583.362637362638, "last_run_timestamp": "2026-09-01T17:50:52.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8375589327830189, "accuracy_std_dev": 0.020056898198209345, "run_count": 9, "total_questions": 1696, "avg_cost_per_task_usd": 0.054286748820754724, "avg_duration_per_task_ms": 282159.9534198113, "avg_output_tokens_per_task": 22660.685141509435, "last_run_timestamp": "2026-09-26T09:50:39.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8371357823193916, "accuracy_std_dev": 0.009460368146759528, "run_count": 6, "total_questions": 1052, "avg_cost_per_task_usd": 0.08486342680608365, "avg_duration_per_task_ms": 599922.150190114, "avg_output_tokens_per_task": 21002.96958174905, "last_run_timestamp": "2026-09-19T08:48:08.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.8371241903959562, "accuracy_std_dev": 0.03104266294105938, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.17545385846672285, "avg_duration_per_task_ms": 1006738.7118786857, "avg_output_tokens_per_task": 53398.904802021905, "last_run_timestamp": "2026-08-29T18:43:32.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8369565347826087, "accuracy_std_dev": 0.019774448331529436, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.07724686594202897, "avg_duration_per_task_ms": 754357.3224637681, "avg_output_tokens_per_task": 41683.36666666667, "last_run_timestamp": "2026-08-22T17:01:35.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.8367000000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06203523232323233, "avg_duration_per_task_ms": 92697.9696969697, "avg_output_tokens_per_task": 13658.787878787878, "last_run_timestamp": "2026-09-01T17:49:43.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.8366999999999999, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04958680555555556, "avg_duration_per_task_ms": 536787.6388888889, "avg_output_tokens_per_task": 49485.26515151515, "last_run_timestamp": "2026-09-26T16:26:15.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.8366909151079136, "accuracy_std_dev": 0.01869795941406443, "run_count": 4, "total_questions": 695, "avg_cost_per_task_usd": 0.020156113669064748, "avg_duration_per_task_ms": 242800.49064748202, "avg_output_tokens_per_task": 40096.50647482014, "last_run_timestamp": "2026-09-26T09:48:09.064Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.8366301736263737, "accuracy_std_dev": 0.012674357340709624, "run_count": 3, "total_questions": 455, "avg_cost_per_task_usd": 0.04995332967032967, "avg_duration_per_task_ms": 107459.2945054945, "avg_output_tokens_per_task": 15090.074725274726, "last_run_timestamp": "2026-09-26T08:53:14.372Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.8365793063670413, "accuracy_std_dev": 0.023337994672784947, "run_count": 7, "total_questions": 1335, "avg_cost_per_task_usd": 0.01897062172284644, "avg_duration_per_task_ms": 793025.5857677902, "avg_output_tokens_per_task": 41115.47715355805, "last_run_timestamp": "2026-09-26T12:41:27.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.83627925, "accuracy_std_dev": 0.037120416716242575, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015161977272727275, "avg_duration_per_task_ms": 425327.9482323232, "avg_output_tokens_per_task": 27500.13005050505, "last_run_timestamp": "2026-08-29T19:58:27.585Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.836234166760247, "accuracy_std_dev": 0.02982784551020275, "run_count": 27, "total_questions": 5343, "avg_cost_per_task_usd": 0.1657105358412877, "avg_duration_per_task_ms": 150009.68556990454, "avg_output_tokens_per_task": 15316.003930376193, "last_run_timestamp": "2026-09-26T08:59:40.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.836139, "accuracy_std_dev": 0.05678065343641852, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005011018518518518, "avg_duration_per_task_ms": 214818.47643097644, "avg_output_tokens_per_task": 17529.722222222223, "last_run_timestamp": "2026-09-26T11:06:41.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.8359790095238095, "accuracy_std_dev": 0.03660393871642171, "run_count": 6, "total_questions": 630, "avg_cost_per_task_usd": 0.10253282222222224, "avg_duration_per_task_ms": 301607.7301587302, "avg_output_tokens_per_task": 23860.25873015873, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.8358585000000001, "accuracy_std_dev": 0.02753853053904659, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1297035244107744, "avg_duration_per_task_ms": 386511.07744107745, "avg_output_tokens_per_task": 35313.07323232323, "last_run_timestamp": "2026-09-26T08:54:10.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.8355473623188407, "accuracy_std_dev": 0.012606082718439233, "run_count": 11, "total_questions": 1656, "avg_cost_per_task_usd": 0.03472509842995169, "avg_duration_per_task_ms": 236602.51086956522, "avg_output_tokens_per_task": 27679.197463768116, "last_run_timestamp": "2026-09-26T16:00:25.681Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.835238659717552, "accuracy_std_dev": 0.014890026205318157, "run_count": 9, "total_questions": 1487, "avg_cost_per_task_usd": 0.09666733691997309, "avg_duration_per_task_ms": 626516.3597848016, "avg_output_tokens_per_task": 30133.80094149294, "last_run_timestamp": "2026-09-26T09:56:28.417Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.8350426615384616, "accuracy_std_dev": 0.022313590127081184, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.08725940705128205, "avg_duration_per_task_ms": 939709.0711538461, "avg_output_tokens_per_task": 40061.0217948718, "last_run_timestamp": "2026-08-30T00:28:09.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto-beta", "model_slug": "openrouter/auto-beta", "model_name": "Auto Router (Beta)", "provider_name": "auto-routing", "accuracy": 0.835017, "accuracy_std_dev": 0.062145249638783075, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025753547138047138, "avg_duration_per_task_ms": 248297.88383838383, "avg_output_tokens_per_task": 30752.395622895623, "last_run_timestamp": "2026-09-15T14:04:40.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.835017, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.036737896464646466, "avg_duration_per_task_ms": 49536.31060606061, "avg_output_tokens_per_task": 14606.348484848484, "last_run_timestamp": "2026-09-01T17:49:40.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8348017092511013, "accuracy_std_dev": 0.013128455115293883, "run_count": 7, "total_questions": 1362, "avg_cost_per_task_usd": 0.03501912334801763, "avg_duration_per_task_ms": 566392.0389133627, "avg_output_tokens_per_task": 32664.345814977973, "last_run_timestamp": "2026-08-29T18:47:45.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8345117, "accuracy_std_dev": 0.03065694363777969, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.21535924595959594, "avg_duration_per_task_ms": 332148.0095959596, "avg_output_tokens_per_task": 48826.94343434343, "last_run_timestamp": "2026-09-26T08:56:02.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.8344516577181208, "accuracy_std_dev": 0.048501499999999975, "run_count": 2, "total_questions": 298, "avg_cost_per_task_usd": 0.0903846644295302, "avg_duration_per_task_ms": 304392.4194630872, "avg_output_tokens_per_task": 22866.939597315435, "last_run_timestamp": "2026-07-18T08:27:05.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.8343404728033473, "accuracy_std_dev": 0.00859056480406806, "run_count": 11, "total_questions": 2151, "avg_cost_per_task_usd": 0.07748330729893073, "avg_duration_per_task_ms": 794488.6968851697, "avg_output_tokens_per_task": 46585.74012087401, "last_run_timestamp": "2026-09-26T16:00:20.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8341984152249134, "accuracy_std_dev": 0.029002188381032377, "run_count": 3, "total_questions": 578, "avg_cost_per_task_usd": 0.005492470588235294, "avg_duration_per_task_ms": 956260.6712802768, "avg_output_tokens_per_task": 15404.778546712803, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.834175, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.028012333333333334, "avg_duration_per_task_ms": 352518.654040404, "avg_output_tokens_per_task": 14875.09595959596, "last_run_timestamp": "2026-09-26T09:19:39.685Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8340065999999998, "accuracy_std_dev": 0.032567889465545674, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.12823596313131314, "avg_duration_per_task_ms": 558007.0247474747, "avg_output_tokens_per_task": 38277.854545454546, "last_run_timestamp": "2026-09-26T08:50:56.054Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8339505777777776, "accuracy_std_dev": 0.01071049161258874, "run_count": 6, "total_questions": 1080, "avg_cost_per_task_usd": 0.025328656481481478, "avg_duration_per_task_ms": 650579.8324074075, "avg_output_tokens_per_task": 40296.40185185185, "last_run_timestamp": "2026-09-26T09:00:45.770Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.8338271333333335, "accuracy_std_dev": 0.012377564032868325, "run_count": 7, "total_questions": 1350, "avg_cost_per_task_usd": 0.007238588888888889, "avg_duration_per_task_ms": 324197.44074074074, "avg_output_tokens_per_task": 26128.143703703703, "last_run_timestamp": "2026-08-29T18:49:32.713Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8337061610738257, "accuracy_std_dev": 0.0693376467642218, "run_count": 5, "total_questions": 894, "avg_cost_per_task_usd": 0.009961087248322148, "avg_duration_per_task_ms": 513095.80536912754, "avg_output_tokens_per_task": 19701.401565995526, "last_run_timestamp": "2026-09-26T08:54:24.468Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.8334047490347491, "accuracy_std_dev": 0.03381421535995953, "run_count": 12, "total_questions": 2331, "avg_cost_per_task_usd": 0.056030425568425565, "avg_duration_per_task_ms": 279504.7528957529, "avg_output_tokens_per_task": 24216.713856713857, "last_run_timestamp": "2026-09-26T10:00:05.608Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.8333332499999999, "accuracy_std_dev": 0.01666596387664092, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.024300950126262627, "avg_duration_per_task_ms": 440107.09532828286, "avg_output_tokens_per_task": 22549.840909090908, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.8333332105263158, "accuracy_std_dev": 0.01599350000000005, "run_count": 2, "total_questions": 342, "avg_cost_per_task_usd": 0.16854262573099413, "avg_duration_per_task_ms": 685525.8274853801, "avg_output_tokens_per_task": 47357.02046783626, "last_run_timestamp": "2026-07-18T08:29:37.313Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.833159912591051, "accuracy_std_dev": 0.01602299589215448, "run_count": 5, "total_questions": 961, "avg_cost_per_task_usd": 0.14537621123829345, "avg_duration_per_task_ms": 277832.8116545265, "avg_output_tokens_per_task": 32850.81373569199, "last_run_timestamp": "2026-09-26T09:19:06.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.8331351764705882, "accuracy_std_dev": 0.011719105633588751, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.25533701039809864, "avg_duration_per_task_ms": 277560.3027332145, "avg_output_tokens_per_task": 55800.19964349376, "last_run_timestamp": "2026-09-26T09:16:57.240Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.8327431107738998, "accuracy_std_dev": 0.03352054924908005, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.1104804410723318, "avg_duration_per_task_ms": 986804.3333333334, "avg_output_tokens_per_task": 34519.57764289327, "last_run_timestamp": "2026-09-26T10:24:21.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.8326118571428571, "accuracy_std_dev": 0.010792989132217443, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.039859138528138535, "avg_duration_per_task_ms": 221347.44805194804, "avg_output_tokens_per_task": 14514.689033189034, "last_run_timestamp": "2026-09-26T09:19:17.245Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.8324914999999999, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.023044303030303033, "avg_duration_per_task_ms": 640134.2070707071, "avg_output_tokens_per_task": 54011.681818181816, "last_run_timestamp": "2026-09-26T08:54:28.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.8323770956022944, "accuracy_std_dev": 0.018917767858550327, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.07184262970044615, "avg_duration_per_task_ms": 878259.2613129382, "avg_output_tokens_per_task": 40890.33397068196, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.8323511666666668, "accuracy_std_dev": 0.013383536583139077, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1359044436026936, "avg_duration_per_task_ms": 151136.74915824915, "avg_output_tokens_per_task": 30646.316498316497, "last_run_timestamp": "2026-09-26T08:54:57.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.8319833976435935, "accuracy_std_dev": 0.0051619355292529005, "run_count": 7, "total_questions": 1358, "avg_cost_per_task_usd": 0.03598950073637703, "avg_duration_per_task_ms": 200380.98085419735, "avg_output_tokens_per_task": 13861.594256259204, "last_run_timestamp": "2026-09-26T09:19:46.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.8315411499051233, "accuracy_std_dev": 0.028738766047579353, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.02477825616698292, "avg_duration_per_task_ms": 479102.12713472487, "avg_output_tokens_per_task": 23444.680581910183, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.8312629347826086, "accuracy_std_dev": 0.044536486139877485, "run_count": 4, "total_questions": 644, "avg_cost_per_task_usd": 0.005694647515527951, "avg_duration_per_task_ms": 270388.4906832298, "avg_output_tokens_per_task": 13521.194099378881, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8311784671163575, "accuracy_std_dev": 0.009516052855602353, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.007403454749859472, "avg_duration_per_task_ms": 398416.0635188308, "avg_output_tokens_per_task": 19072.874648679033, "last_run_timestamp": "2026-08-29T16:40:58.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.8309764, "accuracy_std_dev": 0.005061551248382267, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.24307877272727274, "avg_duration_per_task_ms": 341071.7717171717, "avg_output_tokens_per_task": 40681.778787878786, "last_run_timestamp": "2026-09-26T08:54:33.880Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.8309340194750211, "accuracy_std_dev": 0.0227049955198312, "run_count": 6, "total_questions": 1181, "avg_cost_per_task_usd": 0.0975369652836579, "avg_duration_per_task_ms": 214278.25232853513, "avg_output_tokens_per_task": 28093.950889077052, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.8308604629080119, "accuracy_std_dev": 0.014193547884550253, "run_count": 7, "total_questions": 1348, "avg_cost_per_task_usd": 0.037891371661721074, "avg_duration_per_task_ms": 428497.83827893174, "avg_output_tokens_per_task": 44654.94213649852, "last_run_timestamp": "2026-09-26T08:56:01.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.830808, "accuracy_std_dev": 0.008551060402078817, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.09822325252525253, "avg_duration_per_task_ms": 297130.1747474747, "avg_output_tokens_per_task": 24088.215151515153, "last_run_timestamp": "2026-09-19T08:48:09.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.8306482182585346, "accuracy_std_dev": 0.018880885612038913, "run_count": 14, "total_questions": 2607, "avg_cost_per_task_usd": 0.18192265515918682, "avg_duration_per_task_ms": 363453.91829689295, "avg_output_tokens_per_task": 37006.87917146145, "last_run_timestamp": "2026-09-26T08:52:36.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.8298668701517706, "accuracy_std_dev": 0.04231782629659911, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.16520087015177065, "avg_duration_per_task_ms": 161607.96964586846, "avg_output_tokens_per_task": 16300.88813940416, "last_run_timestamp": "2026-09-26T08:54:23.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8296198191945158, "accuracy_std_dev": 0.010647141525623993, "run_count": 6, "total_questions": 1167, "avg_cost_per_task_usd": 0.1271454147386461, "avg_duration_per_task_ms": 247054.93658954583, "avg_output_tokens_per_task": 29738.498714652957, "last_run_timestamp": "2026-09-26T08:53:15.658Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.8295974490238612, "accuracy_std_dev": 0.024864472254399347, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.15712047794649311, "avg_duration_per_task_ms": 856729.4591467824, "avg_output_tokens_per_task": 51626.77368040492, "last_run_timestamp": "2026-08-29T20:19:40.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8294428739495799, "accuracy_std_dev": 0.015239815528915427, "run_count": 11, "total_questions": 2142, "avg_cost_per_task_usd": 0.009810573762838469, "avg_duration_per_task_ms": 575007.9150326797, "avg_output_tokens_per_task": 34796.17366946779, "last_run_timestamp": "2026-09-26T08:54:38.079Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8291486389610391, "accuracy_std_dev": 0.04390329909775197, "run_count": 6, "total_questions": 1155, "avg_cost_per_task_usd": 0.028787966233766234, "avg_duration_per_task_ms": 286215.91515151516, "avg_output_tokens_per_task": 8890.160173160173, "last_run_timestamp": "2026-09-26T09:45:13.813Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.8287637142857143, "accuracy_std_dev": 0.08258088982275724, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007342964646464646, "avg_duration_per_task_ms": 721777.8398268399, "avg_output_tokens_per_task": 30149.034632034633, "last_run_timestamp": "2026-09-26T11:30:22.248Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.8286193999999999, "accuracy_std_dev": 0.012838925852266618, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.01266686868686869, "avg_duration_per_task_ms": 861046.6656565657, "avg_output_tokens_per_task": 37375.524242424246, "last_run_timestamp": "2026-09-26T08:09:25.366Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.828497570984456, "accuracy_std_dev": 0.015230029187102692, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.11257541554404144, "avg_duration_per_task_ms": 438124.0186528497, "avg_output_tokens_per_task": 31261.672538860104, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.828283, "accuracy_std_dev": 0.016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00548029292929293, "avg_duration_per_task_ms": 1403265.9545454546, "avg_output_tokens_per_task": 25437.79292929293, "last_run_timestamp": "2026-07-18T08:42:46.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.8280124253521126, "accuracy_std_dev": 0.05892056354319715, "run_count": 11, "total_questions": 2130, "avg_cost_per_task_usd": 0.052598692957746475, "avg_duration_per_task_ms": 609008.6507042254, "avg_output_tokens_per_task": 25184.795774647886, "last_run_timestamp": "2026-09-26T08:01:18.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.8279570000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 186, "avg_cost_per_task_usd": 0.024709930107526885, "avg_duration_per_task_ms": 419969.8602150538, "avg_output_tokens_per_task": 20901.392473118278, "last_run_timestamp": "2026-08-01T18:03:43.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.827862, "accuracy_std_dev": 0.003834504661621904, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.009011289141414141, "avg_duration_per_task_ms": 370127.09217171714, "avg_output_tokens_per_task": 21495.685606060608, "last_run_timestamp": "2026-08-01T20:03:34.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.8278202748091604, "accuracy_std_dev": 0.008498300197686612, "run_count": 4, "total_questions": 786, "avg_cost_per_task_usd": 0.007172601781170484, "avg_duration_per_task_ms": 632005.1844783715, "avg_output_tokens_per_task": 17107.625954198473, "last_run_timestamp": "2026-08-01T18:25:37.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Fireworks", "accuracy": 0.8275528689788053, "accuracy_std_dev": 0.01888290543204505, "run_count": 3, "total_questions": 519, "avg_cost_per_task_usd": 0.03400616570327553, "avg_duration_per_task_ms": 162226.8689788054, "avg_output_tokens_per_task": 28516.98458574181, "last_run_timestamp": "2026-07-18T08:31:24.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8269733333333333, "accuracy_std_dev": 0.015136600014534322, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.06164716273849608, "avg_duration_per_task_ms": 496054.37934904604, "avg_output_tokens_per_task": 48584.97418630752, "last_run_timestamp": "2026-09-26T16:19:15.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.8269358, "accuracy_std_dev": 0.006767606146932625, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.04578402121212121, "avg_duration_per_task_ms": 60291.652525252524, "avg_output_tokens_per_task": 11285.754545454545, "last_run_timestamp": "2026-09-26T09:19:03.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.8268798333333333, "accuracy_std_dev": 0.036978311140526084, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1489224217171717, "avg_duration_per_task_ms": 1008310.4612794613, "avg_output_tokens_per_task": 52023.04461279461, "last_run_timestamp": "2026-08-29T18:40:24.764Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.826599, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04722597138047138, "avg_duration_per_task_ms": 258616.0824915825, "avg_output_tokens_per_task": 15105.597643097643, "last_run_timestamp": "2026-09-26T09:35:33.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.826575279270147, "accuracy_std_dev": 0.033978737517600645, "run_count": 10, "total_questions": 1973, "avg_cost_per_task_usd": 0.06587307298530158, "avg_duration_per_task_ms": 1029346.9406994425, "avg_output_tokens_per_task": 33210.919412062845, "last_run_timestamp": "2026-09-26T10:24:45.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.8265687442542787, "accuracy_std_dev": 0.1992607600768644, "run_count": 12, "total_questions": 2045, "avg_cost_per_task_usd": 0.20695833007334963, "avg_duration_per_task_ms": 332203.7985330073, "avg_output_tokens_per_task": 37422.67432762836, "last_run_timestamp": "2026-09-26T08:56:04.781Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8264543613323679, "accuracy_std_dev": 0.018188360323715465, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.20616153874004342, "avg_duration_per_task_ms": 315117.1817523534, "avg_output_tokens_per_task": 46875.9225199131, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8263186666666665, "accuracy_std_dev": 0.011640048205322105, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.04355452946127946, "avg_duration_per_task_ms": 227785.6026936027, "avg_output_tokens_per_task": 14401.111111111111, "last_run_timestamp": "2026-09-26T09:34:17.391Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.8262184258271078, "accuracy_std_dev": 0.01942525748194862, "run_count": 5, "total_questions": 937, "avg_cost_per_task_usd": 0.02446647598719317, "avg_duration_per_task_ms": 331857.0437566702, "avg_output_tokens_per_task": 48723.26467449306, "last_run_timestamp": "2026-09-26T08:54:40.393Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8261094014447884, "accuracy_std_dev": 0.026551028658038853, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.2076026150670795, "avg_duration_per_task_ms": 857894.439628483, "avg_output_tokens_per_task": 47560.28689370485, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8258873707224335, "accuracy_std_dev": 0.009676826643527784, "run_count": 6, "total_questions": 1052, "avg_cost_per_task_usd": 0.025350776615969583, "avg_duration_per_task_ms": 120122.08840304182, "avg_output_tokens_per_task": 20151.38783269962, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.8255894000000001, "accuracy_std_dev": 0.08582614429554669, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007835238383838384, "avg_duration_per_task_ms": 420094.35454545455, "avg_output_tokens_per_task": 28284.40707070707, "last_run_timestamp": "2026-08-29T18:30:11.054Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.8253399469387755, "accuracy_std_dev": 0.06202361079266511, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.048117852040816325, "avg_duration_per_task_ms": 1077180.0714285714, "avg_output_tokens_per_task": 26155.976530612246, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.8249159000000001, "accuracy_std_dev": 0.02643692337035457, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.12457927070707071, "avg_duration_per_task_ms": 523158.02121212124, "avg_output_tokens_per_task": 36100.31464646465, "last_run_timestamp": "2026-09-26T08:54:25.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.82491575, "accuracy_std_dev": 0.04835478209481976, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09927686868686869, "avg_duration_per_task_ms": 820653.2960858586, "avg_output_tokens_per_task": 39684.74116161616, "last_run_timestamp": "2026-08-29T18:40:35.044Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.8249157142857142, "accuracy_std_dev": 0.019837994431424387, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1225871738816739, "avg_duration_per_task_ms": 339042.2626262626, "avg_output_tokens_per_task": 50612.39971139971, "last_run_timestamp": "2026-09-26T08:56:23.149Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.8246351666666666, "accuracy_std_dev": 0.013862659393933843, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.034730324915824916, "avg_duration_per_task_ms": 240485.28282828283, "avg_output_tokens_per_task": 13780.222222222223, "last_run_timestamp": "2026-09-26T09:15:01.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.8246351666666666, "accuracy_std_dev": 0.0064044884369392945, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.13257150673400672, "avg_duration_per_task_ms": 144313.25757575757, "avg_output_tokens_per_task": 32437.94191919192, "last_run_timestamp": "2026-09-26T08:54:42.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.824495, "accuracy_std_dev": 0.00300556117888158, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.04055735353535353, "avg_duration_per_task_ms": 210547.6452020202, "avg_output_tokens_per_task": 15056.00883838384, "last_run_timestamp": "2026-09-26T09:19:00.848Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.82449475, "accuracy_std_dev": 0.010869783171135461, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06350338131313131, "avg_duration_per_task_ms": 123563.64393939394, "avg_output_tokens_per_task": 13357.755050505051, "last_run_timestamp": "2026-09-01T17:56:39.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.8244612319391635, "accuracy_std_dev": 0.0189305399949394, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.021243255386565272, "avg_duration_per_task_ms": 482977.82002534857, "avg_output_tokens_per_task": 22160.196451204054, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.8239101113924051, "accuracy_std_dev": 0.05687472058441742, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.21822399915611815, "avg_duration_per_task_ms": 1618621.7156118143, "avg_output_tokens_per_task": 51785.03459915612, "last_run_timestamp": "2026-08-29T18:42:16.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.823772114732725, "accuracy_std_dev": 0.15112751242638614, "run_count": 8, "total_questions": 1534, "avg_cost_per_task_usd": 0.06556984028683181, "avg_duration_per_task_ms": 284366.7822685789, "avg_output_tokens_per_task": 18120.139504563234, "last_run_timestamp": "2026-08-29T19:36:49.687Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8237374, "accuracy_std_dev": 0.08853685691078036, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.1022956090909091, "avg_duration_per_task_ms": 896671.251010101, "avg_output_tokens_per_task": 43699.027272727275, "last_run_timestamp": "2026-09-26T09:26:01.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.8237132857142857, "accuracy_std_dev": 0.024208164205574968, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.037303974025974025, "avg_duration_per_task_ms": 279740.5851370851, "avg_output_tokens_per_task": 14323.054112554113, "last_run_timestamp": "2026-09-26T09:20:53.278Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8234849, "accuracy_std_dev": 0.025448233802957704, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04444430303030303, "avg_duration_per_task_ms": 358421.2823232323, "avg_output_tokens_per_task": 46840.29242424243, "last_run_timestamp": "2026-09-26T16:24:56.918Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.8232579938524589, "accuracy_std_dev": 0.05951885425846167, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.08256164446721312, "avg_duration_per_task_ms": 628781.7581967213, "avg_output_tokens_per_task": 23867.87807377049, "last_run_timestamp": "2026-09-26T09:35:44.547Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.823232375, "accuracy_std_dev": 0.046813672196105, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.08898095391414142, "avg_duration_per_task_ms": 476497.32765151514, "avg_output_tokens_per_task": 20279.160984848484, "last_run_timestamp": "2026-08-29T18:50:46.982Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.8228419167330677, "accuracy_std_dev": 0.11242608196694337, "run_count": 13, "total_questions": 2510, "avg_cost_per_task_usd": 0.007463425099601593, "avg_duration_per_task_ms": 612656.6079681275, "avg_output_tokens_per_task": 26820.300398406376, "last_run_timestamp": "2026-09-26T09:19:42.439Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.8227513720634919, "accuracy_std_dev": 0.011297792117682792, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.025156736507936508, "avg_duration_per_task_ms": 449467.82285714283, "avg_output_tokens_per_task": 25121.824126984127, "last_run_timestamp": "2026-08-29T16:15:12.678Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.822297, "accuracy_std_dev": 0.014221076658725011, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.31472648260381597, "avg_duration_per_task_ms": 289101.1150392817, "avg_output_tokens_per_task": 20792.76374859708, "last_run_timestamp": "2026-09-26T08:55:59.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.8222220723809524, "accuracy_std_dev": 0.00641547133888074, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.007690131428571429, "avg_duration_per_task_ms": 591255.7904761905, "avg_output_tokens_per_task": 17900.798095238097, "last_run_timestamp": "2026-08-29T18:40:24.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8221650670103093, "accuracy_std_dev": 0.01835186937268852, "run_count": 6, "total_questions": 1164, "avg_cost_per_task_usd": 0.13827411597938147, "avg_duration_per_task_ms": 261027.06529209623, "avg_output_tokens_per_task": 31195.84621993127, "last_run_timestamp": "2026-09-26T09:19:47.728Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.8221100000000001, "accuracy_std_dev": 0.019325780306224248, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.011398650392817058, "avg_duration_per_task_ms": 623049.0549943884, "avg_output_tokens_per_task": 33683.20538720539, "last_run_timestamp": "2026-09-26T09:34:52.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.8221098333333334, "accuracy_std_dev": 0.033805456691766336, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0906014957912458, "avg_duration_per_task_ms": 1460808.6498316498, "avg_output_tokens_per_task": 42233.86784511784, "last_run_timestamp": "2026-08-29T19:12:56.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.821969625, "accuracy_std_dev": 0.0188925810831759, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10310438383838386, "avg_duration_per_task_ms": 642163.4210858586, "avg_output_tokens_per_task": 38222.52777777778, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.8216439045553144, "accuracy_std_dev": 0.014190341770056845, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.18553550686912507, "avg_duration_per_task_ms": 863914.5343456254, "avg_output_tokens_per_task": 49671.55531453362, "last_run_timestamp": "2026-08-29T18:39:39.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.821549, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.001158979797979798, "avg_duration_per_task_ms": 153563.0101010101, "avg_output_tokens_per_task": 18533.79292929293, "last_run_timestamp": "2026-09-23T18:11:58.526Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8215488, "accuracy_std_dev": 0.011711955283384605, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.039391856565656566, "avg_duration_per_task_ms": 244469.52525252526, "avg_output_tokens_per_task": 15370.382828282829, "last_run_timestamp": "2026-09-26T09:19:26.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.821233, "accuracy_std_dev": 0.011708622773409337, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09879417929292929, "avg_duration_per_task_ms": 368109.4835858586, "avg_output_tokens_per_task": 45695.4898989899, "last_run_timestamp": "2026-08-30T00:09:46.578Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.8211350956521739, "accuracy_std_dev": 0.03084416399732497, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.013802629710144926, "avg_duration_per_task_ms": 81921.22101449275, "avg_output_tokens_per_task": 17742.007971014493, "last_run_timestamp": "2026-08-29T18:36:28.592Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.820920596529284, "accuracy_std_dev": 0.007605756273362683, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.019052912509038324, "avg_duration_per_task_ms": 539380.6218365871, "avg_output_tokens_per_task": 18914.664497469268, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.8207635888324873, "accuracy_std_dev": 0.020270393054191847, "run_count": 7, "total_questions": 1379, "avg_cost_per_task_usd": 0.03637786294416244, "avg_duration_per_task_ms": 250330.74111675128, "avg_output_tokens_per_task": 14481.398839738942, "last_run_timestamp": "2026-09-26T08:56:59.861Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.8206135555555556, "accuracy_std_dev": 0.02409886708315059, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03784499887766555, "avg_duration_per_task_ms": 323218.42199775536, "avg_output_tokens_per_task": 39347.21212121212, "last_run_timestamp": "2026-09-26T16:29:49.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.8201054571428571, "accuracy_std_dev": 0.026122159766927572, "run_count": 3, "total_questions": 315, "avg_cost_per_task_usd": 0.055772555555555554, "avg_duration_per_task_ms": 137698.42222222223, "avg_output_tokens_per_task": 23066.479365079365, "last_run_timestamp": "2026-09-12T16:00:25.695Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8198652222222221, "accuracy_std_dev": 0.06079266623857736, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.027504470819304155, "avg_duration_per_task_ms": 620185.7014590348, "avg_output_tokens_per_task": 28854.86756453423, "last_run_timestamp": "2026-09-26T09:17:20.976Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.8193043333333333, "accuracy_std_dev": 0.0031744380430068155, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14333356734006736, "avg_duration_per_task_ms": 321739.1531986532, "avg_output_tokens_per_task": 28661.96127946128, "last_run_timestamp": "2026-09-26T09:35:22.612Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.8193041666666666, "accuracy_std_dev": 0.02287040255742972, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006429800505050506, "avg_duration_per_task_ms": 501214.0008417508, "avg_output_tokens_per_task": 17257.11026936027, "last_run_timestamp": "2026-08-29T19:56:51.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8191512936660268, "accuracy_std_dev": 0.031450804747007274, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.025433834932821493, "avg_duration_per_task_ms": 455424.4030710173, "avg_output_tokens_per_task": 21193.655790147153, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.8186799411764705, "accuracy_std_dev": 0.009390308467211003, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.019136678051865905, "avg_duration_per_task_ms": 558518.3769765971, "avg_output_tokens_per_task": 18938.588867805185, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.8186026249999999, "accuracy_std_dev": 0.021390337134659064, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.04284533396464647, "avg_duration_per_task_ms": 182582.7582070707, "avg_output_tokens_per_task": 23700.921085858587, "last_run_timestamp": "2026-08-29T18:13:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.8184969532163743, "accuracy_std_dev": 0.012420924411426656, "run_count": 8, "total_questions": 1539, "avg_cost_per_task_usd": 0.04671114684860298, "avg_duration_per_task_ms": 443069.649122807, "avg_output_tokens_per_task": 25928.164392462637, "last_run_timestamp": "2026-08-29T16:44:34.185Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8181816666666667, "accuracy_std_dev": 0.013255807590972678, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.17135338047138046, "avg_duration_per_task_ms": 45814.95117845118, "avg_output_tokens_per_task": 3234.355218855219, "last_run_timestamp": "2026-08-15T09:15:35.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.8181631193111932, "accuracy_std_dev": 0.03197110297229838, "run_count": 7, "total_questions": 813, "avg_cost_per_task_usd": 0.06679120172201722, "avg_duration_per_task_ms": 131925.82779827798, "avg_output_tokens_per_task": 28854.319803198032, "last_run_timestamp": "2026-08-15T14:54:40.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.8180902577475433, "accuracy_std_dev": 0.11577773679656365, "run_count": 8, "total_questions": 1323, "avg_cost_per_task_usd": 0.046483944066515494, "avg_duration_per_task_ms": 395321.3703703704, "avg_output_tokens_per_task": 15410.478458049887, "last_run_timestamp": "2026-08-29T18:43:58.355Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.818035862576065, "accuracy_std_dev": 0.01278056208505712, "run_count": 10, "total_questions": 1972, "avg_cost_per_task_usd": 0.0909173985801217, "avg_duration_per_task_ms": 165913.36257606492, "avg_output_tokens_per_task": 22308.299188640973, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.817971375, "accuracy_std_dev": 0.021218711111525495, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0422043106060606, "avg_duration_per_task_ms": 258582.00315656565, "avg_output_tokens_per_task": 14114.647727272728, "last_run_timestamp": "2026-09-26T09:36:02.338Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8177008571428571, "accuracy_std_dev": 0.044711167868022074, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.16188713708513708, "avg_duration_per_task_ms": 160453.3152958153, "avg_output_tokens_per_task": 16045.246753246753, "last_run_timestamp": "2026-09-26T08:55:42.287Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.8176514288840263, "accuracy_std_dev": 0.03360829910201664, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.22697232020423047, "avg_duration_per_task_ms": 936805.9460247994, "avg_output_tokens_per_task": 51848.20277169949, "last_run_timestamp": "2026-08-29T18:07:06.751Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8165784761904761, "accuracy_std_dev": 0.01245241634466425, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.3341618246753246, "avg_duration_per_task_ms": 300449.1738816739, "avg_output_tokens_per_task": 21181.114718614717, "last_run_timestamp": "2026-09-26T08:51:53.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.8158729085714285, "accuracy_std_dev": 0.03348131954164889, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.07726252825396825, "avg_duration_per_task_ms": 943986.486984127, "avg_output_tokens_per_task": 39783.29777777778, "last_run_timestamp": "2026-08-30T00:30:19.495Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.8157559809402797, "accuracy_std_dev": 0.02360825327613207, "run_count": 8, "total_questions": 1574, "avg_cost_per_task_usd": 0.021597616899618807, "avg_duration_per_task_ms": 295953.15184243966, "avg_output_tokens_per_task": 17957.35959339263, "last_run_timestamp": "2026-09-26T08:54:33.107Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.8155159532293987, "accuracy_std_dev": 0.03005979094119869, "run_count": 7, "total_questions": 1347, "avg_cost_per_task_usd": 0.07733159317000743, "avg_duration_per_task_ms": 1465901.4966592428, "avg_output_tokens_per_task": 44227.633259094284, "last_run_timestamp": "2026-08-22T17:50:04.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.8154496285714286, "accuracy_std_dev": 0.0068932906227268985, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.006205518095238096, "avg_duration_per_task_ms": 548094.434920635, "avg_output_tokens_per_task": 18728.389206349206, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.8154462499999999, "accuracy_std_dev": 0.00789342203594232, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.024953141414141414, "avg_duration_per_task_ms": 185879.93371212122, "avg_output_tokens_per_task": 16543.260101010103, "last_run_timestamp": "2026-09-26T08:53:11.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8153759444444445, "accuracy_std_dev": 0.010779567108769665, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.36484757070707075, "avg_duration_per_task_ms": 359825.24242424243, "avg_output_tokens_per_task": 22980.845679012345, "last_run_timestamp": "2026-09-26T08:54:41.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8151659858437146, "accuracy_std_dev": 0.06632272963083027, "run_count": 19, "total_questions": 3532, "avg_cost_per_task_usd": 0.00728941053227633, "avg_duration_per_task_ms": 452529.82446206117, "avg_output_tokens_per_task": 16964.4419592299, "last_run_timestamp": "2026-08-29T20:12:46.798Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8144508860510805, "accuracy_std_dev": 0.03052507251322263, "run_count": 8, "total_questions": 1527, "avg_cost_per_task_usd": 0.1842238932547479, "avg_duration_per_task_ms": 842214.8297314997, "avg_output_tokens_per_task": 52169.59397511461, "last_run_timestamp": "2026-08-29T18:14:11.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8143338571428571, "accuracy_std_dev": 0.014248444761532711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026212479076479073, "avg_duration_per_task_ms": 329590.34992784995, "avg_output_tokens_per_task": 17511.52380952381, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.814024, "accuracy_std_dev": null, "run_count": 1, "total_questions": 164, "avg_cost_per_task_usd": 0.010100225609756097, "avg_duration_per_task_ms": 515065.23780487804, "avg_output_tokens_per_task": 33245.84146341463, "last_run_timestamp": "2026-09-28T09:46:11.365Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.8138834622823985, "accuracy_std_dev": 0.0221782773339139, "run_count": 8, "total_questions": 1551, "avg_cost_per_task_usd": 0.16634050999355257, "avg_duration_per_task_ms": 905404.1012250162, "avg_output_tokens_per_task": 52917.24887169568, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.8136879467680608, "accuracy_std_dev": 0.01151239505652493, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.007660202154626108, "avg_duration_per_task_ms": 445275.62420785805, "avg_output_tokens_per_task": 19701.929657794677, "last_run_timestamp": "2026-08-29T19:19:45.463Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.8136272825651302, "accuracy_std_dev": 0.008773473238499465, "run_count": 3, "total_questions": 499, "avg_cost_per_task_usd": 0.0068851422845691386, "avg_duration_per_task_ms": 1090136, "avg_output_tokens_per_task": 17404.21643286573, "last_run_timestamp": "2026-08-15T10:05:32.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.813131, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.037260742424242425, "avg_duration_per_task_ms": 35156.99494949495, "avg_output_tokens_per_task": 2566.8030303030305, "last_run_timestamp": "2026-08-01T08:32:23.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.8122604827586207, "accuracy_std_dev": 0.00611459963257337, "run_count": 3, "total_questions": 522, "avg_cost_per_task_usd": 0.07136742145593869, "avg_duration_per_task_ms": 345248.30842911877, "avg_output_tokens_per_task": 17346.159003831417, "last_run_timestamp": "2026-07-18T08:30:38.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.812016, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.14135672674418603, "avg_duration_per_task_ms": 305120.88372093026, "avg_output_tokens_per_task": 39843.74418604651, "last_run_timestamp": "2026-08-04T03:11:59.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.8112167885714286, "accuracy_std_dev": 0.01823961098089469, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.022302389206349204, "avg_duration_per_task_ms": 338892.4336507936, "avg_output_tokens_per_task": 19867.52, "last_run_timestamp": "2026-08-29T18:59:02.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.8108436061269148, "accuracy_std_dev": 0.03441946501380775, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.21617651641137853, "avg_duration_per_task_ms": 867715.7636761488, "avg_output_tokens_per_task": 54511.27789934354, "last_run_timestamp": "2026-08-29T18:09:15.344Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.8108163749999999, "accuracy_std_dev": 0.010578591517039247, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03051571590909091, "avg_duration_per_task_ms": 224324.35542929292, "avg_output_tokens_per_task": 24211.31691919192, "last_run_timestamp": "2026-08-29T19:24:09.844Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.8107742, "accuracy_std_dev": 0.00793923498581567, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.032759715151515156, "avg_duration_per_task_ms": 674236.5636363636, "avg_output_tokens_per_task": 15717.559595959596, "last_run_timestamp": "2026-09-26T09:29:37.459Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.8099527992277992, "accuracy_std_dev": 0.02090205677678631, "run_count": 4, "total_questions": 777, "avg_cost_per_task_usd": 0.00813298326898327, "avg_duration_per_task_ms": 668505.0875160876, "avg_output_tokens_per_task": 19405.019305019305, "last_run_timestamp": "2026-08-01T18:07:34.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.8097504682741116, "accuracy_std_dev": 0.10898800173843678, "run_count": 8, "total_questions": 1576, "avg_cost_per_task_usd": 0.1267229282994924, "avg_duration_per_task_ms": 700808.2106598985, "avg_output_tokens_per_task": 40110.88578680203, "last_run_timestamp": "2026-08-29T18:23:09.250Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.8094891786492375, "accuracy_std_dev": 0.052310168338791006, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.23138632244008714, "avg_duration_per_task_ms": 967370.0573710966, "avg_output_tokens_per_task": 53108.82788671024, "last_run_timestamp": "2026-08-29T18:21:26.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.8092030833333335, "accuracy_std_dev": 0.01675517045301904, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.33934875, "avg_duration_per_task_ms": 354134.20664983167, "avg_output_tokens_per_task": 22457.28787878788, "last_run_timestamp": "2026-09-26T08:56:43.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.8091603740458015, "accuracy_std_dev": 0.1072178269085883, "run_count": 12, "total_questions": 2358, "avg_cost_per_task_usd": 0.04859381764206954, "avg_duration_per_task_ms": 198361.56318914334, "avg_output_tokens_per_task": 23259.173452078034, "last_run_timestamp": "2026-09-26T09:18:49.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.8088945473145778, "accuracy_std_dev": 0.022541716685711595, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.00648230179028133, "avg_duration_per_task_ms": 718133.9377664109, "avg_output_tokens_per_task": 17528.813299232737, "last_run_timestamp": "2026-08-29T19:15:43.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.8081382125237192, "accuracy_std_dev": 0.024044703190099895, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.1062299411764706, "avg_duration_per_task_ms": 740176.1967109424, "avg_output_tokens_per_task": 54188.096774193546, "last_run_timestamp": "2026-08-29T19:17:49.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.808081, "accuracy_std_dev": 0.12407749055328285, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.10305112289562289, "avg_duration_per_task_ms": 1499684.5976430976, "avg_output_tokens_per_task": 25661.05218855219, "last_run_timestamp": "2026-09-26T09:01:35.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.8074670728744938, "accuracy_std_dev": 0.01649167806046429, "run_count": 4, "total_questions": 741, "avg_cost_per_task_usd": 0.07785456275303644, "avg_duration_per_task_ms": 468008.1632928475, "avg_output_tokens_per_task": 33473.376518218625, "last_run_timestamp": "2026-08-01T17:52:55.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.8073004516129032, "accuracy_std_dev": 0.05300431140963913, "run_count": 10, "total_questions": 1767, "avg_cost_per_task_usd": 0.046523333333333326, "avg_duration_per_task_ms": 546722.9988681381, "avg_output_tokens_per_task": 25674.418222976798, "last_run_timestamp": "2026-09-26T09:56:00.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.8072388749999999, "accuracy_std_dev": 0.022365952279287706, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.007244868686868686, "avg_duration_per_task_ms": 515270.8522727273, "avg_output_tokens_per_task": 18306.11994949495, "last_run_timestamp": "2026-08-29T19:12:36.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "WandB", "accuracy": 0.8068935127478752, "accuracy_std_dev": 0.009178499999999978, "run_count": 2, "total_questions": 353, "avg_cost_per_task_usd": 0.033640501416430595, "avg_duration_per_task_ms": 174824.1133144476, "avg_output_tokens_per_task": 26792.110481586402, "last_run_timestamp": "2026-07-18T08:04:57.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.8063971999999998, "accuracy_std_dev": 0.01659741351415936, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00614619090909091, "avg_duration_per_task_ms": 535780.8212121212, "avg_output_tokens_per_task": 18359.183838383837, "last_run_timestamp": "2026-08-29T19:12:20.528Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.805556, "accuracy_std_dev": null, "run_count": 1, "total_questions": 192, "avg_cost_per_task_usd": 0.16585933333333333, "avg_duration_per_task_ms": 688743.8177083334, "avg_output_tokens_per_task": 42426.447916666664, "last_run_timestamp": "2026-07-31T00:21:35.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.805555546875, "accuracy_std_dev": 0.01288499999999998, "run_count": 2, "total_questions": 384, "avg_cost_per_task_usd": 0.0362872109375, "avg_duration_per_task_ms": 729621.7864583334, "avg_output_tokens_per_task": 36271.864583333336, "last_run_timestamp": "2026-09-12T16:45:54.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.8054487816091953, "accuracy_std_dev": 0.036888297226220654, "run_count": 4, "total_questions": 783, "avg_cost_per_task_usd": 0.008708275862068965, "avg_duration_per_task_ms": 1038271.9885057472, "avg_output_tokens_per_task": 20826.485312899105, "last_run_timestamp": "2026-08-01T22:57:01.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.8051947142857142, "accuracy_std_dev": 0.014445950710090603, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.022680748196248195, "avg_duration_per_task_ms": 254100.05483405484, "avg_output_tokens_per_task": 17520.435786435788, "last_run_timestamp": "2026-09-26T08:51:26.240Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.805091904400607, "accuracy_std_dev": 0.04892398259146531, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.08381860748609003, "avg_duration_per_task_ms": 790750.7895801719, "avg_output_tokens_per_task": 31756.94537177542, "last_run_timestamp": "2026-09-26T10:35:48.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.8049008888888889, "accuracy_std_dev": 0.023075324904150284, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.3397021683501683, "avg_duration_per_task_ms": 355471.79349046014, "avg_output_tokens_per_task": 22495.737373737375, "last_run_timestamp": "2026-09-26T09:00:45.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.804714, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.024081237373737373, "avg_duration_per_task_ms": 24354.015151515152, "avg_output_tokens_per_task": 1625.5555555555557, "last_run_timestamp": "2026-08-01T08:38:11.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.804676697257384, "accuracy_std_dev": 0.059236782672255274, "run_count": 5, "total_questions": 948, "avg_cost_per_task_usd": 0.023476022151898732, "avg_duration_per_task_ms": 1321432.723628692, "avg_output_tokens_per_task": 50466.419831223626, "last_run_timestamp": "2026-09-30T00:39:58.995Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.8038987567251462, "accuracy_std_dev": 0.03143660042823293, "run_count": 8, "total_questions": 855, "avg_cost_per_task_usd": 0.02817445146198831, "avg_duration_per_task_ms": 97917.72046783626, "avg_output_tokens_per_task": 30151.957894736843, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.802867256166983, "accuracy_std_dev": 0.022011618153474646, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.02734331878557875, "avg_duration_per_task_ms": 434011.0303605313, "avg_output_tokens_per_task": 22775.223276407338, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8025491428571428, "accuracy_std_dev": 0.025111070964409142, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.33407534559884555, "avg_duration_per_task_ms": 352449.7582972583, "avg_output_tokens_per_task": 22169.862914862915, "last_run_timestamp": "2026-09-26T08:59:18.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.8023003671874999, "accuracy_std_dev": 0.01483082484009234, "run_count": 8, "total_questions": 1536, "avg_cost_per_task_usd": 0.016652225911458332, "avg_duration_per_task_ms": 384547.912109375, "avg_output_tokens_per_task": 15953.287760416666, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8021885000000001, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04024864141414142, "avg_duration_per_task_ms": 73206.09595959596, "avg_output_tokens_per_task": 26709.396464646463, "last_run_timestamp": "2026-09-01T17:53:21.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8016385562742562, "accuracy_std_dev": 0.007745278218856984, "run_count": 4, "total_questions": 773, "avg_cost_per_task_usd": 0.0594386532988357, "avg_duration_per_task_ms": 220979.64165588617, "avg_output_tokens_per_task": 17329.77360931436, "last_run_timestamp": "2026-09-26T08:53:08.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8013465, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.041531338383838386, "avg_duration_per_task_ms": 80828.55555555556, "avg_output_tokens_per_task": 27572.47474747475, "last_run_timestamp": "2026-09-01T17:50:00.305Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.8009767048665621, "accuracy_std_dev": 0.1773041368489748, "run_count": 10, "total_questions": 1911, "avg_cost_per_task_usd": 0.0910193940345369, "avg_duration_per_task_ms": 503465.8440607012, "avg_output_tokens_per_task": 24546.500784929358, "last_run_timestamp": "2026-09-26T08:54:41.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.8007155, "accuracy_std_dev": 0.04390787837905629, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.039541025883838385, "avg_duration_per_task_ms": 942411.1104797979, "avg_output_tokens_per_task": 24048.43876262626, "last_run_timestamp": "2026-09-26T09:34:13.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.8001721395348838, "accuracy_std_dev": 0.0356745, "run_count": 2, "total_questions": 387, "avg_cost_per_task_usd": 0.184302338501292, "avg_duration_per_task_ms": 1402883.9819121447, "avg_output_tokens_per_task": 51056.865633074936, "last_run_timestamp": "2026-09-28T18:08:56.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.8001197163375224, "accuracy_std_dev": 0.08791149969002918, "run_count": 8, "total_questions": 1114, "avg_cost_per_task_usd": 0.010599297127468581, "avg_duration_per_task_ms": 62868.36624775583, "avg_output_tokens_per_task": 10694.411131059245, "last_run_timestamp": "2026-08-29T18:12:40.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.7988215, "accuracy_std_dev": 0.010942500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.25381196717171717, "avg_duration_per_task_ms": 225693.02525252526, "avg_output_tokens_per_task": 25403.512626262625, "last_run_timestamp": "2026-09-01T13:22:15.928Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.7987011428571428, "accuracy_std_dev": 0.00925850994535102, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04827445887445887, "avg_duration_per_task_ms": 463705.8015873016, "avg_output_tokens_per_task": 25011.621933621933, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.79840075, "accuracy_std_dev": 0.009249753655503483, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.014099073863636364, "avg_duration_per_task_ms": 333383.3731060606, "avg_output_tokens_per_task": 14661.142676767677, "last_run_timestamp": "2026-08-30T00:01:09.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.79798, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2411195303030303, "avg_duration_per_task_ms": 184685.6616161616, "avg_output_tokens_per_task": 24018.91919191919, "last_run_timestamp": "2026-09-01T13:34:01.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.79798, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03602157070707071, "avg_duration_per_task_ms": 32858.843434343435, "avg_output_tokens_per_task": 2479.818181818182, "last_run_timestamp": "2026-08-01T08:32:36.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.7979798, "accuracy_std_dev": 0.09258320044241286, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010301554545454545, "avg_duration_per_task_ms": 568986.8757575757, "avg_output_tokens_per_task": 24756.19292929293, "last_run_timestamp": "2026-08-29T19:51:27.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7979040119760479, "accuracy_std_dev": 0.02389207889003172, "run_count": 4, "total_questions": 668, "avg_cost_per_task_usd": 0.15626441916167666, "avg_duration_per_task_ms": 1788980.868263473, "avg_output_tokens_per_task": 46933.58532934132, "last_run_timestamp": "2026-08-22T16:44:47.658Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.7974987142857143, "accuracy_std_dev": 0.009164318316387839, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01350666378066378, "avg_duration_per_task_ms": 299956.4531024531, "avg_output_tokens_per_task": 14063.395382395382, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Fireworks", "accuracy": 0.797043, "accuracy_std_dev": null, "run_count": 1, "total_questions": 124, "avg_cost_per_task_usd": 0.11967770967741935, "avg_duration_per_task_ms": 375576.0080645161, "avg_output_tokens_per_task": 29097.58064516129, "last_run_timestamp": "2026-07-18T08:32:17.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7969696999999999, "accuracy_std_dev": 0.10557091126825607, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.09371603636363636, "avg_duration_per_task_ms": 530749.2, "avg_output_tokens_per_task": 34572.13787878788, "last_run_timestamp": "2026-09-26T09:22:12.564Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7969276249999999, "accuracy_std_dev": 0.020340632081731754, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.027590136994949492, "avg_duration_per_task_ms": 278109.5505050505, "avg_output_tokens_per_task": 18237.960227272728, "last_run_timestamp": "2026-08-29T18:07:47.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.7968618091603052, "accuracy_std_dev": 0.006526907862839814, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.055094477735368956, "avg_duration_per_task_ms": 1236078.9319338421, "avg_output_tokens_per_task": 25116.7868956743, "last_run_timestamp": "2026-08-29T20:02:23.660Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7964375923664123, "accuracy_std_dev": 0.11378331267012533, "run_count": 4, "total_questions": 655, "avg_cost_per_task_usd": 0.011028116030534352, "avg_duration_per_task_ms": 270893.28854961833, "avg_output_tokens_per_task": 23218.567938931297, "last_run_timestamp": "2026-09-12T08:53:48.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7962965, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.044405648989898995, "avg_duration_per_task_ms": 249218.0202020202, "avg_output_tokens_per_task": 22108.909090909092, "last_run_timestamp": "2026-09-01T18:00:27.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.796296, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04404542424242425, "avg_duration_per_task_ms": 221111.49494949495, "avg_output_tokens_per_task": 21928.032828282827, "last_run_timestamp": "2026-09-01T18:07:51.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.7956228, "accuracy_std_dev": 0.021522407276138952, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006844225252525253, "avg_duration_per_task_ms": 363476.91515151516, "avg_output_tokens_per_task": 16652.98888888889, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7955469441233141, "accuracy_std_dev": 0.07292560689454357, "run_count": 8, "total_questions": 1557, "avg_cost_per_task_usd": 0.18441899165061015, "avg_duration_per_task_ms": 1050598.7052023122, "avg_output_tokens_per_task": 59274.58445728966, "last_run_timestamp": "2026-08-29T18:11:51.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7945011215854313, "accuracy_std_dev": 0.05670915857959413, "run_count": 12, "total_questions": 1867, "avg_cost_per_task_usd": 0.05605855168719871, "avg_duration_per_task_ms": 749155.3829673273, "avg_output_tokens_per_task": 45440.918050348155, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7942543479087453, "accuracy_std_dev": 0.04886147848238298, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.1139862959442332, "avg_duration_per_task_ms": 955060.933460076, "avg_output_tokens_per_task": 52224.47401774398, "last_run_timestamp": "2026-08-29T18:40:30.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.79419175, "accuracy_std_dev": 0.04229542576553049, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.027481944444444444, "avg_duration_per_task_ms": 377974.2102272727, "avg_output_tokens_per_task": 25769.56186868687, "last_run_timestamp": "2026-08-29T19:18:38.408Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7936028, "accuracy_std_dev": 0.016273367695716804, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.017573431313131313, "avg_duration_per_task_ms": 59390.138383838384, "avg_output_tokens_per_task": 5706.428282828283, "last_run_timestamp": "2026-08-29T18:05:48.084Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7930948249452954, "accuracy_std_dev": 0.06762071684847804, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.17062498103574034, "avg_duration_per_task_ms": 873257.2749817652, "avg_output_tokens_per_task": 58045.98614150255, "last_run_timestamp": "2026-08-29T19:22:50.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7929289999999999, "accuracy_std_dev": 0.016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03961820454545454, "avg_duration_per_task_ms": 540887.0883838384, "avg_output_tokens_per_task": 13782.598484848484, "last_run_timestamp": "2026-08-08T09:30:55.143Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7922558, "accuracy_std_dev": 0.03991858831121162, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04258375404040404, "avg_duration_per_task_ms": 463527.07676767674, "avg_output_tokens_per_task": 42373.13080808081, "last_run_timestamp": "2026-09-26T16:31:04.519Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7921716999999999, "accuracy_std_dev": 0.051077414533333594, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.17702268080808084, "avg_duration_per_task_ms": 826914.541919192, "avg_output_tokens_per_task": 57978.44292929293, "last_run_timestamp": "2026-09-26T09:47:43.340Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7918424506329113, "accuracy_std_dev": 0.007923520859160772, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.18873121181434602, "avg_duration_per_task_ms": 1008777.9578059071, "avg_output_tokens_per_task": 47257.703797468355, "last_run_timestamp": "2026-08-29T20:08:40.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7915958188529246, "accuracy_std_dev": 0.016282913854712845, "run_count": 10, "total_questions": 1761, "avg_cost_per_task_usd": 0.004947599091425326, "avg_duration_per_task_ms": 1178125.1942078364, "avg_output_tokens_per_task": 32554.07495741056, "last_run_timestamp": "2026-09-26T21:21:20.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.7908496524064171, "accuracy_std_dev": 0.011274463999331494, "run_count": 6, "total_questions": 1122, "avg_cost_per_task_usd": 0.013792627450980394, "avg_duration_per_task_ms": 448206.95098039217, "avg_output_tokens_per_task": 14371.752228163992, "last_run_timestamp": "2026-08-15T14:56:55.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.7908466547406083, "accuracy_std_dev": 0.0326514777957277, "run_count": 7, "total_questions": 1118, "avg_cost_per_task_usd": 0.0524859221824687, "avg_duration_per_task_ms": 809286.446332737, "avg_output_tokens_per_task": 22749.67352415027, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7907120376940132, "accuracy_std_dev": 0.04068089691283433, "run_count": 7, "total_questions": 1353, "avg_cost_per_task_usd": 0.1386707028824834, "avg_duration_per_task_ms": 1542626.872135994, "avg_output_tokens_per_task": 46143.74427198817, "last_run_timestamp": "2026-08-29T18:08:11.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.790404, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0189197702020202, "avg_duration_per_task_ms": 459328.30303030304, "avg_output_tokens_per_task": 25138.35606060606, "last_run_timestamp": "2026-08-29T19:17:42.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7902669767441859, "accuracy_std_dev": 0.0482520453161055, "run_count": 8, "total_questions": 1548, "avg_cost_per_task_usd": 0.09614282041343669, "avg_duration_per_task_ms": 503336.3320413437, "avg_output_tokens_per_task": 28837.058785529716, "last_run_timestamp": "2026-08-29T18:54:50.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7885914761215629, "accuracy_std_dev": 0.02234077768730239, "run_count": 7, "total_questions": 1382, "avg_cost_per_task_usd": 0.06106606946454414, "avg_duration_per_task_ms": 179070.87192474675, "avg_output_tokens_per_task": 19226.02532561505, "last_run_timestamp": "2026-09-26T09:36:55.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7884858931297708, "accuracy_std_dev": 0.009706627796897349, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.01825112468193384, "avg_duration_per_task_ms": 147479.76017811705, "avg_output_tokens_per_task": 15055.673664122138, "last_run_timestamp": "2026-09-26T09:15:51.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7884401666666667, "accuracy_std_dev": 0.023761946338748348, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.04315188552188553, "avg_duration_per_task_ms": 84212.74747474748, "avg_output_tokens_per_task": 26885.046296296296, "last_run_timestamp": "2026-09-01T17:55:54.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.7878980417457305, "accuracy_std_dev": 0.013974852474623847, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.05333916508538899, "avg_duration_per_task_ms": 933037.6869070209, "avg_output_tokens_per_task": 24223.848829854524, "last_run_timestamp": "2026-08-29T19:47:52.368Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7877272418181819, "accuracy_std_dev": 0.05234185111680073, "run_count": 6, "total_questions": 1100, "avg_cost_per_task_usd": 0.007575785454545454, "avg_duration_per_task_ms": 581307.9736363636, "avg_output_tokens_per_task": 17149.295454545456, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7875873916083916, "accuracy_std_dev": 0.10155079262062182, "run_count": 9, "total_questions": 1716, "avg_cost_per_task_usd": 0.08341448135198136, "avg_duration_per_task_ms": 882544.3578088578, "avg_output_tokens_per_task": 30968.915501165502, "last_run_timestamp": "2026-09-26T10:34:37.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.7867566666666665, "accuracy_std_dev": 0.004199386807089298, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.021028161616161615, "avg_duration_per_task_ms": 321362.4663299663, "avg_output_tokens_per_task": 12498.824915824916, "last_run_timestamp": "2026-07-18T16:16:42.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7848129301310043, "accuracy_std_dev": 0.07293711306212178, "run_count": 7, "total_questions": 1374, "avg_cost_per_task_usd": 0.19101991703056767, "avg_duration_per_task_ms": 1250087.5604075692, "avg_output_tokens_per_task": 61492.3442503639, "last_run_timestamp": "2026-08-29T18:49:03.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7847221249999999, "accuracy_std_dev": 0.15720335838129978, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.017239123106060607, "avg_duration_per_task_ms": 568625.6464646464, "avg_output_tokens_per_task": 27456.18876262626, "last_run_timestamp": "2026-09-26T08:08:15.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.784512, "accuracy_std_dev": 0.043771000000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.030820969696969697, "avg_duration_per_task_ms": 734052.446969697, "avg_output_tokens_per_task": 25681.72474747475, "last_run_timestamp": "2026-07-18T09:15:42.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.7841148783269962, "accuracy_std_dev": 0.01731134085469697, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.016873125475285175, "avg_duration_per_task_ms": 173156.85044359948, "avg_output_tokens_per_task": 6557.945500633714, "last_run_timestamp": "2026-08-01T17:44:11.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7839965421750663, "accuracy_std_dev": 0.016550632751952416, "run_count": 10, "total_questions": 1885, "avg_cost_per_task_usd": 0.008975832360742704, "avg_duration_per_task_ms": 426995.76127320953, "avg_output_tokens_per_task": 35628.419628647214, "last_run_timestamp": "2026-09-26T17:33:23.516Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.7832240326797386, "accuracy_std_dev": 0.07801749999999996, "run_count": 2, "total_questions": 153, "avg_cost_per_task_usd": 0.07358333333333333, "avg_duration_per_task_ms": 500220.6274509804, "avg_output_tokens_per_task": 27733.385620915033, "last_run_timestamp": "2026-08-15T15:24:07.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.7830773446153846, "accuracy_std_dev": 0.015297999999999978, "run_count": 2, "total_questions": 325, "avg_cost_per_task_usd": 0.07515540923076923, "avg_duration_per_task_ms": 234345.21538461538, "avg_output_tokens_per_task": 20795.20923076923, "last_run_timestamp": "2026-07-18T08:37:00.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "WandB", "accuracy": 0.7826841480519481, "accuracy_std_dev": 0.051647, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.02775078961038961, "avg_duration_per_task_ms": 131229.6181818182, "avg_output_tokens_per_task": 22060.571428571428, "last_run_timestamp": "2026-07-18T16:00:08.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.7824072499999999, "accuracy_std_dev": 0.0034452088742919643, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.04613476767676768, "avg_duration_per_task_ms": 199616.12626262626, "avg_output_tokens_per_task": 21878.295454545456, "last_run_timestamp": "2026-09-01T19:10:24.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.782216, "accuracy_std_dev": 0.021976765048568925, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0050231294765840226, "avg_duration_per_task_ms": 272954.46831955924, "avg_output_tokens_per_task": 32942.98806244261, "last_run_timestamp": "2026-09-26T16:14:01.905Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.7819865, "accuracy_std_dev": 0.016169511173295674, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.252381888047138, "avg_duration_per_task_ms": 227878.92592592593, "avg_output_tokens_per_task": 25272.329124579126, "last_run_timestamp": "2026-09-01T13:35:35.064Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.7808641666666666, "accuracy_std_dev": 0.010351374971739527, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.021081101010101008, "avg_duration_per_task_ms": 286187.4873737374, "avg_output_tokens_per_task": 12540.748316498317, "last_run_timestamp": "2026-08-30T00:44:42.602Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.7805236273764259, "accuracy_std_dev": 0.04118353988226817, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.013252240811153358, "avg_duration_per_task_ms": 364659.76552598225, "avg_output_tokens_per_task": 13873.679974651457, "last_run_timestamp": "2026-08-29T16:52:23.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7804641392649903, "accuracy_std_dev": 0.04111153382981473, "run_count": 5, "total_questions": 517, "avg_cost_per_task_usd": 0.012311303675048357, "avg_duration_per_task_ms": 53239.41392649903, "avg_output_tokens_per_task": 16349.947775628627, "last_run_timestamp": "2026-08-29T19:07:43.372Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.7796308899266178, "accuracy_std_dev": 0.043835688799196484, "run_count": 8, "total_questions": 1499, "avg_cost_per_task_usd": 0.04210462508338893, "avg_duration_per_task_ms": 691803.9633088725, "avg_output_tokens_per_task": 19458.18612408272, "last_run_timestamp": "2026-08-29T16:22:26.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.7794614999999999, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013140674242424242, "avg_duration_per_task_ms": 69975.75757575757, "avg_output_tokens_per_task": 10390.04797979798, "last_run_timestamp": "2026-09-01T17:51:03.252Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.7792592914285713, "accuracy_std_dev": 0.02420591387160119, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.05840109523809524, "avg_duration_per_task_ms": 642342.9288888889, "avg_output_tokens_per_task": 25397.8926984127, "last_run_timestamp": "2026-08-29T16:48:15.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7787957975567189, "accuracy_std_dev": 0.01661363892243051, "run_count": 7, "total_questions": 1146, "avg_cost_per_task_usd": 0.054749945898778365, "avg_duration_per_task_ms": 994167.5584642234, "avg_output_tokens_per_task": 28666.913612565444, "last_run_timestamp": "2026-08-29T18:31:11.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7781144000000001, "accuracy_std_dev": 0.02296971973359712, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.20166651515151518, "avg_duration_per_task_ms": 657386.6868686868, "avg_output_tokens_per_task": 48003.93838383839, "last_run_timestamp": "2026-08-29T18:10:32.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7777777692307692, "accuracy_std_dev": 0.019987758265998724, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.015498306410256409, "avg_duration_per_task_ms": 422600.68076923076, "avg_output_tokens_per_task": 15705.085256410257, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.7752525, "accuracy_std_dev": 0.021487984431227306, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02151301430976431, "avg_duration_per_task_ms": 770288.1026936027, "avg_output_tokens_per_task": 37496.628787878784, "last_run_timestamp": "2026-08-29T18:36:10.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7744108571428571, "accuracy_std_dev": 0.04740344394188064, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.17528118686868685, "avg_duration_per_task_ms": 1415587.722943723, "avg_output_tokens_per_task": 45015.56204906205, "last_run_timestamp": "2026-08-29T19:12:26.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.774186536121673, "accuracy_std_dev": 0.045556886694000515, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.0076262953105196455, "avg_duration_per_task_ms": 513514.7541191382, "avg_output_tokens_per_task": 15020.147021546261, "last_run_timestamp": "2026-09-26T08:55:07.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.7741623318872016, "accuracy_std_dev": 0.020647498917569188, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.05963888141720896, "avg_duration_per_task_ms": 613056.7744034707, "avg_output_tokens_per_task": 27371.603036876357, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7722750914285714, "accuracy_std_dev": 0.032206594225552256, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.11735591428571429, "avg_duration_per_task_ms": 772416.0933333334, "avg_output_tokens_per_task": 45815.12253968254, "last_run_timestamp": "2026-08-29T19:11:37.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.7722621887655128, "accuracy_std_dev": 0.13097706644765858, "run_count": 8, "total_questions": 1531, "avg_cost_per_task_usd": 0.02291441149575441, "avg_duration_per_task_ms": 346959.66819072503, "avg_output_tokens_per_task": 23385.622468974525, "last_run_timestamp": "2026-08-29T20:00:43.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7722220999999999, "accuracy_std_dev": 0.025764007034815056, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.005924544949494949, "avg_duration_per_task_ms": 705308.8757575757, "avg_output_tokens_per_task": 37071.080808080806, "last_run_timestamp": "2026-09-26T19:06:56.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7718254330357142, "accuracy_std_dev": 0.0208914466356596, "run_count": 7, "total_questions": 1344, "avg_cost_per_task_usd": 0.016976154761904765, "avg_duration_per_task_ms": 826873.3988095238, "avg_output_tokens_per_task": 42408.13392857143, "last_run_timestamp": "2026-08-29T16:45:34.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7714368913043479, "accuracy_std_dev": 0.014261246953428234, "run_count": 3, "total_questions": 552, "avg_cost_per_task_usd": 0.23206160869565215, "avg_duration_per_task_ms": 164099.60869565216, "avg_output_tokens_per_task": 15307.03079710145, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7703205704697986, "accuracy_std_dev": 0.02824210987088577, "run_count": 7, "total_questions": 1341, "avg_cost_per_task_usd": 0.016340362416107385, "avg_duration_per_task_ms": 580033.3325876212, "avg_output_tokens_per_task": 40993.44817300522, "last_run_timestamp": "2026-08-29T16:43:11.936Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Phala", "accuracy": 0.7688491071428571, "accuracy_std_dev": 0.036143499999999995, "run_count": 2, "total_questions": 168, "avg_cost_per_task_usd": 0.013457690476190477, "avg_duration_per_task_ms": 239500.45238095237, "avg_output_tokens_per_task": 24312.886904761905, "last_run_timestamp": "2026-07-18T09:01:08.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7687991666666666, "accuracy_std_dev": 0.023722281329421564, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.017075120370370372, "avg_duration_per_task_ms": 891145.8181818182, "avg_output_tokens_per_task": 43562.35101010101, "last_run_timestamp": "2026-08-29T18:49:24.022Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.7686387142857142, "accuracy_std_dev": 0.039191927622605004, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.05355179797979799, "avg_duration_per_task_ms": 647628.132034632, "avg_output_tokens_per_task": 24095.015151515152, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.7685185, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013512386363636364, "avg_duration_per_task_ms": 71792.87121212122, "avg_output_tokens_per_task": 10688.070707070707, "last_run_timestamp": "2026-09-01T17:55:24.769Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7676545740944017, "accuracy_std_dev": 0.04018594976550331, "run_count": 6, "total_questions": 911, "avg_cost_per_task_usd": 0.14126989352360045, "avg_duration_per_task_ms": 1023387.3852908892, "avg_output_tokens_per_task": 40525.553238199784, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.7672560000000002, "accuracy_std_dev": 0.0089379450938121, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013739436868686868, "avg_duration_per_task_ms": 117371.45833333333, "avg_output_tokens_per_task": 10352.156565656565, "last_run_timestamp": "2026-09-01T17:58:15.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7672559999999999, "accuracy_std_dev": 0.008469894233892938, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.27037940151515155, "avg_duration_per_task_ms": 263755.6675084175, "avg_output_tokens_per_task": 17837.74579124579, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7671957142857142, "accuracy_std_dev": 0.009679973770836427, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.017645118326118325, "avg_duration_per_task_ms": 105010.88528138529, "avg_output_tokens_per_task": 6033.81746031746, "last_run_timestamp": "2026-09-26T09:17:54.791Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.01935999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2670539015151515, "avg_duration_per_task_ms": 177095.6616161616, "avg_output_tokens_per_task": 17616.689393939392, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05139547999999999, "avg_duration_per_task_ms": 172458.62, "avg_output_tokens_per_task": 21216.74, "last_run_timestamp": "2026-09-14T19:42:46.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.76638602247191, "accuracy_std_dev": 0.0119351260850762, "run_count": 3, "total_questions": 356, "avg_cost_per_task_usd": 0.002961258426966292, "avg_duration_per_task_ms": 69864.89325842696, "avg_output_tokens_per_task": 3595.182584269663, "last_run_timestamp": "2026-08-15T14:54:48.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.766309, "accuracy_std_dev": 0.007116882358167777, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.014960216540404043, "avg_duration_per_task_ms": 157155.3813131313, "avg_output_tokens_per_task": 29913.797348484848, "last_run_timestamp": "2026-08-29T18:39:41.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.765993, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08016306060606061, "avg_duration_per_task_ms": 425226.22222222225, "avg_output_tokens_per_task": 69822.58585858585, "last_run_timestamp": "2026-07-18T08:29:23.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.7659360351925629, "accuracy_std_dev": 0.01122486854878489, "run_count": 8, "total_questions": 1506, "avg_cost_per_task_usd": 0.024484757636122175, "avg_duration_per_task_ms": 73000.31474103585, "avg_output_tokens_per_task": 25656.97609561753, "last_run_timestamp": "2026-08-29T19:51:33.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.7653383396584439, "accuracy_std_dev": 0.03752103948599206, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.01665043263757116, "avg_duration_per_task_ms": 488848.29917773564, "avg_output_tokens_per_task": 16454.457305502845, "last_run_timestamp": "2026-08-30T00:18:59.365Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7647061294117647, "accuracy_std_dev": 0.11983655888350603, "run_count": 5, "total_questions": 680, "avg_cost_per_task_usd": 0.015231964705882355, "avg_duration_per_task_ms": 558302.2632352941, "avg_output_tokens_per_task": 21406.642647058823, "last_run_timestamp": "2026-09-26T10:43:59.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.7644630358126723, "accuracy_std_dev": 0.017799283895356573, "run_count": 4, "total_questions": 726, "avg_cost_per_task_usd": 0.019927535812672176, "avg_duration_per_task_ms": 237713.65702479339, "avg_output_tokens_per_task": 26486.89256198347, "last_run_timestamp": "2026-08-29T18:11:51.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7643024133099825, "accuracy_std_dev": 0.031491855215101075, "run_count": 6, "total_questions": 1142, "avg_cost_per_task_usd": 0.04750308056042031, "avg_duration_per_task_ms": 265287.42907180387, "avg_output_tokens_per_task": 17893.702276707532, "last_run_timestamp": "2026-09-26T09:16:55.239Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.7611532500000001, "accuracy_std_dev": 0.007114550894996836, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.017736026515151517, "avg_duration_per_task_ms": 86462.99116161616, "avg_output_tokens_per_task": 29343.33207070707, "last_run_timestamp": "2026-08-29T18:36:51.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.760943, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08179280303030302, "avg_duration_per_task_ms": 426340.89898989897, "avg_output_tokens_per_task": 70994.31818181818, "last_run_timestamp": "2026-07-18T08:26:55.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.760674964876033, "accuracy_std_dev": 0.01645419404771925, "run_count": 5, "total_questions": 968, "avg_cost_per_task_usd": 0.010708528925619836, "avg_duration_per_task_ms": 425619.78925619833, "avg_output_tokens_per_task": 25673.311983471074, "last_run_timestamp": "2026-08-29T18:56:40.728Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.7606652169197398, "accuracy_std_dev": 0.04591514075032818, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.004993729573391178, "avg_duration_per_task_ms": 317882.5820679682, "avg_output_tokens_per_task": 12242.755603759942, "last_run_timestamp": "2026-08-29T18:46:19.350Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.7605443265306122, "accuracy_std_dev": 0.0343317834601635, "run_count": 3, "total_questions": 490, "avg_cost_per_task_usd": 0.0037374408163265305, "avg_duration_per_task_ms": 668153.0571428571, "avg_output_tokens_per_task": 14206.583673469388, "last_run_timestamp": "2026-08-08T11:42:26.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.75974, "accuracy_std_dev": 0.052236688581877, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0108920873015873, "avg_duration_per_task_ms": 734819.8246753247, "avg_output_tokens_per_task": 32324.756132756134, "last_run_timestamp": "2026-09-12T09:07:23.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.759259, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018210444444444442, "avg_duration_per_task_ms": 254627.46717171717, "avg_output_tokens_per_task": 24079.401515151516, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7590111668383109, "accuracy_std_dev": 0.016611473377157127, "run_count": 5, "total_questions": 971, "avg_cost_per_task_usd": 0.004071515962924819, "avg_duration_per_task_ms": 4054.4541709577757, "avg_output_tokens_per_task": 2252.576725025747, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.758207125, "accuracy_std_dev": 0.00985024896179661, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01902252525252525, "avg_duration_per_task_ms": 176301.23106060605, "avg_output_tokens_per_task": 31511.469696969696, "last_run_timestamp": "2026-08-29T20:01:12.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.7572815631067961, "accuracy_std_dev": 0.10286820253600068, "run_count": 9, "total_questions": 721, "avg_cost_per_task_usd": 0.002934966712898752, "avg_duration_per_task_ms": 194322.5145631068, "avg_output_tokens_per_task": 13374.00693481276, "last_run_timestamp": "2026-09-26T16:00:24.612Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.7565450000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.020916916230366494, "avg_duration_per_task_ms": 124793.11518324607, "avg_output_tokens_per_task": 18115.47120418848, "last_run_timestamp": "2026-07-18T08:26:49.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7559323627118644, "accuracy_std_dev": 0.007798453236949536, "run_count": 3, "total_questions": 590, "avg_cost_per_task_usd": 0.002668813559322034, "avg_duration_per_task_ms": 91653.82203389831, "avg_output_tokens_per_task": 31189.77966101695, "last_run_timestamp": "2026-09-01T13:23:13.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7558922, "accuracy_std_dev": 0.10699962710663996, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13022594646464647, "avg_duration_per_task_ms": 1023351.8464646465, "avg_output_tokens_per_task": 39743.50404040404, "last_run_timestamp": "2026-08-22T17:17:31.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7558506982922201, "accuracy_std_dev": 0.013474818336988598, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.018912632194813407, "avg_duration_per_task_ms": 195928.35926628715, "avg_output_tokens_per_task": 31299.83870967742, "last_run_timestamp": "2026-08-29T16:18:21.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.7551707142857145, "accuracy_std_dev": 0.008190904148841594, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02236280663780664, "avg_duration_per_task_ms": 98622.52741702742, "avg_output_tokens_per_task": 14763.670274170274, "last_run_timestamp": "2026-08-29T19:17:49.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7548771259541985, "accuracy_std_dev": 0.012936525817791087, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.016541959287531806, "avg_duration_per_task_ms": 307024.9662849873, "avg_output_tokens_per_task": 28293.188295165393, "last_run_timestamp": "2026-08-29T18:30:03.855Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.754696375, "accuracy_std_dev": 0.022698605518617125, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.020891089595375723, "avg_duration_per_task_ms": 144594.61271676302, "avg_output_tokens_per_task": 12072.543352601157, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7545347341659232, "accuracy_std_dev": 0.02761225844139473, "run_count": 8, "total_questions": 1121, "avg_cost_per_task_usd": 0.05582876717216771, "avg_duration_per_task_ms": 404528.79750223015, "avg_output_tokens_per_task": 16141.22033898305, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.754260582822086, "accuracy_std_dev": 0.0330868929772635, "run_count": 3, "total_questions": 489, "avg_cost_per_task_usd": 0.010235226993865029, "avg_duration_per_task_ms": 357931.47034764825, "avg_output_tokens_per_task": 31124.41717791411, "last_run_timestamp": "2026-07-18T08:04:08.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.752765857142857, "accuracy_std_dev": 0.045870754858246085, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.05275206060606061, "avg_duration_per_task_ms": 979408.9415584416, "avg_output_tokens_per_task": 27425.04473304473, "last_run_timestamp": "2026-08-29T18:19:18.341Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7525250000000001, "accuracy_std_dev": 0.010101000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.056976058080808085, "avg_duration_per_task_ms": 98919.65151515152, "avg_output_tokens_per_task": 22701.11111111111, "last_run_timestamp": "2026-09-01T17:54:55.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.751473, "accuracy_std_dev": 0.14287927657379149, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03140934785353535, "avg_duration_per_task_ms": 145498.60984848486, "avg_output_tokens_per_task": 24064.433712121212, "last_run_timestamp": "2026-08-29T19:13:10.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.750842, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01581958585858586, "avg_duration_per_task_ms": 326624.7373737374, "avg_output_tokens_per_task": 13062.823232323231, "last_run_timestamp": "2026-09-12T09:42:09.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.75, "accuracy_std_dev": null, "run_count": 1, "total_questions": 150, "avg_cost_per_task_usd": 0.0067602800000000005, "avg_duration_per_task_ms": 1824454.58, "avg_output_tokens_per_task": 23925.293333333335, "last_run_timestamp": "2026-09-22T21:48:57.116Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7493372537960954, "accuracy_std_dev": 0.017786308528225433, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.16125216847433116, "avg_duration_per_task_ms": 602672.0441070137, "avg_output_tokens_per_task": 50469.859725234994, "last_run_timestamp": "2026-08-29T18:15:24.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7485970000000001, "accuracy_std_dev": 0.07972918647312704, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006700525252525253, "avg_duration_per_task_ms": 93845.37542087543, "avg_output_tokens_per_task": 16482.319865319867, "last_run_timestamp": "2026-08-29T18:45:38.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.748106, "accuracy_std_dev": 0.006730599731821838, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.013436075126262624, "avg_duration_per_task_ms": 99232.33333333333, "avg_output_tokens_per_task": 14767.26452020202, "last_run_timestamp": "2026-08-29T18:17:28.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.7476619999999999, "accuracy_std_dev": 0.008415500381240979, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.014419837822671156, "avg_duration_per_task_ms": 105834.91358024691, "avg_output_tokens_per_task": 14297.092031425365, "last_run_timestamp": "2026-08-29T18:11:58.511Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7475968772093022, "accuracy_std_dev": 0.030626747173151808, "run_count": 7, "total_questions": 1075, "avg_cost_per_task_usd": 0.006931252093023256, "avg_duration_per_task_ms": 969560.9004651163, "avg_output_tokens_per_task": 17890.426046511628, "last_run_timestamp": "2026-08-29T19:19:08.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.7474749999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08105034848484848, "avg_duration_per_task_ms": 422601, "avg_output_tokens_per_task": 70367.68181818182, "last_run_timestamp": "2026-07-18T08:27:39.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7468043761061948, "accuracy_std_dev": 0.051672782249735434, "run_count": 7, "total_questions": 1356, "avg_cost_per_task_usd": 0.001709050884955752, "avg_duration_per_task_ms": 588006.3584070796, "avg_output_tokens_per_task": 11440.941002949852, "last_run_timestamp": "2026-08-22T17:47:50.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7465941264367815, "accuracy_std_dev": 0.10532480903970348, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.04816395977011494, "avg_duration_per_task_ms": 468882.4540229885, "avg_output_tokens_per_task": 36470.09642401022, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7463394360465117, "accuracy_std_dev": 0.07095671544505143, "run_count": 8, "total_questions": 1548, "avg_cost_per_task_usd": 0.12446177519379846, "avg_duration_per_task_ms": 998515.9916020671, "avg_output_tokens_per_task": 55402.337855297155, "last_run_timestamp": "2026-08-29T18:23:09.250Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.7462122499999999, "accuracy_std_dev": 0.013006497229750218, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.017694700757575756, "avg_duration_per_task_ms": 99445.50378787878, "avg_output_tokens_per_task": 6284.411616161616, "last_run_timestamp": "2026-08-01T17:45:27.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7462122499999999, "accuracy_std_dev": 0.013222542538691302, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0095508547979798, "avg_duration_per_task_ms": 202963.16287878787, "avg_output_tokens_per_task": 8137.35101010101, "last_run_timestamp": "2026-08-29T18:16:24.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.7455108333333333, "accuracy_std_dev": 0.020304992673532824, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.08322730471380473, "avg_duration_per_task_ms": 907303.9099326599, "avg_output_tokens_per_task": 23781.781144781144, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.7449942438162545, "accuracy_std_dev": 0.01819967836712137, "run_count": 3, "total_questions": 283, "avg_cost_per_task_usd": 0.01679327561837456, "avg_duration_per_task_ms": 97120.16254416961, "avg_output_tokens_per_task": 9655.678445229682, "last_run_timestamp": "2026-08-30T00:01:09.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.7431723333333333, "accuracy_std_dev": 0.011056875789400136, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0071705791245791255, "avg_duration_per_task_ms": 175275.17003367003, "avg_output_tokens_per_task": 8334.754208754208, "last_run_timestamp": "2026-08-29T18:27:42.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.742845125, "accuracy_std_dev": 0.010902042072904301, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.022606026515151516, "avg_duration_per_task_ms": 99614.75883838384, "avg_output_tokens_per_task": 14962.358585858587, "last_run_timestamp": "2026-08-29T18:28:10.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.7426647142857141, "accuracy_std_dev": 0.007278926328092368, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0053309747474747475, "avg_duration_per_task_ms": 143246.7821067821, "avg_output_tokens_per_task": 8614.1443001443, "last_run_timestamp": "2026-08-29T18:49:21.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7423279333333332, "accuracy_std_dev": 0.047881077561854994, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.007047760000000001, "avg_duration_per_task_ms": 1019395.5453968254, "avg_output_tokens_per_task": 18421.787936507935, "last_run_timestamp": "2026-08-29T20:11:32.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Groq", "accuracy": 0.7418629999999999, "accuracy_std_dev": 0.004827531943619518, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01876800505050505, "avg_duration_per_task_ms": 32883.444444444445, "avg_output_tokens_per_task": 6023.489898989899, "last_run_timestamp": "2026-08-01T17:45:49.025Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.7413018888888889, "accuracy_std_dev": 0.010764979449270195, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.004904684624017957, "avg_duration_per_task_ms": 109945.37485970819, "avg_output_tokens_per_task": 7999.2755331088665, "last_run_timestamp": "2026-08-29T18:15:48.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.7402804187380497, "accuracy_std_dev": 0.076460867937462, "run_count": 8, "total_questions": 1569, "avg_cost_per_task_usd": 0.010684235181644361, "avg_duration_per_task_ms": 919191.3709369025, "avg_output_tokens_per_task": 11540.983428935628, "last_run_timestamp": "2026-08-30T01:20:44.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.7398376524390244, "accuracy_std_dev": 0.030475562908008763, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.004495964430894309, "avg_duration_per_task_ms": 781618.3089430894, "avg_output_tokens_per_task": 27377.12906504065, "last_run_timestamp": "2026-08-29T18:16:54.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7392771671732522, "accuracy_std_dev": 0.07975374971554629, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.005161332320162108, "avg_duration_per_task_ms": 757299.1337386018, "avg_output_tokens_per_task": 34332.25075987842, "last_run_timestamp": "2026-09-26T18:04:35.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7388467499999999, "accuracy_std_dev": 0.07747406520692911, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01745328409090909, "avg_duration_per_task_ms": 2275674.0732323234, "avg_output_tokens_per_task": 17131.02398989899, "last_run_timestamp": "2026-08-22T18:24:57.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.7384959999999999, "accuracy_std_dev": 0.0076941959727230055, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.058646202861952865, "avg_duration_per_task_ms": 123198.74579124579, "avg_output_tokens_per_task": 23489.968855218856, "last_run_timestamp": "2026-09-01T17:53:51.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.7380156260869567, "accuracy_std_dev": 0.012608807218939267, "run_count": 8, "total_questions": 1495, "avg_cost_per_task_usd": 0.008862770568561874, "avg_duration_per_task_ms": 132109.78528428092, "avg_output_tokens_per_task": 11237.499665551839, "last_run_timestamp": "2026-08-29T18:11:53.163Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7374254884910486, "accuracy_std_dev": 0.06440394854096035, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.14750449019607842, "avg_duration_per_task_ms": 1116895.3742540495, "avg_output_tokens_per_task": 40733.769820971866, "last_run_timestamp": "2026-08-22T17:24:19.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.7373736666666667, "accuracy_std_dev": 0.017869435923448234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004179791245791245, "avg_duration_per_task_ms": 604833.3282828283, "avg_output_tokens_per_task": 30919.042087542086, "last_run_timestamp": "2026-07-18T16:28:34.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.736111, "accuracy_std_dev": null, "run_count": 1, "total_questions": 168, "avg_cost_per_task_usd": 0.05987852380952381, "avg_duration_per_task_ms": 1235915.3511904762, "avg_output_tokens_per_task": 62102.07738095238, "last_run_timestamp": "2026-07-18T08:29:05.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Together", "accuracy": 0.73569, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005349656565656565, "avg_duration_per_task_ms": 222485.2626262626, "avg_output_tokens_per_task": 8728.030303030304, "last_run_timestamp": "2026-07-04T17:56:44.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7355811297709923, "accuracy_std_dev": 0.10861849313307564, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.00556993893129771, "avg_duration_per_task_ms": 485638.58396946563, "avg_output_tokens_per_task": 24035.00954198473, "last_run_timestamp": "2026-09-26T09:37:20.718Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.7351291666666667, "accuracy_std_dev": 0.05591704578336457, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015923739057239056, "avg_duration_per_task_ms": 844758.271885522, "avg_output_tokens_per_task": 41206.31481481482, "last_run_timestamp": "2026-08-29T19:32:52.576Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.7336406894705532, "accuracy_std_dev": 0.016215018082375493, "run_count": 17, "total_questions": 3362, "avg_cost_per_task_usd": 0.0069099631171921465, "avg_duration_per_task_ms": 93516.04104699583, "avg_output_tokens_per_task": 7210.586555621654, "last_run_timestamp": "2026-08-29T19:15:44.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7314815, "accuracy_std_dev": 0.11424146120352568, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.13496367760942762, "avg_duration_per_task_ms": 1650116.6127946128, "avg_output_tokens_per_task": 45746.51094276094, "last_run_timestamp": "2026-08-29T18:41:58.319Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7312871538461537, "accuracy_std_dev": 0.012238898554301565, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.20442489957264962, "avg_duration_per_task_ms": 242818.47999223, "avg_output_tokens_per_task": 38189.194832944835, "last_run_timestamp": "2026-09-26T08:58:38.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.7309068406130269, "accuracy_std_dev": 0.06991914710924596, "run_count": 8, "total_questions": 1305, "avg_cost_per_task_usd": 0.03057965057471263, "avg_duration_per_task_ms": 649469.8950191571, "avg_output_tokens_per_task": 33236.169348659, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.7307611537001898, "accuracy_std_dev": 0.03501680507605419, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.009550828589500316, "avg_duration_per_task_ms": 215086.75015812778, "avg_output_tokens_per_task": 18410.464263124606, "last_run_timestamp": "2026-08-29T19:17:18.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.7307607964071856, "accuracy_std_dev": 0.09245964826175526, "run_count": 8, "total_questions": 1503, "avg_cost_per_task_usd": 0.14931497338656022, "avg_duration_per_task_ms": 776563.6879574185, "avg_output_tokens_per_task": 54559.805721889556, "last_run_timestamp": "2026-08-29T16:33:36.680Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.7300085000000001, "accuracy_std_dev": 0.01549688586458583, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005400070075757575, "avg_duration_per_task_ms": 260007.1976010101, "avg_output_tokens_per_task": 8023.359217171717, "last_run_timestamp": "2026-08-29T18:12:05.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.7298248631578947, "accuracy_std_dev": 0.025121594911328576, "run_count": 3, "total_questions": 570, "avg_cost_per_task_usd": 0.004760936842105263, "avg_duration_per_task_ms": 667212.4614035088, "avg_output_tokens_per_task": 27657.705263157895, "last_run_timestamp": "2026-07-18T16:27:55.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.7297043333333333, "accuracy_std_dev": 0.017207018593843885, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1851553911335578, "avg_duration_per_task_ms": 289835.51515151514, "avg_output_tokens_per_task": 36869.942199775534, "last_run_timestamp": "2026-09-26T09:21:00.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.729592, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.0030441326530612246, "avg_duration_per_task_ms": 200954.75, "avg_output_tokens_per_task": 31361.45408163265, "last_run_timestamp": "2026-09-01T13:20:55.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Phala", "accuracy": 0.7290074809160306, "accuracy_std_dev": 0.07204749999999999, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.12394920356234096, "avg_duration_per_task_ms": 1193060.445292621, "avg_output_tokens_per_task": 40638.114503816796, "last_run_timestamp": "2026-07-11T16:02:27.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.7283226758169934, "accuracy_std_dev": 0.19151515448196257, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.019633686274509805, "avg_duration_per_task_ms": 252454.9568627451, "avg_output_tokens_per_task": 16304.679738562092, "last_run_timestamp": "2026-08-01T18:21:26.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7274615892439593, "accuracy_std_dev": 0.08374107689388129, "run_count": 7, "total_questions": 1283, "avg_cost_per_task_usd": 0.030927738893219012, "avg_duration_per_task_ms": 152143.38581449728, "avg_output_tokens_per_task": 28647.951675759938, "last_run_timestamp": "2026-08-29T19:16:48.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.727459888888889, "accuracy_std_dev": 0.010021023926885385, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1903540600448934, "avg_duration_per_task_ms": 241489.60381593715, "avg_output_tokens_per_task": 37880.08529741863, "last_run_timestamp": "2026-09-26T08:54:32.344Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7272725, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.058774820707070706, "avg_duration_per_task_ms": 121424.5, "avg_output_tokens_per_task": 23508.14898989899, "last_run_timestamp": "2026-09-01T17:49:53.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.7269571249999999, "accuracy_std_dev": 0.006573317074306892, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0035786773989898986, "avg_duration_per_task_ms": 533271.5606060605, "avg_output_tokens_per_task": 8145.644570707071, "last_run_timestamp": "2026-08-29T18:39:40.767Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.7259116944971536, "accuracy_std_dev": 0.014910946473459525, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.00463467109424415, "avg_duration_per_task_ms": 214923.00189753322, "avg_output_tokens_per_task": 7865.01201771031, "last_run_timestamp": "2026-08-29T18:16:39.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7245957898383372, "accuracy_std_dev": 0.009250125041530539, "run_count": 9, "total_questions": 1732, "avg_cost_per_task_usd": 0.1865489249422633, "avg_duration_per_task_ms": 350199.4688221709, "avg_output_tokens_per_task": 37119.776558891455, "last_run_timestamp": "2026-09-26T08:53:09.141Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.7241462857142857, "accuracy_std_dev": 0.008790615315929416, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006113787157287157, "avg_duration_per_task_ms": 404593.2886002886, "avg_output_tokens_per_task": 8122.329725829726, "last_run_timestamp": "2026-08-22T16:50:17.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.7234044255319149, "accuracy_std_dev": 0.1917994784299478, "run_count": 3, "total_questions": 517, "avg_cost_per_task_usd": 0.033565591876208896, "avg_duration_per_task_ms": 242772.43907156674, "avg_output_tokens_per_task": 37102.30560928433, "last_run_timestamp": "2026-09-29T20:57:20.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.722853625, "accuracy_std_dev": 0.04324929642473245, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006985363636363637, "avg_duration_per_task_ms": 129043.11742424243, "avg_output_tokens_per_task": 7756.648358585859, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.722222125, "accuracy_std_dev": 0.01884093724073658, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006819729797979799, "avg_duration_per_task_ms": 350972.87373737374, "avg_output_tokens_per_task": 27084.810606060608, "last_run_timestamp": "2026-08-29T18:36:32.803Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7219416666666667, "accuracy_std_dev": 0.05789083119880804, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.011145885521885522, "avg_duration_per_task_ms": 86502.09595959596, "avg_output_tokens_per_task": 4478.992424242424, "last_run_timestamp": "2026-09-26T09:45:38.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.7219408075949366, "accuracy_std_dev": 0.025598999999999983, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.27939689620253166, "avg_duration_per_task_ms": 450249.90379746835, "avg_output_tokens_per_task": 45941.00253164557, "last_run_timestamp": "2026-08-15T08:16:15.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7211966144756279, "accuracy_std_dev": 0.09962827751441757, "run_count": 7, "total_questions": 1354, "avg_cost_per_task_usd": 0.04244912850812408, "avg_duration_per_task_ms": 1668691.0731166913, "avg_output_tokens_per_task": 17565.30502215657, "last_run_timestamp": "2026-08-29T20:16:12.181Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.7204764444444446, "accuracy_std_dev": 0.010281167113216102, "run_count": 27, "total_questions": 5346, "avg_cost_per_task_usd": 0.1972407962962963, "avg_duration_per_task_ms": 261625.21979049756, "avg_output_tokens_per_task": 36783.956041900485, "last_run_timestamp": "2026-09-26T09:21:19.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.71875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 192, "avg_cost_per_task_usd": 0.014737661458333334, "avg_duration_per_task_ms": 950475.9635416666, "avg_output_tokens_per_task": 52272.296875, "last_run_timestamp": "2026-09-29T03:31:44.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7184345000000002, "accuracy_std_dev": 0.00575544305418794, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006676434343434343, "avg_duration_per_task_ms": 1198975.5896464647, "avg_output_tokens_per_task": 54074.54671717172, "last_run_timestamp": "2026-09-26T17:35:27.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7183463229974161, "accuracy_std_dev": 0.029264511720208542, "run_count": 6, "total_questions": 1161, "avg_cost_per_task_usd": 0.17074779758828595, "avg_duration_per_task_ms": 1524939.6339362618, "avg_output_tokens_per_task": 41593.74849267872, "last_run_timestamp": "2026-08-22T16:49:30.177Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7170990662824207, "accuracy_std_dev": 0.10692249999999998, "run_count": 2, "total_questions": 347, "avg_cost_per_task_usd": 0.005610605187319885, "avg_duration_per_task_ms": 269295.4322766571, "avg_output_tokens_per_task": 27309.671469740635, "last_run_timestamp": "2026-09-01T13:46:27.106Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.7147435038461539, "accuracy_std_dev": 0.013460967196281816, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.00823847564102564, "avg_duration_per_task_ms": 74470.71923076923, "avg_output_tokens_per_task": 10465.133974358974, "last_run_timestamp": "2026-08-29T18:12:18.859Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.7142555025380709, "accuracy_std_dev": 0.033662331581457615, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.0112786192893401, "avg_duration_per_task_ms": 562577.1395939087, "avg_output_tokens_per_task": 56130.946700507615, "last_run_timestamp": "2026-09-26T09:18:34.055Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.7132454451219512, "accuracy_std_dev": 0.06047841361477665, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.01395117174796748, "avg_duration_per_task_ms": 1210466.6656504066, "avg_output_tokens_per_task": 35695.419715447155, "last_run_timestamp": "2026-08-22T10:02:15.507Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.7118334872169467, "accuracy_std_dev": 0.05025937914195097, "run_count": 8, "total_questions": 1369, "avg_cost_per_task_usd": 0.00946499707815924, "avg_duration_per_task_ms": 383206.33820306795, "avg_output_tokens_per_task": 15939.81811541271, "last_run_timestamp": "2026-08-29T19:16:51.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7100695976562501, "accuracy_std_dev": 0.008928047441035206, "run_count": 8, "total_questions": 1536, "avg_cost_per_task_usd": 0.01613465299479167, "avg_duration_per_task_ms": 90625.380859375, "avg_output_tokens_per_task": 10675.574869791666, "last_run_timestamp": "2026-08-29T19:12:26.066Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.708820908625731, "accuracy_std_dev": 0.016533293775325746, "run_count": 7, "total_questions": 1368, "avg_cost_per_task_usd": 0.010723485380116957, "avg_duration_per_task_ms": 104832.341374269, "avg_output_tokens_per_task": 9909.211988304094, "last_run_timestamp": "2026-08-29T19:10:03.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.708754, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01329097474747475, "avg_duration_per_task_ms": 223430.93434343435, "avg_output_tokens_per_task": 33132.76262626263, "last_run_timestamp": "2026-09-01T18:00:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.7077671902912621, "accuracy_std_dev": 0.13908047205911944, "run_count": 4, "total_questions": 515, "avg_cost_per_task_usd": 0.03580529126213592, "avg_duration_per_task_ms": 1364969.3572815533, "avg_output_tokens_per_task": 24706.64077669903, "last_run_timestamp": "2026-08-01T23:27:37.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.7075006595744683, "accuracy_std_dev": 0.010031238556847082, "run_count": 8, "total_questions": 1551, "avg_cost_per_task_usd": 0.011043542875564152, "avg_duration_per_task_ms": 107250.29013539651, "avg_output_tokens_per_task": 10405.526756931013, "last_run_timestamp": "2026-08-29T18:42:28.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.7074916250000001, "accuracy_std_dev": 0.009879437230145022, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008883945707070707, "avg_duration_per_task_ms": 142295.2904040404, "avg_output_tokens_per_task": 5760.14898989899, "last_run_timestamp": "2026-08-29T18:10:22.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7070005146036162, "accuracy_std_dev": 0.06882289073774799, "run_count": 4, "total_questions": 719, "avg_cost_per_task_usd": 0.07411282058414463, "avg_duration_per_task_ms": 612487.7343532684, "avg_output_tokens_per_task": 29410.726008344922, "last_run_timestamp": "2026-08-15T09:29:41.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7067488941418294, "accuracy_std_dev": 0.11976383227569164, "run_count": 5, "total_questions": 973, "avg_cost_per_task_usd": 0.22542014594039053, "avg_duration_per_task_ms": 1724549.0164439876, "avg_output_tokens_per_task": 61812.13874614594, "last_run_timestamp": "2026-08-22T09:29:12.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.7059191574642126, "accuracy_std_dev": 0.05245735925848092, "run_count": 15, "total_questions": 2934, "avg_cost_per_task_usd": 0.008334460804362644, "avg_duration_per_task_ms": 468794.78834355826, "avg_output_tokens_per_task": 25898.645875937287, "last_run_timestamp": "2026-08-29T19:12:34.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.7055976250000001, "accuracy_std_dev": 0.010151381518511415, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00599510542929293, "avg_duration_per_task_ms": 107464.72411616161, "avg_output_tokens_per_task": 5562.948863636364, "last_run_timestamp": "2026-08-29T19:09:55.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.7053707181996087, "accuracy_std_dev": 0.016091084149595845, "run_count": 8, "total_questions": 1533, "avg_cost_per_task_usd": 0.01048878082191781, "avg_duration_per_task_ms": 154483.25962165688, "avg_output_tokens_per_task": 9469.20156555773, "last_run_timestamp": "2026-08-29T18:09:55.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.7050869311010947, "accuracy_std_dev": 0.036824859327421404, "run_count": 8, "total_questions": 1553, "avg_cost_per_task_usd": 0.01090031036703155, "avg_duration_per_task_ms": 296639.77012234385, "avg_output_tokens_per_task": 47139.87057308435, "last_run_timestamp": "2026-09-26T08:54:38.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.70328275, "accuracy_std_dev": 0.02153875190644758, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0019388005050505052, "avg_duration_per_task_ms": 636092.8409090909, "avg_output_tokens_per_task": 14000.35101010101, "last_run_timestamp": "2026-09-05T08:28:07.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7024411666666667, "accuracy_std_dev": 0.14218853509503576, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013173934343434343, "avg_duration_per_task_ms": 786438.8409090909, "avg_output_tokens_per_task": 27569.127104377105, "last_run_timestamp": "2026-09-22T02:20:03.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7024411249999999, "accuracy_std_dev": 0.013087863819561053, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01130629292929293, "avg_duration_per_task_ms": 202130.5946969697, "avg_output_tokens_per_task": 5888.216540404041, "last_run_timestamp": "2026-08-29T18:30:44.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.7002430000000002, "accuracy_std_dev": 0.010305162546564278, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0022877149270482604, "avg_duration_per_task_ms": 203069.73456790124, "avg_output_tokens_per_task": 5122.809203142537, "last_run_timestamp": "2026-08-29T16:33:36.891Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.6984128571428573, "accuracy_std_dev": 0.02323031510412064, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.011283265512265512, "avg_duration_per_task_ms": 64876.18759018759, "avg_output_tokens_per_task": 9342.030303030304, "last_run_timestamp": "2026-08-29T18:10:09.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.6979317142857143, "accuracy_std_dev": 0.03403895099405253, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.012199084415584417, "avg_duration_per_task_ms": 1274707.3217893217, "avg_output_tokens_per_task": 11028.597402597403, "last_run_timestamp": "2026-08-22T17:28:27.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.695926669535284, "accuracy_std_dev": 0.09652330217621027, "run_count": 6, "total_questions": 1162, "avg_cost_per_task_usd": 0.009071061101549054, "avg_duration_per_task_ms": 470826.79862306366, "avg_output_tokens_per_task": 31779.3339070568, "last_run_timestamp": "2026-08-29T20:02:35.387Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.69549675, "accuracy_std_dev": 0.017785276598566026, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008984150252525254, "avg_duration_per_task_ms": 269134.8390151515, "avg_output_tokens_per_task": 6325.36047979798, "last_run_timestamp": "2026-08-29T18:14:24.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6948820649606299, "accuracy_std_dev": 0.027729042227588305, "run_count": 8, "total_questions": 1524, "avg_cost_per_task_usd": 0.015564356299212597, "avg_duration_per_task_ms": 765563.3963254594, "avg_output_tokens_per_task": 77634.76837270342, "last_run_timestamp": "2026-09-26T16:34:21.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.6940944351371386, "accuracy_std_dev": 0.0463087281397186, "run_count": 7, "total_questions": 1349, "avg_cost_per_task_usd": 0.005615137879911044, "avg_duration_per_task_ms": 396156.75315048185, "avg_output_tokens_per_task": 15382.206078576724, "last_run_timestamp": "2026-08-29T19:25:16.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.6931105699373695, "accuracy_std_dev": 0.08497842021164353, "run_count": 6, "total_questions": 958, "avg_cost_per_task_usd": 0.0069657160751565765, "avg_duration_per_task_ms": 481855.8267223382, "avg_output_tokens_per_task": 32574.692066805845, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6917087499999999, "accuracy_std_dev": 0.2503720824541696, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.025392156565656566, "avg_duration_per_task_ms": 357863.4059343434, "avg_output_tokens_per_task": 15932.088383838383, "last_run_timestamp": "2026-08-29T19:03:14.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.6896746666666668, "accuracy_std_dev": 0.04960523518294776, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04329939057239058, "avg_duration_per_task_ms": 1345214.5555555555, "avg_output_tokens_per_task": 31899.07744107744, "last_run_timestamp": "2026-08-22T16:20:35.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DeepInfra", "accuracy": 0.6890942823218998, "accuracy_std_dev": 0.1415725, "run_count": 2, "total_questions": 379, "avg_cost_per_task_usd": 0.020441316622691292, "avg_duration_per_task_ms": 430944.6226912929, "avg_output_tokens_per_task": 17733.345646437996, "last_run_timestamp": "2026-07-11T15:51:15.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.6885525, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013761295454545455, "avg_duration_per_task_ms": 259532.94696969696, "avg_output_tokens_per_task": 32681.065656565657, "last_run_timestamp": "2026-09-01T18:15:18.821Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6863611111111112, "accuracy_std_dev": 0.022997260124245744, "run_count": 6, "total_questions": 1134, "avg_cost_per_task_usd": 0.010325405643738976, "avg_duration_per_task_ms": 361986.35802469135, "avg_output_tokens_per_task": 46144.19135802469, "last_run_timestamp": "2026-09-26T08:53:22.738Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.6857475882352941, "accuracy_std_dev": 0.043243076491900706, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.014387779886148006, "avg_duration_per_task_ms": 244155.07653383934, "avg_output_tokens_per_task": 7771.313725490196, "last_run_timestamp": "2026-08-29T19:24:48.378Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.685185, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.007298616161616161, "avg_duration_per_task_ms": 578052.9191919192, "avg_output_tokens_per_task": 17923.358585858587, "last_run_timestamp": "2026-07-18T08:44:31.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403:free", "model_slug": null, "model_name": "Gemma 4 26B A4B IT (free)", "provider_name": "Darkbloom", "accuracy": 0.684722, "accuracy_std_dev": null, "run_count": 1, "total_questions": 120, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 382535.98333333334, "avg_output_tokens_per_task": 6994.291666666667, "last_run_timestamp": "2026-08-04T05:36:23.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.6824516220136518, "accuracy_std_dev": 0.08182193124974196, "run_count": 8, "total_questions": 1172, "avg_cost_per_task_usd": 0.03734705034129693, "avg_duration_per_task_ms": 243375.2090443686, "avg_output_tokens_per_task": 14147.406996587031, "last_run_timestamp": "2026-08-29T19:20:11.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.681700944587629, "accuracy_std_dev": 0.10151777573533613, "run_count": 4, "total_questions": 776, "avg_cost_per_task_usd": 0.061980561855670094, "avg_duration_per_task_ms": 552656.2654639175, "avg_output_tokens_per_task": 25817.090206185567, "last_run_timestamp": "2026-08-15T10:07:37.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Friendli", "accuracy": 0.681187, "accuracy_std_dev": 0.0554640690582651, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005700236111111112, "avg_duration_per_task_ms": 258962.23358585857, "avg_output_tokens_per_task": 6935.555555555556, "last_run_timestamp": "2026-08-01T17:52:16.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.678451, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0025699040404040404, "avg_duration_per_task_ms": 57532.25252525252, "avg_output_tokens_per_task": 2314.656565656566, "last_run_timestamp": "2026-09-01T13:44:51.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.678451, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01359871212121212, "avg_duration_per_task_ms": 222801.71212121213, "avg_output_tokens_per_task": 33900.580808080806, "last_run_timestamp": "2026-09-01T18:09:36.992Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.675926, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023789772727272726, "avg_duration_per_task_ms": 1250422.4444444445, "avg_output_tokens_per_task": 14190.916666666666, "last_run_timestamp": "2026-07-18T09:58:41.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6757497120822621, "accuracy_std_dev": 0.20642627550360443, "run_count": 10, "total_questions": 1945, "avg_cost_per_task_usd": 0.014975519280205657, "avg_duration_per_task_ms": 371086.62262210797, "avg_output_tokens_per_task": 18143.37994858612, "last_run_timestamp": "2026-09-26T08:53:03.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6750842, "accuracy_std_dev": 0.023641210809939493, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005652539393939394, "avg_duration_per_task_ms": 500294.62828282826, "avg_output_tokens_per_task": 16033.422222222222, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.6717173333333334, "accuracy_std_dev": 0.009621770673264316, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.009016287878787879, "avg_duration_per_task_ms": 610741.1986531987, "avg_output_tokens_per_task": 69134.77777777778, "last_run_timestamp": "2026-07-18T17:04:25.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6708755, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014901020202020204, "avg_duration_per_task_ms": 14005.674242424242, "avg_output_tokens_per_task": 1670.229797979798, "last_run_timestamp": "2026-09-01T13:21:57.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.6700393558178753, "accuracy_std_dev": 0.03269417355990866, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.0023676661045531197, "avg_duration_per_task_ms": 206054.30691399664, "avg_output_tokens_per_task": 5153.2276559865095, "last_run_timestamp": "2026-08-29T17:07:45.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.6651728911651728, "accuracy_std_dev": 0.015545660212339019, "run_count": 8, "total_questions": 1562, "avg_cost_per_task_usd": 0.007795919974391807, "avg_duration_per_task_ms": 518511.552496799, "avg_output_tokens_per_task": 55598.74967989757, "last_run_timestamp": "2026-08-29T21:19:16.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.664983, "accuracy_std_dev": 0.045452036199346076, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008639549663299663, "avg_duration_per_task_ms": 254582.55808080808, "avg_output_tokens_per_task": 42899.02777777778, "last_run_timestamp": "2026-09-26T09:38:02.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.661616, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003362974747474748, "avg_duration_per_task_ms": 65871.15656565657, "avg_output_tokens_per_task": 2362.121212121212, "last_run_timestamp": "2026-08-01T09:30:53.884Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.6613758571428571, "accuracy_std_dev": 0.010355400129243448, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.002499241702741702, "avg_duration_per_task_ms": 96661.21500721501, "avg_output_tokens_per_task": 2316.605339105339, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.6609084779270633, "accuracy_std_dev": 0.011670726056569915, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.00226812412028151, "avg_duration_per_task_ms": 204254.08637236085, "avg_output_tokens_per_task": 3106.995521433141, "last_run_timestamp": "2026-08-29T18:47:21.578Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6594517142857144, "accuracy_std_dev": 0.023871300387789566, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02789490909090909, "avg_duration_per_task_ms": 918713.1623376623, "avg_output_tokens_per_task": 19192.68759018759, "last_run_timestamp": "2026-08-29T18:31:43.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6583050357142856, "accuracy_std_dev": 0.015770411714311358, "run_count": 12, "total_questions": 2352, "avg_cost_per_task_usd": 0.005204377125850341, "avg_duration_per_task_ms": 107952.35246598639, "avg_output_tokens_per_task": 9805.454931972788, "last_run_timestamp": "2026-09-26T08:51:07.739Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.6582523611650485, "accuracy_std_dev": 0.011234936657031045, "run_count": 8, "total_questions": 1545, "avg_cost_per_task_usd": 0.002745144336569579, "avg_duration_per_task_ms": 78708.58122977347, "avg_output_tokens_per_task": 2915.294498381877, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Wafer", "accuracy": 0.6578945157894737, "accuracy_std_dev": 0.06863835015499714, "run_count": 3, "total_questions": 570, "avg_cost_per_task_usd": 0.011776470175438595, "avg_duration_per_task_ms": 345542.28596491227, "avg_output_tokens_per_task": 29245.92456140351, "last_run_timestamp": "2026-07-18T08:04:42.524Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.6572561285289749, "accuracy_std_dev": 0.17360433671530034, "run_count": 8, "total_questions": 1346, "avg_cost_per_task_usd": 0.005431277860326895, "avg_duration_per_task_ms": 197404.2719167905, "avg_output_tokens_per_task": 6718.176820208024, "last_run_timestamp": "2026-08-29T19:47:54.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.6571234233128835, "accuracy_std_dev": 0.018221993227964927, "run_count": 15, "total_questions": 2934, "avg_cost_per_task_usd": 0.009085255623721884, "avg_duration_per_task_ms": 351031.9914792093, "avg_output_tokens_per_task": 60257.48977505112, "last_run_timestamp": "2026-08-29T22:02:09.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.6539546483050847, "accuracy_std_dev": 0.06028980821738355, "run_count": 3, "total_questions": 236, "avg_cost_per_task_usd": 0.09281549576271186, "avg_duration_per_task_ms": 1220256.3305084745, "avg_output_tokens_per_task": 32723.118644067796, "last_run_timestamp": "2026-08-01T18:16:39.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.653757604130809, "accuracy_std_dev": 0.015316614138466363, "run_count": 3, "total_questions": 581, "avg_cost_per_task_usd": 0.01179434079173838, "avg_duration_per_task_ms": 1792911.3149741825, "avg_output_tokens_per_task": 50132.19449225473, "last_run_timestamp": "2026-08-29T20:13:41.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.6529002334801761, "accuracy_std_dev": 0.09643724771974778, "run_count": 5, "total_questions": 908, "avg_cost_per_task_usd": 0.007793224669603524, "avg_duration_per_task_ms": 161069.71255506607, "avg_output_tokens_per_task": 9664.236784140969, "last_run_timestamp": "2026-08-08T16:46:11.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.6517065189340814, "accuracy_std_dev": 0.01961235432447518, "run_count": 11, "total_questions": 2139, "avg_cost_per_task_usd": 0.0068649700794763905, "avg_duration_per_task_ms": 223412.84338475924, "avg_output_tokens_per_task": 11118.252454417952, "last_run_timestamp": "2026-09-26T08:55:02.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.6512783352272727, "accuracy_std_dev": 0.07301920016707937, "run_count": 5, "total_questions": 704, "avg_cost_per_task_usd": 0.06126106249999999, "avg_duration_per_task_ms": 1570715.0482954546, "avg_output_tokens_per_task": 22881.873579545456, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.650363823580786, "accuracy_std_dev": 0.007103203700131033, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.0036315545851528385, "avg_duration_per_task_ms": 58503.03493449782, "avg_output_tokens_per_task": 2202.661135371179, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.6500593861209965, "accuracy_std_dev": 0.05876659054376805, "run_count": 6, "total_questions": 1124, "avg_cost_per_task_usd": 0.04040380338078292, "avg_duration_per_task_ms": 414098.88790035585, "avg_output_tokens_per_task": 20106.83896797153, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6489900000000001, "accuracy_std_dev": 0.01599299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.015694313131313132, "avg_duration_per_task_ms": 13831.43686868687, "avg_output_tokens_per_task": 1769.391414141414, "last_run_timestamp": "2026-09-01T13:20:53.165Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.6478542063492063, "accuracy_std_dev": 0.03133768881430511, "run_count": 3, "total_questions": 567, "avg_cost_per_task_usd": 0.025561564373897706, "avg_duration_per_task_ms": 917973.6243386243, "avg_output_tokens_per_task": 30424.42680776014, "last_run_timestamp": "2026-08-01T17:59:38.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.6477275, "accuracy_std_dev": 0.015583769481418798, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01589866666666667, "avg_duration_per_task_ms": 19920.939393939392, "avg_output_tokens_per_task": 1699.0972222222222, "last_run_timestamp": "2026-09-01T13:27:17.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6475168750000001, "accuracy_std_dev": 0.1144747818369154, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005135561868686868, "avg_duration_per_task_ms": 346231.56313131313, "avg_output_tokens_per_task": 5662.786616161616, "last_run_timestamp": "2026-08-29T19:11:19.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.6471702265758092, "accuracy_std_dev": 0.024540547409049426, "run_count": 9, "total_questions": 1761, "avg_cost_per_task_usd": 0.01087929528676888, "avg_duration_per_task_ms": 696427.9392390687, "avg_output_tokens_per_task": 56280.081771720616, "last_run_timestamp": "2026-08-29T21:14:10.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.646465, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0029047154882154884, "avg_duration_per_task_ms": 53981.06734006734, "avg_output_tokens_per_task": 2363.939393939394, "last_run_timestamp": "2026-08-29T16:33:11.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.6461538384615385, "accuracy_std_dev": 0.010277671231242019, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.0025117333333333335, "avg_duration_per_task_ms": 66425.87692307692, "avg_output_tokens_per_task": 2714.4269230769232, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.6459835714285715, "accuracy_std_dev": 0.011408025080832253, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0032850678210678214, "avg_duration_per_task_ms": 40715.905483405484, "avg_output_tokens_per_task": 3121.487012987013, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.6450217142857143, "accuracy_std_dev": 0.013022733909966664, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0023176284271284268, "avg_duration_per_task_ms": 69183.5432900433, "avg_output_tokens_per_task": 2807.774891774892, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.6446141374045801, "accuracy_std_dev": 0.005257999999999985, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.0025083384223918577, "avg_duration_per_task_ms": 104571.39694656488, "avg_output_tokens_per_task": 8089.860050890586, "last_run_timestamp": "2026-07-18T08:33:31.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.643518425, "accuracy_std_dev": 0.06758775688688005, "run_count": 4, "total_questions": 720, "avg_cost_per_task_usd": 0.028923294444444445, "avg_duration_per_task_ms": 727470.7069444444, "avg_output_tokens_per_task": 35275.89166666667, "last_run_timestamp": "2026-08-01T18:07:49.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.643390096687556, "accuracy_std_dev": 0.16884591783010003, "run_count": 6, "total_questions": 1117, "avg_cost_per_task_usd": 0.08375195344673231, "avg_duration_per_task_ms": 1334122.5317815577, "avg_output_tokens_per_task": 37138.017009847805, "last_run_timestamp": "2026-08-22T17:02:13.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.6431472852791877, "accuracy_std_dev": 0.05705437174474449, "run_count": 8, "total_questions": 985, "avg_cost_per_task_usd": 0.00601549847715736, "avg_duration_per_task_ms": 640442.6040609137, "avg_output_tokens_per_task": 40019.25786802031, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.6430122421874999, "accuracy_std_dev": 0.010735540414902267, "run_count": 8, "total_questions": 1536, "avg_cost_per_task_usd": 0.0024762552083333333, "avg_duration_per_task_ms": 124311.69205729167, "avg_output_tokens_per_task": 8736.557942708334, "last_run_timestamp": "2026-08-29T19:32:59.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.6414145, "accuracy_std_dev": 0.001683499999999949, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004025537878787879, "avg_duration_per_task_ms": 31194.16414141414, "avg_output_tokens_per_task": 2323.563131313131, "last_run_timestamp": "2026-09-01T17:49:44.421Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.641414, "accuracy_std_dev": 0.003367000000000009, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004276166666666667, "avg_duration_per_task_ms": 30535.468434343435, "avg_output_tokens_per_task": 2347.165404040404, "last_run_timestamp": "2026-09-01T17:50:18.482Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6410628652173913, "accuracy_std_dev": 0.012160856360823096, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.0027363427536231887, "avg_duration_per_task_ms": 152044.62463768115, "avg_output_tokens_per_task": 8852.080434782609, "last_run_timestamp": "2026-08-29T18:23:33.062Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.6410423139534884, "accuracy_std_dev": 0.013444699531204668, "run_count": 8, "total_questions": 1548, "avg_cost_per_task_usd": 0.004495651162790697, "avg_duration_per_task_ms": 73088.3649870801, "avg_output_tokens_per_task": 8450.452196382428, "last_run_timestamp": "2026-08-29T18:43:21.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.6408923801169591, "accuracy_std_dev": 0.018618251481661085, "run_count": 8, "total_questions": 1539, "avg_cost_per_task_usd": 0.00244093502274204, "avg_duration_per_task_ms": 207367.10331384017, "avg_output_tokens_per_task": 7914.494476933073, "last_run_timestamp": "2026-08-29T18:40:25.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.6408530000000001, "accuracy_std_dev": 0.00731669260435796, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00720101430976431, "avg_duration_per_task_ms": 742062.9217171717, "avg_output_tokens_per_task": 55987.864478114476, "last_run_timestamp": "2026-08-29T20:02:26.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6397305, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004182681818181818, "avg_duration_per_task_ms": 33306.957070707074, "avg_output_tokens_per_task": 2424.3232323232323, "last_run_timestamp": "2026-09-01T17:50:13.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.6394904285714286, "accuracy_std_dev": 0.01660321769035958, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0016073823953823953, "avg_duration_per_task_ms": 115775.33910533911, "avg_output_tokens_per_task": 8123.611832611833, "last_run_timestamp": "2026-08-29T19:02:06.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.637803, "accuracy_std_dev": null, "run_count": 1, "total_questions": 179, "avg_cost_per_task_usd": 0.002030703910614525, "avg_duration_per_task_ms": 92260.82681564246, "avg_output_tokens_per_task": 6534.692737430168, "last_run_timestamp": "2026-07-18T09:17:11.788Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6359762020460358, "accuracy_std_dev": 0.023468612837149117, "run_count": 4, "total_questions": 782, "avg_cost_per_task_usd": 0.006740489769820971, "avg_duration_per_task_ms": 885605.4833759591, "avg_output_tokens_per_task": 37192.53964194373, "last_run_timestamp": "2026-09-26T08:26:39.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.6352413333333333, "accuracy_std_dev": 0.011527714209195522, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007043973063973064, "avg_duration_per_task_ms": 845264.6161616162, "avg_output_tokens_per_task": 50357.55892255892, "last_run_timestamp": "2026-08-29T22:11:02.681Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.6352257765567765, "accuracy_std_dev": 0.022072905759475025, "run_count": 3, "total_questions": 546, "avg_cost_per_task_usd": 0.013415796703296703, "avg_duration_per_task_ms": 548616.1373626373, "avg_output_tokens_per_task": 46655.3021978022, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.6337451904761905, "accuracy_std_dev": 0.08567913487347233, "run_count": 3, "total_questions": 567, "avg_cost_per_task_usd": 0.03312850793650794, "avg_duration_per_task_ms": 835240.3333333334, "avg_output_tokens_per_task": 32942.40211640212, "last_run_timestamp": "2026-08-01T18:05:37.080Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.6334175, "accuracy_std_dev": 0.007154500000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001171128787878788, "avg_duration_per_task_ms": 257331.09848484848, "avg_output_tokens_per_task": 11528.555555555555, "last_run_timestamp": "2026-08-29T19:38:59.849Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.6332621918976546, "accuracy_std_dev": 0.028783664609624945, "run_count": 8, "total_questions": 1407, "avg_cost_per_task_usd": 0.022304811656005688, "avg_duration_per_task_ms": 1028113.3994314143, "avg_output_tokens_per_task": 92736.27292110874, "last_run_timestamp": "2026-09-26T16:32:03.216Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.6330053363286265, "accuracy_std_dev": 0.03165786628011624, "run_count": 12, "total_questions": 2337, "avg_cost_per_task_usd": 0.0070095879332477545, "avg_duration_per_task_ms": 114460.44458707745, "avg_output_tokens_per_task": 9781.493367565254, "last_run_timestamp": "2026-09-26T09:22:12.174Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.6321081114302126, "accuracy_std_dev": 0.011032621401446443, "run_count": 9, "total_questions": 1741, "avg_cost_per_task_usd": 0.017073134979896612, "avg_duration_per_task_ms": 470536.2297530155, "avg_output_tokens_per_task": 85368.48765077542, "last_run_timestamp": "2026-09-26T16:13:51.663Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.6317296185567011, "accuracy_std_dev": 0.03997713513997718, "run_count": 3, "total_questions": 582, "avg_cost_per_task_usd": 0.02984942783505154, "avg_duration_per_task_ms": 817292.206185567, "avg_output_tokens_per_task": 36396.65120274914, "last_run_timestamp": "2026-08-01T18:25:56.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.6313133333333334, "accuracy_std_dev": 0.009621770673264316, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.012725252525252525, "avg_duration_per_task_ms": 428447.4292929293, "avg_output_tokens_per_task": 30647.838383838385, "last_run_timestamp": "2026-07-18T16:20:00.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6308680607902736, "accuracy_std_dev": 0.011456259937693471, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.0034108257345491387, "avg_duration_per_task_ms": 140057.4062816616, "avg_output_tokens_per_task": 11472.941236068895, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.6300085652173913, "accuracy_std_dev": 0.019291258106544862, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.013243481670929242, "avg_duration_per_task_ms": 617828.7595907928, "avg_output_tokens_per_task": 14030.67348678602, "last_run_timestamp": "2026-09-19T10:52:13.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6297115344283006, "accuracy_std_dev": 0.021927190654171258, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.017034094756790904, "avg_duration_per_task_ms": 638036.0972836387, "avg_output_tokens_per_task": 83517.71572962729, "last_run_timestamp": "2026-09-26T16:16:58.507Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.6294425555555555, "accuracy_std_dev": 0.01594050097575434, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0016517962962962962, "avg_duration_per_task_ms": 198352.56228956228, "avg_output_tokens_per_task": 9040.280583613918, "last_run_timestamp": "2026-08-29T17:04:59.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Venice", "accuracy": 0.6292411856287424, "accuracy_std_dev": 0.0117815, "run_count": 2, "total_questions": 334, "avg_cost_per_task_usd": 0.0071094161676646716, "avg_duration_per_task_ms": 112258.81736526947, "avg_output_tokens_per_task": 28232.077844311378, "last_run_timestamp": "2026-08-15T08:02:44.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.628788, "accuracy_std_dev": 0.01430999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017457550505050507, "avg_duration_per_task_ms": 1341424.7676767677, "avg_output_tokens_per_task": 31206.515151515152, "last_run_timestamp": "2026-07-18T17:46:02.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.626263, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005995020202020202, "avg_duration_per_task_ms": 131491.52525252526, "avg_output_tokens_per_task": 48474.92929292929, "last_run_timestamp": "2026-09-28T18:12:26.736Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Phala", "accuracy": 0.6250000954545455, "accuracy_std_dev": 0.23227730753610482, "run_count": 3, "total_questions": 440, "avg_cost_per_task_usd": 0.01408210681818182, "avg_duration_per_task_ms": 344190.18863636366, "avg_output_tokens_per_task": 10377.513636363636, "last_run_timestamp": "2026-07-18T16:47:35.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.6248440193508115, "accuracy_std_dev": 0.023396789803541672, "run_count": 9, "total_questions": 1602, "avg_cost_per_task_usd": 0.007693858926342072, "avg_duration_per_task_ms": 37290.070536828964, "avg_output_tokens_per_task": 12740.69850187266, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.6248197142857143, "accuracy_std_dev": 0.11942661157585356, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.6020819559884559, "avg_duration_per_task_ms": 624611.841991342, "avg_output_tokens_per_task": 40107.266955266954, "last_run_timestamp": "2026-09-26T08:54:32.678Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.6235651617933724, "accuracy_std_dev": 0.15897481174775574, "run_count": 8, "total_questions": 1539, "avg_cost_per_task_usd": 0.14709008641975307, "avg_duration_per_task_ms": 1168622.465237167, "avg_output_tokens_per_task": 80206.91033138402, "last_run_timestamp": "2026-08-29T19:10:33.460Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.6218904017412935, "accuracy_std_dev": 0.10844110444806741, "run_count": 6, "total_questions": 804, "avg_cost_per_task_usd": 0.010471062189054726, "avg_duration_per_task_ms": 43908.85696517413, "avg_output_tokens_per_task": 6777.06592039801, "last_run_timestamp": "2026-08-22T17:34:08.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6212123333333334, "accuracy_std_dev": 0.009622003960829715, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00826737037037037, "avg_duration_per_task_ms": 370134.4276094276, "avg_output_tokens_per_task": 15814.858585858587, "last_run_timestamp": "2026-08-01T18:08:49.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6180857142857143, "accuracy_std_dev": 0.07026707458326266, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.004998886002886003, "avg_duration_per_task_ms": 573044.753968254, "avg_output_tokens_per_task": 18652.590909090908, "last_run_timestamp": "2026-08-22T22:18:10.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.6170035, "accuracy_std_dev": 0.020976392200519146, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005440873737373737, "avg_duration_per_task_ms": 588685.2001262626, "avg_output_tokens_per_task": 18061.924873737375, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.6166527916666666, "accuracy_std_dev": 0.12145770743554425, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.6553831144781144, "avg_duration_per_task_ms": 484995.4349747475, "avg_output_tokens_per_task": 40791.21506734007, "last_run_timestamp": "2026-09-26T09:11:13.767Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6165192925559605, "accuracy_std_dev": 0.018521943327847644, "run_count": 10, "total_questions": 1921, "avg_cost_per_task_usd": 0.00823595002602811, "avg_duration_per_task_ms": 513809.8833940656, "avg_output_tokens_per_task": 13523.150963040083, "last_run_timestamp": "2026-09-26T09:22:23.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.614478375, "accuracy_std_dev": 0.12366286539715297, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.6154596931818181, "avg_duration_per_task_ms": 632046.8207070708, "avg_output_tokens_per_task": 40881.84659090909, "last_run_timestamp": "2026-09-26T09:34:52.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.6139495138993238, "accuracy_std_dev": 0.029024384950147974, "run_count": 8, "total_questions": 1331, "avg_cost_per_task_usd": 0.02793150338091661, "avg_duration_per_task_ms": 583500.4891059354, "avg_output_tokens_per_task": 29316.752817430504, "last_run_timestamp": "2026-09-26T16:19:07.764Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6129581867007673, "accuracy_std_dev": 0.0015394999999999714, "run_count": 2, "total_questions": 391, "avg_cost_per_task_usd": 0.0032793452685421997, "avg_duration_per_task_ms": 346994.51406649617, "avg_output_tokens_per_task": 13256.76726342711, "last_run_timestamp": "2026-09-01T13:34:17.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.6126859920896506, "accuracy_std_dev": 0.1240297456369234, "run_count": 23, "total_questions": 4551, "avg_cost_per_task_usd": 0.6596944438584925, "avg_duration_per_task_ms": 597097.6884201275, "avg_output_tokens_per_task": 41139.12920237311, "last_run_timestamp": "2026-09-26T09:32:36.461Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "NextBit", "accuracy": 0.6111113258426966, "accuracy_std_dev": 0.003763737002265482, "run_count": 3, "total_questions": 534, "avg_cost_per_task_usd": 0.009120874531835207, "avg_duration_per_task_ms": 904452.6797752809, "avg_output_tokens_per_task": 16552.734082397004, "last_run_timestamp": "2026-07-18T08:03:56.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.6104652558139534, "accuracy_std_dev": 0.010142144016429655, "run_count": 8, "total_questions": 1548, "avg_cost_per_task_usd": 0.0036393624031007747, "avg_duration_per_task_ms": 34858.98837209302, "avg_output_tokens_per_task": 2148.659560723514, "last_run_timestamp": "2026-08-29T18:49:22.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.6103109080932785, "accuracy_std_dev": 0.006004573355992887, "run_count": 8, "total_questions": 1458, "avg_cost_per_task_usd": 0.00404280109739369, "avg_duration_per_task_ms": 46573.90877914952, "avg_output_tokens_per_task": 2150.791495198903, "last_run_timestamp": "2026-08-29T18:43:09.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.609428, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0023919898989898988, "avg_duration_per_task_ms": 73719.80808080808, "avg_output_tokens_per_task": 2205.409090909091, "last_run_timestamp": "2026-09-01T13:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.608871193548387, "accuracy_std_dev": 0.0048793077890209365, "run_count": 4, "total_questions": 744, "avg_cost_per_task_usd": 0.007782237903225806, "avg_duration_per_task_ms": 296799.64919354836, "avg_output_tokens_per_task": 15367.672043010753, "last_run_timestamp": "2026-08-01T18:16:00.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.6087965, "accuracy_std_dev": 0.12831918473770787, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.6162571098484848, "avg_duration_per_task_ms": 633013.7184343435, "avg_output_tokens_per_task": 41027.7380050505, "last_run_timestamp": "2026-09-26T09:34:56.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.6085801047835991, "accuracy_std_dev": 0.25326376299160264, "run_count": 3, "total_questions": 439, "avg_cost_per_task_usd": 0.01201555353075171, "avg_duration_per_task_ms": 304422.4123006834, "avg_output_tokens_per_task": 10169.57630979499, "last_run_timestamp": "2026-07-18T16:53:38.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.607744, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003953611111111112, "avg_duration_per_task_ms": 57224.242424242424, "avg_output_tokens_per_task": 2094.1161616161617, "last_run_timestamp": "2026-08-01T17:44:33.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6065783472485767, "accuracy_std_dev": 0.011150535787995776, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.00814792915876028, "avg_duration_per_task_ms": 295371.79759645794, "avg_output_tokens_per_task": 17556.894370651487, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.6060606666666666, "accuracy_std_dev": 0.0013745720142000956, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008301892255892255, "avg_duration_per_task_ms": 153838.79292929292, "avg_output_tokens_per_task": 15775.223905723906, "last_run_timestamp": "2026-08-01T17:51:29.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.605886368522073, "accuracy_std_dev": 0.02016173504399793, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.07002656621880997, "avg_duration_per_task_ms": 1050392.2853486885, "avg_output_tokens_per_task": 26862.438899552144, "last_run_timestamp": "2026-08-30T00:22:07.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.6058202261904763, "accuracy_std_dev": 0.04086605084220276, "run_count": 8, "total_questions": 1512, "avg_cost_per_task_usd": 0.022986974867724868, "avg_duration_per_task_ms": 1010656.5641534391, "avg_output_tokens_per_task": 59799.07671957672, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.6053392857142857, "accuracy_std_dev": 0.004486622487678077, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01102401875901876, "avg_duration_per_task_ms": 34344.32323232323, "avg_output_tokens_per_task": 2100.313131313131, "last_run_timestamp": "2026-08-29T18:32:24.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.6047306974789916, "accuracy_std_dev": 0.2173110611308259, "run_count": 6, "total_questions": 1071, "avg_cost_per_task_usd": 0.059062111111111114, "avg_duration_per_task_ms": 1034544.568627451, "avg_output_tokens_per_task": 19772.375350140057, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.604539935483871, "accuracy_std_dev": 0.0028258301828351473, "run_count": 3, "total_questions": 558, "avg_cost_per_task_usd": 0.004098256272401434, "avg_duration_per_task_ms": 59405.28315412186, "avg_output_tokens_per_task": 2189.059139784946, "last_run_timestamp": "2026-07-18T08:26:39.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.6035355000000001, "accuracy_std_dev": 0.004208499999999948, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.002990320707070707, "avg_duration_per_task_ms": 555382.6085858586, "avg_output_tokens_per_task": 13221.70707070707, "last_run_timestamp": "2026-09-01T13:04:03.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6026938333333334, "accuracy_std_dev": 0.05444530556260618, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.011141401515151514, "avg_duration_per_task_ms": 1165344.244949495, "avg_output_tokens_per_task": 32945.469696969696, "last_run_timestamp": "2026-08-29T19:15:54.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.6025891961165049, "accuracy_std_dev": 0.006394638535523308, "run_count": 8, "total_questions": 1545, "avg_cost_per_task_usd": 0.0033181844660194176, "avg_duration_per_task_ms": 55680.37928802589, "avg_output_tokens_per_task": 2110.4776699029126, "last_run_timestamp": "2026-08-29T18:47:46.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.6014871513560806, "accuracy_std_dev": 0.0142573356060427, "run_count": 6, "total_questions": 1143, "avg_cost_per_task_usd": 0.002170363954505687, "avg_duration_per_task_ms": 45118.168853893265, "avg_output_tokens_per_task": 2030.7734033245845, "last_run_timestamp": "2026-08-29T18:37:17.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.6005190778210118, "accuracy_std_dev": 0.014559222441784965, "run_count": 8, "total_questions": 1542, "avg_cost_per_task_usd": 0.00329739364461738, "avg_duration_per_task_ms": 57092.898184176396, "avg_output_tokens_per_task": 2093.910505836576, "last_run_timestamp": "2026-08-29T19:00:24.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.6004493333333333, "accuracy_std_dev": 0.0031744380430067634, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.014951090909090909, "avg_duration_per_task_ms": 287967.1666666667, "avg_output_tokens_per_task": 12268.760942760942, "last_run_timestamp": "2026-08-15T15:26:03.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.5946250426829269, "accuracy_std_dev": 0.04090406951159473, "run_count": 8, "total_questions": 1476, "avg_cost_per_task_usd": 0.09460423780487806, "avg_duration_per_task_ms": 840149.4261517616, "avg_output_tokens_per_task": 30270.114498644987, "last_run_timestamp": "2026-08-29T19:11:06.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.594276, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.002775611111111111, "avg_duration_per_task_ms": 509237.26767676766, "avg_output_tokens_per_task": 10972.9898989899, "last_run_timestamp": "2026-09-01T13:31:26.819Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.5922561000000001, "accuracy_std_dev": 0.11721268776753647, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.6146375166666666, "avg_duration_per_task_ms": 629521.5186868687, "avg_output_tokens_per_task": 40864.73838383838, "last_run_timestamp": "2026-09-26T09:23:13.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5900675, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017035757575757576, "avg_duration_per_task_ms": 123031.02272727272, "avg_output_tokens_per_task": 42470.74747474748, "last_run_timestamp": "2026-09-01T18:10:49.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.5872781331360947, "accuracy_std_dev": 0.004687499999999956, "run_count": 2, "total_questions": 338, "avg_cost_per_task_usd": 0.09507673668639052, "avg_duration_per_task_ms": 191708.50591715975, "avg_output_tokens_per_task": 21455.86094674556, "last_run_timestamp": "2026-08-01T18:26:20.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.5832802341650671, "accuracy_std_dev": 0.015118863604368572, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.006713769673704416, "avg_duration_per_task_ms": 723452, "avg_output_tokens_per_task": 44499.71785028791, "last_run_timestamp": "2026-08-29T16:53:28.602Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.5778790920852359, "accuracy_std_dev": 0.14127804644250064, "run_count": 8, "total_questions": 1314, "avg_cost_per_task_usd": 0.014400366057838659, "avg_duration_per_task_ms": 93590.71461187214, "avg_output_tokens_per_task": 47817.760273972606, "last_run_timestamp": "2026-08-29T19:48:18.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.577862, "accuracy_std_dev": 0.03661599999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006390335858585858, "avg_duration_per_task_ms": 667369.1111111111, "avg_output_tokens_per_task": 32604.280303030304, "last_run_timestamp": "2026-09-26T08:13:27.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.5765865295404814, "accuracy_std_dev": 0.016216367287404426, "run_count": 3, "total_questions": 457, "avg_cost_per_task_usd": 0.0037329715536105032, "avg_duration_per_task_ms": 2186448.089715536, "avg_output_tokens_per_task": 36169.22100656455, "last_run_timestamp": "2026-07-18T16:00:09.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.5763891250000002, "accuracy_std_dev": 0.016770417924410086, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.010578772727272727, "avg_duration_per_task_ms": 12384.83143939394, "avg_output_tokens_per_task": 13650.558712121212, "last_run_timestamp": "2026-08-29T18:50:56.950Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.5749159999999999, "accuracy_std_dev": 0.02739257190371141, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0045310864898989904, "avg_duration_per_task_ms": 339529.7127525252, "avg_output_tokens_per_task": 15954.368055555555, "last_run_timestamp": "2026-08-29T19:16:45.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.5646441153846153, "accuracy_std_dev": 0.09153255016108605, "run_count": 11, "total_questions": 1846, "avg_cost_per_task_usd": 0.009133297399783316, "avg_duration_per_task_ms": 547859.6289274106, "avg_output_tokens_per_task": 45474.855904658725, "last_run_timestamp": "2026-09-26T16:17:38.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5637339187817259, "accuracy_std_dev": 0.03764953294383463, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.025736050761421318, "avg_duration_per_task_ms": 1231284.9746192894, "avg_output_tokens_per_task": 65120.38747884941, "last_run_timestamp": "2026-08-30T00:49:28.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.55303, "accuracy_std_dev": 0.061447999999999975, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006801247474747475, "avg_duration_per_task_ms": 137063.60858585857, "avg_output_tokens_per_task": 7268.926767676768, "last_run_timestamp": "2026-09-01T13:21:33.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Chutes", "accuracy": 0.5521092952853598, "accuracy_std_dev": 0.2509952963755828, "run_count": 3, "total_questions": 403, "avg_cost_per_task_usd": 0.04710845161290323, "avg_duration_per_task_ms": 665624.1687344913, "avg_output_tokens_per_task": 19826.736972704715, "last_run_timestamp": "2026-07-18T16:43:10.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.5514706253196932, "accuracy_std_dev": 0.20217034155010072, "run_count": 8, "total_questions": 1564, "avg_cost_per_task_usd": 0.02285590537084399, "avg_duration_per_task_ms": 745462.75, "avg_output_tokens_per_task": 32123.599104859335, "last_run_timestamp": "2026-09-26T16:19:31.961Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5496635000000001, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017014479797979796, "avg_duration_per_task_ms": 123100.59595959596, "avg_output_tokens_per_task": 42507.18939393939, "last_run_timestamp": "2026-09-01T17:51:18.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5478889309989701, "accuracy_std_dev": 0.06042322607462552, "run_count": 7, "total_questions": 971, "avg_cost_per_task_usd": 0.053327887744593204, "avg_duration_per_task_ms": 699757.6158599382, "avg_output_tokens_per_task": 25522.51184346035, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.5476354460285131, "accuracy_std_dev": 0.04258471863891501, "run_count": 9, "total_questions": 1473, "avg_cost_per_task_usd": 0.016013412763068568, "avg_duration_per_task_ms": 701597.7447386286, "avg_output_tokens_per_task": 40502.957230142565, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.5433505, "accuracy_std_dev": 0.016294800251920854, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.017506126262626265, "avg_duration_per_task_ms": 128742.19065656565, "avg_output_tokens_per_task": 43717.62373737374, "last_run_timestamp": "2026-09-01T18:06:03.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.5431397499999999, "accuracy_std_dev": 0.09309990509897151, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00908125946969697, "avg_duration_per_task_ms": 307947.68244949495, "avg_output_tokens_per_task": 26991.902777777777, "last_run_timestamp": "2026-08-29T17:01:22.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.5429295000000001, "accuracy_std_dev": 0.010942500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001760580808080808, "avg_duration_per_task_ms": 80209.29040404041, "avg_output_tokens_per_task": 1718.6565656565656, "last_run_timestamp": "2026-09-01T13:31:30.699Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.542088, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023943484848484846, "avg_duration_per_task_ms": 10793.646464646465, "avg_output_tokens_per_task": 1334.611111111111, "last_run_timestamp": "2026-08-01T08:53:34.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.5388051381909548, "accuracy_std_dev": 0.02361431811919149, "run_count": 8, "total_questions": 1194, "avg_cost_per_task_usd": 0.09965015829145729, "avg_duration_per_task_ms": 888064.2537688442, "avg_output_tokens_per_task": 28386.65661641541, "last_run_timestamp": "2026-08-29T18:52:09.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.5367792126099705, "accuracy_std_dev": 0.08908980723622514, "run_count": 7, "total_questions": 1364, "avg_cost_per_task_usd": 0.02643020747800586, "avg_duration_per_task_ms": 2484462.840175953, "avg_output_tokens_per_task": 22046.382697947214, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.53367, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01864569696969697, "avg_duration_per_task_ms": 17970.575757575756, "avg_output_tokens_per_task": 1670.050505050505, "last_run_timestamp": "2026-09-01T13:20:31.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.5317058333333333, "accuracy_std_dev": 0.3019237221956437, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02117137962962963, "avg_duration_per_task_ms": 300182.6590909091, "avg_output_tokens_per_task": 22447.186026936026, "last_run_timestamp": "2026-08-15T15:24:35.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.530724, "accuracy_std_dev": 0.006341146150342195, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005484556818181818, "avg_duration_per_task_ms": 124126.7058080808, "avg_output_tokens_per_task": 11845.058080808081, "last_run_timestamp": "2026-08-01T17:48:17.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.521886, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.016360343434343436, "avg_duration_per_task_ms": 13511.040404040405, "avg_output_tokens_per_task": 1443.3737373737374, "last_run_timestamp": "2026-09-01T14:02:01.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.5202023333333333, "accuracy_std_dev": 0.0023810642345154617, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0035723956228956226, "avg_duration_per_task_ms": 90157.08922558922, "avg_output_tokens_per_task": 2721.4242424242425, "last_run_timestamp": "2026-08-15T08:46:42.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.5190796666666666, "accuracy_std_dev": 0.008837261503932561, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004016927609427609, "avg_duration_per_task_ms": 28916.838383838385, "avg_output_tokens_per_task": 2584.1296296296296, "last_run_timestamp": "2026-08-15T09:25:42.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.517396, "accuracy_std_dev": 0.02012409515978297, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015192781144781144, "avg_duration_per_task_ms": 14836.205387205388, "avg_output_tokens_per_task": 1327.0218855218855, "last_run_timestamp": "2026-09-01T13:22:30.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.516835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018182969696969697, "avg_duration_per_task_ms": 17464.777777777777, "avg_output_tokens_per_task": 1627.171717171717, "last_run_timestamp": "2026-09-01T13:20:49.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.5157126666666666, "accuracy_std_dev": 0.02305568803763809, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004710382154882156, "avg_duration_per_task_ms": 211036.34006734006, "avg_output_tokens_per_task": 2694.722222222222, "last_run_timestamp": "2026-08-15T09:11:29.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.5134683333333333, "accuracy_std_dev": 0.00836132925370652, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008388316498316498, "avg_duration_per_task_ms": 89120.7138047138, "avg_output_tokens_per_task": 2792.7188552188554, "last_run_timestamp": "2026-08-15T08:49:03.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.511785, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01733130303030303, "avg_duration_per_task_ms": 13903.823232323231, "avg_output_tokens_per_task": 1536.9141414141413, "last_run_timestamp": "2026-09-01T13:47:27.116Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.50968, "accuracy_std_dev": 0.006561008459070931, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.00660445202020202, "avg_duration_per_task_ms": 165597.95580808082, "avg_output_tokens_per_task": 13339.318181818182, "last_run_timestamp": "2026-08-01T18:08:24.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.5095547757619738, "accuracy_std_dev": 0.07224117651877167, "run_count": 7, "total_questions": 1378, "avg_cost_per_task_usd": 0.003949532656023222, "avg_duration_per_task_ms": 149830.31567489114, "avg_output_tokens_per_task": 4238.851959361394, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.5092595, "accuracy_std_dev": 0.009259499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008195479797979797, "avg_duration_per_task_ms": 19948.4797979798, "avg_output_tokens_per_task": 1859.0176767676767, "last_run_timestamp": "2026-09-01T17:50:32.282Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.506734, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0237055303030303, "avg_duration_per_task_ms": 11478.515151515152, "avg_output_tokens_per_task": 1318.7474747474748, "last_run_timestamp": "2026-08-01T08:34:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02292151515151515, "avg_duration_per_task_ms": 12797.353535353535, "avg_output_tokens_per_task": 1266.479797979798, "last_run_timestamp": "2026-08-01T08:47:47.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.5033673333333334, "accuracy_std_dev": 0.009622237242739144, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008440841750841751, "avg_duration_per_task_ms": 14338.025252525253, "avg_output_tokens_per_task": 1851.915824915825, "last_run_timestamp": "2026-09-01T17:51:18.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.503367, "accuracy_std_dev": 0.008418000000000037, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013204560606060607, "avg_duration_per_task_ms": 13084.876262626263, "avg_output_tokens_per_task": 1127.6338383838383, "last_run_timestamp": "2026-09-01T13:20:22.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.4964458888888889, "accuracy_std_dev": 0.015601016017087856, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00236331088664422, "avg_duration_per_task_ms": 11415.718294051627, "avg_output_tokens_per_task": 2344.6705948372614, "last_run_timestamp": "2026-08-29T19:15:06.404Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.4882155000000001, "accuracy_std_dev": 0.016835500000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008046262626262627, "avg_duration_per_task_ms": 20667.484848484848, "avg_output_tokens_per_task": 1820.4444444444443, "last_run_timestamp": "2026-09-01T17:50:43.769Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.4863289203860072, "accuracy_std_dev": 0.11980845067942446, "run_count": 7, "total_questions": 829, "avg_cost_per_task_usd": 0.031845395657418576, "avg_duration_per_task_ms": 57849.05307599517, "avg_output_tokens_per_task": 6810.792521109771, "last_run_timestamp": "2026-08-29T16:54:06.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.48477567101648344, "accuracy_std_dev": 0.036899227161410524, "run_count": 8, "total_questions": 1456, "avg_cost_per_task_usd": 0.011051652472527472, "avg_duration_per_task_ms": 444726.33997252746, "avg_output_tokens_per_task": 25860.356456043955, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.4834454444444444, "accuracy_std_dev": 0.005804812220354784, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0023795224466891134, "avg_duration_per_task_ms": 13393.97418630752, "avg_output_tokens_per_task": 2241.2485970819303, "last_run_timestamp": "2026-08-29T16:43:47.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.48292587155963307, "accuracy_std_dev": 0.12738947709449225, "run_count": 7, "total_questions": 1308, "avg_cost_per_task_usd": 0.0038326207951070333, "avg_duration_per_task_ms": 105139.82721712538, "avg_output_tokens_per_task": 3476.9533639143733, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.481481, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.001115070707070707, "avg_duration_per_task_ms": 120135.48484848485, "avg_output_tokens_per_task": 2056.4848484848485, "last_run_timestamp": "2026-09-01T14:05:19.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.48106050000000006, "accuracy_std_dev": 0.018107337511351573, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0010583617424242422, "avg_duration_per_task_ms": 63968.31502525252, "avg_output_tokens_per_task": 2267.661616161616, "last_run_timestamp": "2026-08-29T19:21:45.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.4789527698717949, "accuracy_std_dev": 0.04728636391114779, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.010833255128205129, "avg_duration_per_task_ms": 439559.2346153846, "avg_output_tokens_per_task": 27024.191025641026, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.47661788888888884, "accuracy_std_dev": 0.012408007588689965, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0031713282828282827, "avg_duration_per_task_ms": 53718.556116722786, "avg_output_tokens_per_task": 2243.5089786756453, "last_run_timestamp": "2026-08-29T18:45:20.686Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.476434512295082, "accuracy_std_dev": 0.3072418769536471, "run_count": 3, "total_questions": 488, "avg_cost_per_task_usd": 0.05052364344262295, "avg_duration_per_task_ms": 618195.0081967213, "avg_output_tokens_per_task": 20701.57786885246, "last_run_timestamp": "2026-07-18T16:15:33.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.47643079999999993, "accuracy_std_dev": 0.007896357575490103, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0023637252525252526, "avg_duration_per_task_ms": 39731.30202020202, "avg_output_tokens_per_task": 2520.320202020202, "last_run_timestamp": "2026-08-29T18:16:38.845Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.4758696666666666, "accuracy_std_dev": 0.013768379457615502, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0023374680134680133, "avg_duration_per_task_ms": 37003.887205387204, "avg_output_tokens_per_task": 2485.9175084175085, "last_run_timestamp": "2026-07-18T16:13:31.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.4757993750000001, "accuracy_std_dev": 0.012652503437437788, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0014489943181818182, "avg_duration_per_task_ms": 65459.88446969697, "avg_output_tokens_per_task": 2312.2613636363635, "last_run_timestamp": "2026-08-29T18:24:23.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.47530849999999997, "accuracy_std_dev": 0.00867533989247684, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.04888047474747474, "avg_duration_per_task_ms": 45790.32744107744, "avg_output_tokens_per_task": 17137.850168350167, "last_run_timestamp": "2026-08-15T14:46:10.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.4748365805432723, "accuracy_std_dev": 0.012294356500036914, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.001331595072646873, "avg_duration_per_task_ms": 48334.044219835756, "avg_output_tokens_per_task": 2397.6392924826278, "last_run_timestamp": "2026-08-29T17:00:17.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.47447243417721524, "accuracy_std_dev": 0.01660465778733108, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.0012851841772151898, "avg_duration_per_task_ms": 95695.09050632911, "avg_output_tokens_per_task": 2940.8867088607594, "last_run_timestamp": "2026-08-29T19:00:36.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.47327425, "accuracy_std_dev": 0.012469075374200769, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0010748301767676768, "avg_duration_per_task_ms": 178643.0707070707, "avg_output_tokens_per_task": 2430.4804292929293, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.46742483824768327, "accuracy_std_dev": 0.006948085005956674, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.002137747262005055, "avg_duration_per_task_ms": 34994.02274641955, "avg_output_tokens_per_task": 2587.217354675653, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.46569850000000007, "accuracy_std_dev": 0.0211435505355652, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0009726685606060605, "avg_duration_per_task_ms": 187606.375, "avg_output_tokens_per_task": 2789.0145202020203, "last_run_timestamp": "2026-08-29T18:48:55.759Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.462542, "accuracy_std_dev": 0.005471000000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001783989898989899, "avg_duration_per_task_ms": 27644.777777777777, "avg_output_tokens_per_task": 1397.5252525252524, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.46128619685039374, "accuracy_std_dev": 0.075159, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.00731248031496063, "avg_duration_per_task_ms": 1016477.8543307086, "avg_output_tokens_per_task": 20586.740157480315, "last_run_timestamp": "2026-08-08T10:11:59.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.4461280000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0009776919191919191, "avg_duration_per_task_ms": 90556.41414141415, "avg_output_tokens_per_task": 1759.8131313131314, "last_run_timestamp": "2026-09-01T14:15:49.249Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.4444445, "accuracy_std_dev": 0.015151500000000012, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0015239469696969696, "avg_duration_per_task_ms": 59255.792929292926, "avg_output_tokens_per_task": 1592.9924242424242, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.4402355, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0013545, "avg_duration_per_task_ms": 68001.7196969697, "avg_output_tokens_per_task": 1535.1843434343434, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.4335015, "accuracy_std_dev": 0.009259500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010084242424242425, "avg_duration_per_task_ms": 21177.93181818182, "avg_output_tokens_per_task": 1486.2676767676767, "last_run_timestamp": "2026-09-01T17:49:46.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4301345, "accuracy_std_dev": 0.009259500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010125732323232322, "avg_duration_per_task_ms": 17946.272727272728, "avg_output_tokens_per_task": 1495.5707070707072, "last_run_timestamp": "2026-09-01T17:55:31.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.4297135, "accuracy_std_dev": 0.007469839539775939, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010784545454545454, "avg_duration_per_task_ms": 21933.41792929293, "avg_output_tokens_per_task": 1515.1376262626263, "last_run_timestamp": "2026-09-01T17:31:07.952Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Google", "accuracy": 0.39295421951219517, "accuracy_std_dev": 0.1609075, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.004463872628726287, "avg_duration_per_task_ms": 406770.0298102981, "avg_output_tokens_per_task": 25760.539295392955, "last_run_timestamp": "2026-07-11T15:34:14.755Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy-mt2-30b-a3b-20260521", "model_slug": "tencent/hy-mt2-30b-a3b", "model_name": "Tencent: Hy-MT2-30B-A3B", "provider_name": "Reka", "accuracy": 0.388889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00023147979797979797, "avg_duration_per_task_ms": 8604.353535353535, "avg_output_tokens_per_task": 710.540404040404, "last_run_timestamp": "2026-09-24T06:00:03.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.3860826666666667, "accuracy_std_dev": 0.005722799567422292, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00015977104377104374, "avg_duration_per_task_ms": 6254.590909090909, "avg_output_tokens_per_task": 953.9612794612794, "last_run_timestamp": "2026-09-01T13:54:02.609Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.38548754421768705, "accuracy_std_dev": 0.12183173346965978, "run_count": 3, "total_questions": 441, "avg_cost_per_task_usd": 0.009109496598639456, "avg_duration_per_task_ms": 405162.566893424, "avg_output_tokens_per_task": 14046.952380952382, "last_run_timestamp": "2026-08-01T18:13:08.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.36363599999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00041437878787878784, "avg_duration_per_task_ms": 34981.92424242424, "avg_output_tokens_per_task": 1665.6666666666667, "last_run_timestamp": "2026-08-01T08:55:19.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.3526882276497696, "accuracy_std_dev": 0.11263472264421345, "run_count": 6, "total_questions": 1085, "avg_cost_per_task_usd": 0.013297605529953918, "avg_duration_per_task_ms": 2750242.6949308757, "avg_output_tokens_per_task": 14171.58801843318, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Together", "accuracy": 0.3417510000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0007054343434343434, "avg_duration_per_task_ms": 14064.868686868687, "avg_output_tokens_per_task": 1539, "last_run_timestamp": "2026-08-01T08:06:23.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Nebius", "accuracy": 0.338384, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06806519191919191, "avg_duration_per_task_ms": 227007.78787878787, "avg_output_tokens_per_task": 16827.81313131313, "last_run_timestamp": "2026-07-11T08:07:17.820Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.328283, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005318181818181818, "avg_duration_per_task_ms": 36388.32323232323, "avg_output_tokens_per_task": 2041.050505050505, "last_run_timestamp": "2026-08-01T08:36:11.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.3251026825396825, "accuracy_std_dev": 0.01882667006019798, "run_count": 3, "total_questions": 567, "avg_cost_per_task_usd": 0.0001578130511463845, "avg_duration_per_task_ms": 9924.358024691359, "avg_output_tokens_per_task": 577.2874779541446, "last_run_timestamp": "2026-07-18T09:09:45.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.324373, "accuracy_std_dev": 0.011389673110127261, "run_count": 3, "total_questions": 558, "avg_cost_per_task_usd": 0.00011326881720430107, "avg_duration_per_task_ms": 11535.883512544802, "avg_output_tokens_per_task": 458.0806451612903, "last_run_timestamp": "2026-07-18T09:08:52.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.3174145461538462, "accuracy_std_dev": 0.06193726613628906, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.0030096519230769234, "avg_duration_per_task_ms": 33381.006410256414, "avg_output_tokens_per_task": 1258.1852564102564, "last_run_timestamp": "2026-08-29T17:41:08.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DeepInfra", "accuracy": 0.3152173260869565, "accuracy_std_dev": 0.010785089831182077, "run_count": 3, "total_questions": 552, "avg_cost_per_task_usd": 0.000030884057971014495, "avg_duration_per_task_ms": 13871.666666666666, "avg_output_tokens_per_task": 431.25, "last_run_timestamp": "2026-07-18T08:00:15.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Novita", "accuracy": 0.3119177802607076, "accuracy_std_dev": 0.024779336369016997, "run_count": 3, "total_questions": 537, "avg_cost_per_task_usd": 0.000113, "avg_duration_per_task_ms": 43675.3947858473, "avg_output_tokens_per_task": 480.8175046554935, "last_run_timestamp": "2026-07-18T08:00:15.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.29040400000000005, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.000060893939393939394, "avg_duration_per_task_ms": 9271.39393939394, "avg_output_tokens_per_task": 573.0479797979798, "last_run_timestamp": "2026-09-01T13:00:51.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.2866762857142857, "accuracy_std_dev": 0.018435693808589945, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0014998318903318908, "avg_duration_per_task_ms": 64419.20274170274, "avg_output_tokens_per_task": 19046.510822510823, "last_run_timestamp": "2026-08-29T18:21:00.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.2780711647307286, "accuracy_std_dev": 0.014628725522067875, "run_count": 5, "total_questions": 947, "avg_cost_per_task_usd": 0.0063300253431890175, "avg_duration_per_task_ms": 245873.9429778247, "avg_output_tokens_per_task": 28094.863780359028, "last_run_timestamp": "2026-08-29T18:18:26.937Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DekaLLM", "accuracy": 0.27607364417177915, "accuracy_std_dev": 0.013048869410369953, "run_count": 3, "total_questions": 489, "avg_cost_per_task_usd": 0.000023339468302658487, "avg_duration_per_task_ms": 14320.529652351739, "avg_output_tokens_per_task": 284.7852760736196, "last_run_timestamp": "2026-07-18T08:04:03.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.275533, "accuracy_std_dev": 0.009152576176501717, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0007405521885521885, "avg_duration_per_task_ms": 48360.410774410775, "avg_output_tokens_per_task": 2532.1531986531986, "last_run_timestamp": "2026-08-15T09:28:07.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.2722029671280277, "accuracy_std_dev": 0.017795645653423824, "run_count": 9, "total_questions": 1734, "avg_cost_per_task_usd": 0.0002858858131487889, "avg_duration_per_task_ms": 345712.0732410611, "avg_output_tokens_per_task": 6992.540369088812, "last_run_timestamp": "2026-08-29T16:57:41.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.27185566666666666, "accuracy_std_dev": 0.008747009330178078, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.00006098307952622674, "avg_duration_per_task_ms": 9307.97461928934, "avg_output_tokens_per_task": 612.7817258883249, "last_run_timestamp": "2026-09-01T12:41:31.862Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.2700818571428571, "accuracy_std_dev": 0.012593595701541222, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0017664740259740262, "avg_duration_per_task_ms": 42604.03391053391, "avg_output_tokens_per_task": 22219.9329004329, "last_run_timestamp": "2026-08-29T18:45:59.793Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.26936, "accuracy_std_dev": 0.00673399999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0000632449494949495, "avg_duration_per_task_ms": 9455.388888888889, "avg_output_tokens_per_task": 592.1616161616162, "last_run_timestamp": "2026-09-01T13:21:46.100Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.268788525198939, "accuracy_std_dev": 0.004147499999999998, "run_count": 2, "total_questions": 377, "avg_cost_per_task_usd": 0.0000679681697612732, "avg_duration_per_task_ms": 9180.114058355437, "avg_output_tokens_per_task": 568.5623342175066, "last_run_timestamp": "2026-09-01T13:04:23.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.258137, "accuracy_std_dev": 0.0432133818895644, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00887828367003367, "avg_duration_per_task_ms": 545067.4250841751, "avg_output_tokens_per_task": 21968.73063973064, "last_run_timestamp": "2026-08-29T19:48:22.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "WandB", "accuracy": 0.25591029787234043, "accuracy_std_dev": 0.0038087773191231088, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.0013007553191489362, "avg_duration_per_task_ms": 38112.132978723406, "avg_output_tokens_per_task": 5103.014184397163, "last_run_timestamp": "2026-07-18T08:26:34.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.24990534129692835, "accuracy_std_dev": 0.019806748083779954, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.00038077246871444826, "avg_duration_per_task_ms": 63806.37883959044, "avg_output_tokens_per_task": 7291.984072810012, "last_run_timestamp": "2026-08-29T16:55:14.828Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.23833585714285715, "accuracy_std_dev": 0.09800656715959478, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.001642277056277056, "avg_duration_per_task_ms": 571156.3896103896, "avg_output_tokens_per_task": 5738.018037518037, "last_run_timestamp": "2026-08-29T17:16:14.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Parasail", "accuracy": 0.21734899999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.0003523450292397661, "avg_duration_per_task_ms": 72727.70760233919, "avg_output_tokens_per_task": 4269.023391812865, "last_run_timestamp": "2026-08-01T08:05:56.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.19696999999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0190694696969697, "avg_duration_per_task_ms": 2493159.101010101, "avg_output_tokens_per_task": 105666.86363636363, "last_run_timestamp": "2026-09-03T21:14:17.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.16687712500000002, "accuracy_std_dev": 0.10885759932641073, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0013347329545454543, "avg_duration_per_task_ms": 43166.87626262626, "avg_output_tokens_per_task": 1152.546085858586, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "auto-routing", "accuracy": 0.116162, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0001537979797979798, "avg_duration_per_task_ms": 29722.909090909092, "avg_output_tokens_per_task": 1836.7929292929293, "last_run_timestamp": "2026-08-01T09:03:02.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.114478, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.017837661616161616, "avg_duration_per_task_ms": 71618.32323232324, "avg_output_tokens_per_task": 11767.727272727272, "last_run_timestamp": "2026-09-01T12:56:59.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.10606100000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.017886560606060607, "avg_duration_per_task_ms": 21072.777777777777, "avg_output_tokens_per_task": 11801.328282828283, "last_run_timestamp": "2026-09-01T13:00:01.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018059262626262628, "avg_duration_per_task_ms": 21777.424242424244, "avg_output_tokens_per_task": 11808.9898989899, "last_run_timestamp": "2026-09-01T13:21:37.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.060606, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.000040161616161616165, "avg_duration_per_task_ms": 3671.1060606060605, "avg_output_tokens_per_task": 93.72222222222223, "last_run_timestamp": "2026-08-01T08:05:34.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Together", "accuracy": 0.005051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00002468181818181818, "avg_duration_per_task_ms": 955.2929292929293, "avg_output_tokens_per_task": 122.03030303030303, "last_run_timestamp": "2026-07-04T16:00:23.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.002806333333333333, "accuracy_std_dev": 0.0015872190215034036, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0035528451178451176, "avg_duration_per_task_ms": 168324.14646464647, "avg_output_tokens_per_task": 3352.3703703703704, "last_run_timestamp": "2026-08-15T14:54:58.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.002187456692913386, "accuracy_std_dev": 0.0006963147187156109, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.0035705761154855645, "avg_duration_per_task_ms": 176293.88188976378, "avg_output_tokens_per_task": 3370.4238845144355, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.822917, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.0087319375, "avg_duration_per_task_ms": 31283.875, "avg_output_tokens_per_task": 9903.5625, "last_run_timestamp": "2026-09-26T08:05:12.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20567228, "avg_duration_per_task_ms": 69455.26, "avg_output_tokens_per_task": 15737.28, "last_run_timestamp": "2026-08-15T00:15:11.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.805556, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.07709047916666667, "avg_duration_per_task_ms": 121300.85416666667, "avg_output_tokens_per_task": 14694.6875, "last_run_timestamp": "2026-08-15T00:15:15.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.804166625, "accuracy_std_dev": 0.021065488167483215, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.8715094075, "avg_duration_per_task_ms": 137048.4925, "avg_output_tokens_per_task": 5786.43, "last_run_timestamp": "2026-09-26T08:21:11.883Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8, "accuracy_std_dev": 0.033110611481316166, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.7322859599999999, "avg_duration_per_task_ms": 146639.07333333333, "avg_output_tokens_per_task": 5472.34, "last_run_timestamp": "2026-08-15T08:09:14.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.7977776666666667, "accuracy_std_dev": 0.01911642045177097, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008363246666666668, "avg_duration_per_task_ms": 272879.14, "avg_output_tokens_per_task": 12625.026666666667, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.795833375, "accuracy_std_dev": 0.02846769733811245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.9313334725, "avg_duration_per_task_ms": 130292.9175, "avg_output_tokens_per_task": 5748.73, "last_run_timestamp": "2026-09-26T08:18:08.440Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.7933333333333333, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.6872690333333334, "avg_duration_per_task_ms": 114143.74666666667, "avg_output_tokens_per_task": 6925.7266666666665, "last_run_timestamp": "2026-09-26T08:22:07.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.793333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10986884, "avg_duration_per_task_ms": 261740.51, "avg_output_tokens_per_task": 11828.99, "last_run_timestamp": "2026-08-29T16:51:02.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7911111666666667, "accuracy_std_dev": 0.019116304205369344, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.8711063166666666, "avg_duration_per_task_ms": 151211.67333333334, "avg_output_tokens_per_task": 5696.93, "last_run_timestamp": "2026-09-26T08:09:55.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.790833375, "accuracy_std_dev": 0.01630862714897777, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.8205446474999999, "avg_duration_per_task_ms": 138181.955, "avg_output_tokens_per_task": 5731.4725, "last_run_timestamp": "2026-09-26T08:05:12.739Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.788889, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08521802666666665, "avg_duration_per_task_ms": 39816.48, "avg_output_tokens_per_task": 4684.933333333333, "last_run_timestamp": "2026-09-26T08:13:10.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.78833325, "accuracy_std_dev": 0.009860203404975974, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.8196274600000001, "avg_duration_per_task_ms": 183041.66, "avg_output_tokens_per_task": 7098.595, "last_run_timestamp": "2026-09-26T08:15:33.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.7883064274193549, "accuracy_std_dev": 0.013624850889459287, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.07264496370967742, "avg_duration_per_task_ms": 503105.06048387097, "avg_output_tokens_per_task": 9382.189516129032, "last_run_timestamp": "2026-08-01T17:45:43.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7877778666666666, "accuracy_std_dev": 0.02049995242552745, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.9492467306666665, "avg_duration_per_task_ms": 98954.73466666667, "avg_output_tokens_per_task": 5602.965333333334, "last_run_timestamp": "2026-09-26T08:12:28.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.7875, "accuracy_std_dev": 0.024593980747329203, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.48769549000000006, "avg_duration_per_task_ms": 129773.455, "avg_output_tokens_per_task": 7130.5275, "last_run_timestamp": "2026-09-26T08:22:26.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.7870992438478748, "accuracy_std_dev": 0.008959550928450943, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.11607393959731543, "avg_duration_per_task_ms": 143483.8120805369, "avg_output_tokens_per_task": 11288.324384787473, "last_run_timestamp": "2026-09-26T08:27:30.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "WandB", "accuracy": 0.786667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14780512, "avg_duration_per_task_ms": 461958.93, "avg_output_tokens_per_task": 11472.92, "last_run_timestamp": "2026-07-18T08:04:04.764Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.8471384599999998, "avg_duration_per_task_ms": 331793.46, "avg_output_tokens_per_task": 22741.94, "last_run_timestamp": "2026-08-01T08:06:01.602Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Morph", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.17120372, "avg_duration_per_task_ms": 466039.6, "avg_output_tokens_per_task": 22362.94, "last_run_timestamp": "2026-07-04T16:00:23.544Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7866669999999999, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2795196199999999, "avg_duration_per_task_ms": 104647.27, "avg_output_tokens_per_task": 5686.75, "last_run_timestamp": "2026-09-01T13:03:28.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.7866665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12536731, "avg_duration_per_task_ms": 118841.15, "avg_output_tokens_per_task": 19578.58, "last_run_timestamp": "2026-09-01T17:30:58.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.7859532642140468, "accuracy_std_dev": 0.022060806735148097, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.028781180602006688, "avg_duration_per_task_ms": 346700.6989966555, "avg_output_tokens_per_task": 11874.802675585284, "last_run_timestamp": "2026-08-29T17:06:21.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7857143728222997, "accuracy_std_dev": 0.022636034639976627, "run_count": 6, "total_questions": 287, "avg_cost_per_task_usd": 0.11757857142857142, "avg_duration_per_task_ms": 73705.68641114983, "avg_output_tokens_per_task": 8241.627177700348, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.784471297188755, "accuracy_std_dev": 0.02769278737577711, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.09832240562248996, "avg_duration_per_task_ms": 406639.9036144578, "avg_output_tokens_per_task": 17335.586345381525, "last_run_timestamp": "2026-09-26T08:23:09.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.7837037777777778, "accuracy_std_dev": 0.023117842088731067, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04375683111111111, "avg_duration_per_task_ms": 211952.00222222222, "avg_output_tokens_per_task": 9463.306666666667, "last_run_timestamp": "2026-08-29T17:11:32.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.7833334999999999, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02978465, "avg_duration_per_task_ms": 411185.87, "avg_output_tokens_per_task": 14644.16, "last_run_timestamp": "2026-07-18T08:03:40.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.7833334999999999, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06834454, "avg_duration_per_task_ms": 91367.71, "avg_output_tokens_per_task": 11815.81, "last_run_timestamp": "2026-07-18T08:04:03.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.78166675, "accuracy_std_dev": 0.018484295136344808, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1845917575, "avg_duration_per_task_ms": 205801.49, "avg_output_tokens_per_task": 9205.5075, "last_run_timestamp": "2026-08-29T16:12:45.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.7813334, "accuracy_std_dev": 0.022861647653657863, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.016082664, "avg_duration_per_task_ms": 178213.516, "avg_output_tokens_per_task": 11864.648, "last_run_timestamp": "2026-08-08T08:29:14.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.78041675, "accuracy_std_dev": 0.02409111033789228, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.4685391825, "avg_duration_per_task_ms": 111064.53375, "avg_output_tokens_per_task": 7216.75875, "last_run_timestamp": "2026-09-26T08:13:10.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.78000025, "accuracy_std_dev": 0.008164965810553043, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.7813667750000001, "avg_duration_per_task_ms": 113768.35, "avg_output_tokens_per_task": 6852.055, "last_run_timestamp": "2026-09-26T08:13:59.150Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.78, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.19113603999999998, "avg_duration_per_task_ms": 79138.93, "avg_output_tokens_per_task": 6244.06, "last_run_timestamp": "2026-09-29T12:00:07.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.78, "accuracy_std_dev": 0.033993485215716136, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04603017555555555, "avg_duration_per_task_ms": 205800.68, "avg_output_tokens_per_task": 9387.753333333334, "last_run_timestamp": "2026-08-29T16:11:46.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12014696000000001, "avg_duration_per_task_ms": 363719.42, "avg_output_tokens_per_task": 9566.28, "last_run_timestamp": "2026-07-18T16:00:09.009Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7788889166666667, "accuracy_std_dev": 0.02298676618897032, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.4834922383333334, "avg_duration_per_task_ms": 127667.31666666667, "avg_output_tokens_per_task": 7153.7075, "last_run_timestamp": "2026-09-26T08:12:13.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.7783335, "accuracy_std_dev": 0.028037782049406136, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.099593165, "avg_duration_per_task_ms": 81195.23, "avg_output_tokens_per_task": 4770.2, "last_run_timestamp": "2026-09-26T08:13:27.135Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.77833325, "accuracy_std_dev": 0.026404708125398767, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.807941035, "avg_duration_per_task_ms": 183353.72, "avg_output_tokens_per_task": 6938.485, "last_run_timestamp": "2026-09-26T08:11:40.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.77813332, "accuracy_std_dev": 0.02505255531991896, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.5623465103999999, "avg_duration_per_task_ms": 118279.7304, "avg_output_tokens_per_task": 7235.6968, "last_run_timestamp": "2026-09-26T08:16:10.311Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.777778, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.012714583333333333, "avg_duration_per_task_ms": 516568.7291666667, "avg_output_tokens_per_task": 14230.208333333334, "last_run_timestamp": "2026-08-03T17:20:47.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.7771428571428571, "accuracy_std_dev": 0.03204321345321288, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.027308874285714286, "avg_duration_per_task_ms": 297009.5657142857, "avg_output_tokens_per_task": 12274.965714285714, "last_run_timestamp": "2026-08-29T16:33:11.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.7770961820040899, "accuracy_std_dev": 0.01908827114120082, "run_count": 10, "total_questions": 489, "avg_cost_per_task_usd": 0.07789318609406953, "avg_duration_per_task_ms": 97210.93251533742, "avg_output_tokens_per_task": 4258.01226993865, "last_run_timestamp": "2026-09-26T08:11:43.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7770371111111111, "accuracy_std_dev": 0.02224700963397825, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4357714733333334, "avg_duration_per_task_ms": 133682.32888888888, "avg_output_tokens_per_task": 7264.266666666666, "last_run_timestamp": "2026-09-26T08:17:01.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.7766665, "accuracy_std_dev": 0.019148513142887433, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008114253333333333, "avg_duration_per_task_ms": 195047.99, "avg_output_tokens_per_task": 21425.716666666667, "last_run_timestamp": "2026-08-08T08:30:38.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.776, "accuracy_std_dev": 0.022939553421982742, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.007782092000000001, "avg_duration_per_task_ms": 211248.992, "avg_output_tokens_per_task": 4235.592, "last_run_timestamp": "2026-09-26T08:27:53.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.7755555000000001, "accuracy_std_dev": 0.0236225357991756, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.4783927666666667, "avg_duration_per_task_ms": 128906.20333333334, "avg_output_tokens_per_task": 7843.846666666666, "last_run_timestamp": "2026-09-26T08:22:36.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.7750001249999999, "accuracy_std_dev": 0.025331099134450842, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0651128975, "avg_duration_per_task_ms": 279584.37, "avg_output_tokens_per_task": 14735.76, "last_run_timestamp": "2026-08-29T17:42:47.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.775, "accuracy_std_dev": 0.029011578149938697, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09548250750000001, "avg_duration_per_task_ms": 233151.12, "avg_output_tokens_per_task": 11897.7625, "last_run_timestamp": "2026-08-29T16:59:11.598Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.7746666, "accuracy_std_dev": 0.02902873978387625, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.15693796000000002, "avg_duration_per_task_ms": 605970.636, "avg_output_tokens_per_task": 12427.38, "last_run_timestamp": "2026-09-26T08:23:03.870Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.7746666, "accuracy_std_dev": 0.022861927596770998, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.080879576, "avg_duration_per_task_ms": 126701.14, "avg_output_tokens_per_task": 4510.112, "last_run_timestamp": "2026-09-26T08:11:43.279Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.7746666, "accuracy_std_dev": 0.020395973461445754, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.120308296, "avg_duration_per_task_ms": 80105.144, "avg_output_tokens_per_task": 4494.34, "last_run_timestamp": "2026-09-26T08:09:25.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7746664000000001, "accuracy_std_dev": 0.016546172434735443, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.09735419199999999, "avg_duration_per_task_ms": 130067.476, "avg_output_tokens_per_task": 4561.928, "last_run_timestamp": "2026-09-26T08:10:40.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.7733333333333333, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.18828580666666667, "avg_duration_per_task_ms": 67452, "avg_output_tokens_per_task": 6335.053333333333, "last_run_timestamp": "2026-09-29T12:00:08.126Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7733333333333333, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02996874, "avg_duration_per_task_ms": 113373.20666666667, "avg_output_tokens_per_task": 12010.053333333333, "last_run_timestamp": "2026-08-29T16:46:02.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.7733332727272727, "accuracy_std_dev": 0.018422746153456893, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4337744636363636, "avg_duration_per_task_ms": 96869.80363636363, "avg_output_tokens_per_task": 5661.309090909091, "last_run_timestamp": "2026-09-26T08:11:40.823Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.7733332, "accuracy_std_dev": 0.004216264811417815, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.007032796, "avg_duration_per_task_ms": 161320.244, "avg_output_tokens_per_task": 16830.132, "last_run_timestamp": "2026-08-08T08:27:33.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.773333, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12487065999999998, "avg_duration_per_task_ms": 149585.85, "avg_output_tokens_per_task": 20477.48, "last_run_timestamp": "2026-09-01T17:31:27.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02012318, "avg_duration_per_task_ms": 229027.46, "avg_output_tokens_per_task": 10940.04, "last_run_timestamp": "2026-07-18T08:04:46.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06535734, "avg_duration_per_task_ms": 368327.08, "avg_output_tokens_per_task": 13147.48, "last_run_timestamp": "2026-09-23T15:00:14.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7726666, "accuracy_std_dev": 0.03641728405359191, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.113975304, "avg_duration_per_task_ms": 177276.2, "avg_output_tokens_per_task": 9278.056, "last_run_timestamp": "2026-08-29T16:15:12.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7725924444444445, "accuracy_std_dev": 0.024433234243142, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07833540444444444, "avg_duration_per_task_ms": 224489.25555555554, "avg_output_tokens_per_task": 9337.3, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.7725, "accuracy_std_dev": 0.030810865688909155, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.47031593, "avg_duration_per_task_ms": 117109.07, "avg_output_tokens_per_task": 7210.535, "last_run_timestamp": "2026-09-26T08:12:30.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7725, "accuracy_std_dev": 0.03302967521033168, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.22805288999999998, "avg_duration_per_task_ms": 411518.45, "avg_output_tokens_per_task": 13622.02, "last_run_timestamp": "2026-09-26T08:05:12.789Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7722223333333333, "accuracy_std_dev": 0.03890480468042762, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07840497666666667, "avg_duration_per_task_ms": 265427.81666666665, "avg_output_tokens_per_task": 11547.233333333334, "last_run_timestamp": "2026-09-26T08:13:10.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.77166675, "accuracy_std_dev": 0.023033966466666143, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.5286243174999999, "avg_duration_per_task_ms": 174531.3525, "avg_output_tokens_per_task": 10507.29, "last_run_timestamp": "2026-09-26T08:05:12.739Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.7715355505617978, "accuracy_std_dev": 0.03676049559271777, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.08241994157303371, "avg_duration_per_task_ms": 1113393.984269663, "avg_output_tokens_per_task": 27003.040449438202, "last_run_timestamp": "2026-09-26T08:24:45.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7715151818181818, "accuracy_std_dev": 0.029417231634840356, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5523802672727273, "avg_duration_per_task_ms": 183979.0109090909, "avg_output_tokens_per_task": 10526.685454545455, "last_run_timestamp": "2026-09-26T08:05:12.761Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7710438282828282, "accuracy_std_dev": 0.01667057037866429, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.1615302070707071, "avg_duration_per_task_ms": 338222.0707070707, "avg_output_tokens_per_task": 20443.252525252527, "last_run_timestamp": "2026-09-26T08:27:30.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.7708335, "accuracy_std_dev": 0.022345177208963923, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.17567465999999998, "avg_duration_per_task_ms": 650826.8375, "avg_output_tokens_per_task": 27562.8475, "last_run_timestamp": "2026-08-29T17:01:22.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.7706668, "accuracy_std_dev": 0.023702035544653124, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.074997112, "avg_duration_per_task_ms": 263018.86, "avg_output_tokens_per_task": 12780.056, "last_run_timestamp": "2026-09-26T08:10:10.216Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.7706668, "accuracy_std_dev": 0.01818435958069462, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.071001692, "avg_duration_per_task_ms": 34525.74, "avg_output_tokens_per_task": 4762.176, "last_run_timestamp": "2026-09-26T08:16:16.877Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.770303, "accuracy_std_dev": 0.018282610525752703, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4931648818181818, "avg_duration_per_task_ms": 129914.10363636364, "avg_output_tokens_per_task": 8068.774545454546, "last_run_timestamp": "2026-09-26T08:08:19.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.7702221333333333, "accuracy_std_dev": 0.02970127552786618, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.5012201893333333, "avg_duration_per_task_ms": 129604.79333333333, "avg_output_tokens_per_task": 7921.828, "last_run_timestamp": "2026-09-26T08:23:02.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.77, "accuracy_std_dev": 0.03341663757396906, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.4393913750000001, "avg_duration_per_task_ms": 81234.3425, "avg_output_tokens_per_task": 5692.71875, "last_run_timestamp": "2026-09-26T08:16:17.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.77, "accuracy_std_dev": 0.023332999999999993, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.35428131, "avg_duration_per_task_ms": 261243.27, "avg_output_tokens_per_task": 8792.41, "last_run_timestamp": "2026-07-18T08:04:09.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.77, "accuracy_std_dev": 0.007453485392754197, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.081881785, "avg_duration_per_task_ms": 99238.51, "avg_output_tokens_per_task": 4510.805, "last_run_timestamp": "2026-09-26T08:25:16.354Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.77, "accuracy_std_dev": 0.0033330000000000304, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.073975635, "avg_duration_per_task_ms": 45899.64, "avg_output_tokens_per_task": 4662.84, "last_run_timestamp": "2026-09-26T08:21:33.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.7688888888888888, "accuracy_std_dev": 0.01662949930056987, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.1780795511111111, "avg_duration_per_task_ms": 402300.5511111111, "avg_output_tokens_per_task": 27328.78888888889, "last_run_timestamp": "2026-08-29T16:25:47.585Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.7688888333333334, "accuracy_std_dev": 0.014229173651535601, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10475028333333332, "avg_duration_per_task_ms": 333223.94666666666, "avg_output_tokens_per_task": 20225.06333333333, "last_run_timestamp": "2026-09-26T08:21:56.720Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7688886666666668, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0562409, "avg_duration_per_task_ms": 124558.21333333333, "avg_output_tokens_per_task": 4508.253333333333, "last_run_timestamp": "2026-09-26T08:24:47.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.7687072040816327, "accuracy_std_dev": 0.0156945, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 1.7723489081632653, "avg_duration_per_task_ms": 313060.2448979592, "avg_output_tokens_per_task": 23077.85714285714, "last_run_timestamp": "2026-08-01T08:06:07.759Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.768707, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.18590030612244896, "avg_duration_per_task_ms": 362695.9183673469, "avg_output_tokens_per_task": 6540.102040816327, "last_run_timestamp": "2026-09-29T21:00:05.041Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.7684563154362417, "accuracy_std_dev": 0.006017520216464235, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1581116510067114, "avg_duration_per_task_ms": 537705.5637583892, "avg_output_tokens_per_task": 10979.71812080537, "last_run_timestamp": "2026-08-08T08:27:49.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.768421252631579, "accuracy_std_dev": 0.0038080000000000336, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.06421121052631579, "avg_duration_per_task_ms": 69460.44210526315, "avg_output_tokens_per_task": 5883.873684210526, "last_run_timestamp": "2026-07-18T08:06:26.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7683335, "accuracy_std_dev": 0.010929191289843909, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.09395701500000002, "avg_duration_per_task_ms": 284778.19, "avg_output_tokens_per_task": 21938.115, "last_run_timestamp": "2026-09-26T08:23:03.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.76833325, "accuracy_std_dev": 0.02048026828284971, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.4290680700000001, "avg_duration_per_task_ms": 101484.8275, "avg_output_tokens_per_task": 5562.66, "last_run_timestamp": "2026-09-26T08:11:43.281Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.76833325, "accuracy_std_dev": 0.02074978581064154, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.062534375, "avg_duration_per_task_ms": 51740.385, "avg_output_tokens_per_task": 4651.035, "last_run_timestamp": "2026-09-26T08:14:03.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.7675, "accuracy_std_dev": 0.038505122315089464, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11618410500000001, "avg_duration_per_task_ms": 295967.115, "avg_output_tokens_per_task": 12435.425, "last_run_timestamp": "2026-08-29T17:05:17.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.7671155218855219, "accuracy_std_dev": 0.020370521752577246, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.04689751515151515, "avg_duration_per_task_ms": 143803.91919191918, "avg_output_tokens_per_task": 4429.346801346801, "last_run_timestamp": "2026-09-26T08:24:40.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7671130089285715, "accuracy_std_dev": 0.016761300652263098, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.05441544419642857, "avg_duration_per_task_ms": 217468.82142857142, "avg_output_tokens_per_task": 9416.404017857143, "last_run_timestamp": "2026-08-29T16:11:58.048Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.18363575999999998, "avg_duration_per_task_ms": 79763.32, "avg_output_tokens_per_task": 6162.44, "last_run_timestamp": "2026-09-29T12:00:08.100Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.18092455999999998, "avg_duration_per_task_ms": 72930.62, "avg_output_tokens_per_task": 6117.32, "last_run_timestamp": "2026-09-29T12:00:06.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.025223939999999997, "avg_duration_per_task_ms": 165137.1, "avg_output_tokens_per_task": 14240.82, "last_run_timestamp": "2026-09-15T00:00:05.846Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.86423468, "avg_duration_per_task_ms": 335268.18, "avg_output_tokens_per_task": 22931.04, "last_run_timestamp": "2026-08-01T08:05:35.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0119651, "avg_duration_per_task_ms": 540203.32, "avg_output_tokens_per_task": 23288.26, "last_run_timestamp": "2026-09-29T03:00:05.982Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0912162, "avg_duration_per_task_ms": 191018.42, "avg_output_tokens_per_task": 9260.76, "last_run_timestamp": "2026-08-04T03:12:10.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.020366930350175255, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.1362619733333333, "avg_duration_per_task_ms": 318797.34, "avg_output_tokens_per_task": 19441.493333333332, "last_run_timestamp": "2026-09-26T08:05:12.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12935825333333334, "avg_duration_per_task_ms": 58122.44, "avg_output_tokens_per_task": 4752.84, "last_run_timestamp": "2026-09-19T08:15:57.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.766666625, "accuracy_std_dev": 0.025819792151262073, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.03997693, "avg_duration_per_task_ms": 243468.475, "avg_output_tokens_per_task": 12205.0675, "last_run_timestamp": "2026-08-29T16:25:15.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7666665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.21096532, "avg_duration_per_task_ms": 124417.44, "avg_output_tokens_per_task": 5140.16, "last_run_timestamp": "2026-09-01T12:59:37.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.7663316432160805, "accuracy_std_dev": 0.01611815722591683, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.08850016080402008, "avg_duration_per_task_ms": 191747.41206030152, "avg_output_tokens_per_task": 11737.070351758794, "last_run_timestamp": "2026-08-29T16:41:08.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.7661904285714286, "accuracy_std_dev": 0.01872337959311179, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.019808859999999998, "avg_duration_per_task_ms": 510174.85142857145, "avg_output_tokens_per_task": 13353.934285714286, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7659570000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.006887468085106384, "avg_duration_per_task_ms": 773525.340425532, "avg_output_tokens_per_task": 9139.936170212766, "last_run_timestamp": "2026-09-22T21:00:04.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.76583325, "accuracy_std_dev": 0.03239305266855072, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1614212075, "avg_duration_per_task_ms": 219712.16, "avg_output_tokens_per_task": 9777.065, "last_run_timestamp": "2026-08-29T17:16:15.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.7653059591836735, "accuracy_std_dev": 0.005416499999999991, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.02433298979591837, "avg_duration_per_task_ms": 682809.275510204, "avg_output_tokens_per_task": 13469.224489795919, "last_run_timestamp": "2026-08-01T08:05:58.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.765217347826087, "accuracy_std_dev": 0.020893702830421756, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.5270942373913046, "avg_duration_per_task_ms": 123051.67565217392, "avg_output_tokens_per_task": 8091.90347826087, "last_run_timestamp": "2026-09-26T08:26:39.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.7651820931174089, "accuracy_std_dev": 0.03553283679302851, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.09236648582995953, "avg_duration_per_task_ms": 316304.95546558703, "avg_output_tokens_per_task": 9844.623481781377, "last_run_timestamp": "2026-08-29T16:35:10.931Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7651515555555556, "accuracy_std_dev": 0.04160471364201867, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.02325603535353535, "avg_duration_per_task_ms": 506742.0202020202, "avg_output_tokens_per_task": 13406.39393939394, "last_run_timestamp": "2026-08-29T16:33:19.761Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7648808660714286, "accuracy_std_dev": 0.04130098452896405, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.1113698392857143, "avg_duration_per_task_ms": 925496.1897321428, "avg_output_tokens_per_task": 10117.198660714286, "last_run_timestamp": "2026-08-29T16:54:14.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7644446666666667, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13677892666666666, "avg_duration_per_task_ms": 241869.39333333334, "avg_output_tokens_per_task": 11667.466666666667, "last_run_timestamp": "2026-08-08T08:29:31.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.764166625, "accuracy_std_dev": 0.030081035231676705, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.56641688625, "avg_duration_per_task_ms": 181084.34625, "avg_output_tokens_per_task": 10579.67875, "last_run_timestamp": "2026-09-26T08:14:04.078Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.7639931779448622, "accuracy_std_dev": 0.027369852484074517, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.0707108245614035, "avg_duration_per_task_ms": 77146.8395989975, "avg_output_tokens_per_task": 13251.315789473685, "last_run_timestamp": "2026-08-29T17:41:16.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.7633335, "accuracy_std_dev": 0.015275361427802623, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.028111930000000004, "avg_duration_per_task_ms": 550867.43, "avg_output_tokens_per_task": 16398.445, "last_run_timestamp": "2026-08-08T08:29:47.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.7633335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.34846716, "avg_duration_per_task_ms": 136967.1, "avg_output_tokens_per_task": 12613.18, "last_run_timestamp": "2026-09-01T17:30:16.052Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.763333375, "accuracy_std_dev": 0.031091303715578986, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0139452675, "avg_duration_per_task_ms": 180694.13, "avg_output_tokens_per_task": 14861.8225, "last_run_timestamp": "2026-08-29T17:17:18.830Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.763333375, "accuracy_std_dev": 0.022607772755943365, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.040371300000000006, "avg_duration_per_task_ms": 175751.1025, "avg_output_tokens_per_task": 11405.795, "last_run_timestamp": "2026-08-29T17:07:19.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.763333375, "accuracy_std_dev": 0.02708005108441221, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1274939625, "avg_duration_per_task_ms": 269624.8775, "avg_output_tokens_per_task": 11653.1325, "last_run_timestamp": "2026-08-29T16:48:46.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.017743016363840187, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.018957181666666663, "avg_duration_per_task_ms": 239454.035, "avg_output_tokens_per_task": 17272.2, "last_run_timestamp": "2026-09-26T08:11:43.368Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.005773358354328977, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.039311875, "avg_duration_per_task_ms": 88152.73, "avg_output_tokens_per_task": 4482.07, "last_run_timestamp": "2026-09-26T08:07:18.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7633331666666667, "accuracy_std_dev": 0.02631724543853748, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.5436579066666667, "avg_duration_per_task_ms": 181008.31666666668, "avg_output_tokens_per_task": 7997.5633333333335, "last_run_timestamp": "2026-09-01T13:02:58.914Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.763333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10827190999999999, "avg_duration_per_task_ms": 51051.39, "avg_output_tokens_per_task": 12753.93, "last_run_timestamp": "2026-07-18T08:03:41.319Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.7626666, "accuracy_std_dev": 0.016110821761784848, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.09137509199999999, "avg_duration_per_task_ms": 332031.4, "avg_output_tokens_per_task": 19933.684, "last_run_timestamp": "2026-09-26T08:21:56.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7626666, "accuracy_std_dev": 0.03227647784749755, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.146569104, "avg_duration_per_task_ms": 373419.58, "avg_output_tokens_per_task": 24831.152, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.7626262929292928, "accuracy_std_dev": 0.02553140404849484, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.10055969696969695, "avg_duration_per_task_ms": 35430.38888888889, "avg_output_tokens_per_task": 3961.580808080808, "last_run_timestamp": "2026-09-26T08:30:51.947Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.025141417307348132, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08369306, "avg_duration_per_task_ms": 261571.12, "avg_output_tokens_per_task": 19541.486666666668, "last_run_timestamp": "2026-09-26T08:16:17.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.013698571612479268, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06797058666666667, "avg_duration_per_task_ms": 364277.74, "avg_output_tokens_per_task": 20919.446666666667, "last_run_timestamp": "2026-09-26T08:05:25.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.006285707880227596, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.1522076066666667, "avg_duration_per_task_ms": 298029.9066666667, "avg_output_tokens_per_task": 19805.7, "last_run_timestamp": "2026-09-26T08:05:12.730Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.013698571612479268, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09941037999999999, "avg_duration_per_task_ms": 137232.76666666666, "avg_output_tokens_per_task": 5643.006666666667, "last_run_timestamp": "2026-09-26T08:22:37.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.76222225, "accuracy_std_dev": 0.015234723173970049, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.016681795000000003, "avg_duration_per_task_ms": 202557.275, "avg_output_tokens_per_task": 19520.191666666666, "last_run_timestamp": "2026-09-26T08:22:26.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.7622221666666666, "accuracy_std_dev": 0.021659565788327558, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.026935736666666668, "avg_duration_per_task_ms": 147567.38333333333, "avg_output_tokens_per_task": 9119.013333333334, "last_run_timestamp": "2026-09-26T16:00:25.687Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.7620424450867053, "accuracy_std_dev": 0.05069435998456729, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.0817946936416185, "avg_duration_per_task_ms": 410762.72543352604, "avg_output_tokens_per_task": 12839.21387283237, "last_run_timestamp": "2026-08-29T17:06:46.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.7619999000000001, "accuracy_std_dev": 0.014313938818159046, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.008990316, "avg_duration_per_task_ms": 356282.446, "avg_output_tokens_per_task": 14140.9, "last_run_timestamp": "2026-09-26T08:22:54.418Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.7619998, "accuracy_std_dev": 0.02599996410305214, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.171845166, "avg_duration_per_task_ms": 631744.54, "avg_output_tokens_per_task": 40127.626, "last_run_timestamp": "2026-09-26T08:01:18.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.7619049795918367, "accuracy_std_dev": 0.01847199999999999, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.1408940612244898, "avg_duration_per_task_ms": 345711.98979591834, "avg_output_tokens_per_task": 8276.57142857143, "last_run_timestamp": "2026-08-01T08:06:00.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.7619047142857143, "accuracy_std_dev": 0.01942484091285683, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07395918571428572, "avg_duration_per_task_ms": 290755.1314285714, "avg_output_tokens_per_task": 14557.031428571428, "last_run_timestamp": "2026-08-29T17:02:01.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7619047142857143, "accuracy_std_dev": 0.02487158128532752, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03577183142857143, "avg_duration_per_task_ms": 462004.73142857145, "avg_output_tokens_per_task": 13726.342857142858, "last_run_timestamp": "2026-09-26T08:07:18.969Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.76166675, "accuracy_std_dev": 0.03312451676911681, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01217459, "avg_duration_per_task_ms": 159488.2175, "avg_output_tokens_per_task": 14638.645, "last_run_timestamp": "2026-08-29T16:53:19.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.76166675, "accuracy_std_dev": 0.024209737481796473, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.16434805749999998, "avg_duration_per_task_ms": 662935.715, "avg_output_tokens_per_task": 26007.7175, "last_run_timestamp": "2026-08-29T16:14:58.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.76166675, "accuracy_std_dev": 0.028431218171008805, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.051254930000000004, "avg_duration_per_task_ms": 163368.405, "avg_output_tokens_per_task": 4534.255, "last_run_timestamp": "2026-09-26T08:24:09.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.76166675, "accuracy_std_dev": 0.025110926131616508, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06906778, "avg_duration_per_task_ms": 159996.095, "avg_output_tokens_per_task": 6808.63, "last_run_timestamp": "2026-09-26T08:22:24.923Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.761666625, "accuracy_std_dev": 0.020749836011023693, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.4161910325, "avg_duration_per_task_ms": 98455.6425, "avg_output_tokens_per_task": 5629.3925, "last_run_timestamp": "2026-09-26T08:10:40.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.7614156780821918, "accuracy_std_dev": 0.0228848139730987, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.024887965753424657, "avg_duration_per_task_ms": 458119.4315068493, "avg_output_tokens_per_task": 13217.239726027397, "last_run_timestamp": "2026-08-29T16:59:36.100Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.7612120909090909, "accuracy_std_dev": 0.02825828791202695, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5180262836363636, "avg_duration_per_task_ms": 222126.0781818182, "avg_output_tokens_per_task": 10227.196363636363, "last_run_timestamp": "2026-09-26T08:22:25.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.7609524285714285, "accuracy_std_dev": 0.03369861083129663, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07208951428571428, "avg_duration_per_task_ms": 276051.22, "avg_output_tokens_per_task": 11448.085714285715, "last_run_timestamp": "2026-08-29T16:45:57.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.7609522857142857, "accuracy_std_dev": 0.017969598920035443, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08491382571428571, "avg_duration_per_task_ms": 105368.18285714286, "avg_output_tokens_per_task": 24364.49142857143, "last_run_timestamp": "2026-09-01T17:31:46.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.7609424343434344, "accuracy_std_dev": 0.012517, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.02651870707070707, "avg_duration_per_task_ms": 84229.25252525252, "avg_output_tokens_per_task": 17462.434343434343, "last_run_timestamp": "2026-09-26T08:10:56.841Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.760833625, "accuracy_std_dev": 0.018390020316312206, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.4986317424999999, "avg_duration_per_task_ms": 132839.435, "avg_output_tokens_per_task": 8114.2475, "last_run_timestamp": "2026-09-26T08:16:12.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7608334999999999, "accuracy_std_dev": 0.02119686166629392, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.16130322249999998, "avg_duration_per_task_ms": 442219.4275, "avg_output_tokens_per_task": 9533.43, "last_run_timestamp": "2026-08-29T16:32:10.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.7605556666666666, "accuracy_std_dev": 0.013391060222734018, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.12239992333333333, "avg_duration_per_task_ms": 458072.06, "avg_output_tokens_per_task": 22242.516666666666, "last_run_timestamp": "2026-09-26T08:22:24.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.7604689296482412, "accuracy_std_dev": 0.02230978235970715, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.10629371356783922, "avg_duration_per_task_ms": 377132.17587939696, "avg_output_tokens_per_task": 11770.236180904523, "last_run_timestamp": "2026-08-29T16:33:36.999Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.7604167291666667, "accuracy_std_dev": 0.017826499999999967, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.2660930625, "avg_duration_per_task_ms": 146365.54166666666, "avg_output_tokens_per_task": 6163.291666666667, "last_run_timestamp": "2026-09-26T08:05:12.774Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.7603703333333334, "accuracy_std_dev": 0.028822460188771784, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.056350817777777784, "avg_duration_per_task_ms": 98825.95333333334, "avg_output_tokens_per_task": 4242.228888888889, "last_run_timestamp": "2026-09-26T08:32:07.081Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.760080620967742, "accuracy_std_dev": 0.02198073466379138, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.03671067338709678, "avg_duration_per_task_ms": 531209.4012096775, "avg_output_tokens_per_task": 12968.175403225807, "last_run_timestamp": "2026-08-29T17:21:09.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01747962, "avg_duration_per_task_ms": 589736.28, "avg_output_tokens_per_task": 13675.16, "last_run_timestamp": "2026-08-01T08:06:01.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.76, "accuracy_std_dev": 0.034318637633216124, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03080999, "avg_duration_per_task_ms": 262760.8, "avg_output_tokens_per_task": 18663.095, "last_run_timestamp": "2026-09-26T08:09:25.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05756704, "avg_duration_per_task_ms": 256268.94, "avg_output_tokens_per_task": 10406.48, "last_run_timestamp": "2026-09-29T02:27:33.734Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.76, "accuracy_std_dev": 0.033333, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11355728, "avg_duration_per_task_ms": 147828.6, "avg_output_tokens_per_task": 17720.73, "last_run_timestamp": "2026-09-01T17:30:58.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Venice", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19103682, "avg_duration_per_task_ms": 233230.24, "avg_output_tokens_per_task": 15236.66, "last_run_timestamp": "2026-08-04T03:12:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.76, "accuracy_std_dev": 0.020727419665747104, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.0731778, "avg_duration_per_task_ms": 258357.41, "avg_output_tokens_per_task": 11852.263333333334, "last_run_timestamp": "2026-09-26T08:07:46.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.76, "accuracy_std_dev": 0.038682027601113875, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07101471333333334, "avg_duration_per_task_ms": 318099.35333333333, "avg_output_tokens_per_task": 12493.926666666666, "last_run_timestamp": "2026-09-26T08:05:12.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.35068044, "avg_duration_per_task_ms": 248200.34, "avg_output_tokens_per_task": 11571.02, "last_run_timestamp": "2026-09-01T12:41:31.788Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.76, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12663513999999998, "avg_duration_per_task_ms": 73292.09, "avg_output_tokens_per_task": 9837.08, "last_run_timestamp": "2026-07-18T08:04:45.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.027619090000000002, "avg_duration_per_task_ms": 42870.03, "avg_output_tokens_per_task": 4628.08, "last_run_timestamp": "2026-09-26T09:28:27.720Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.76, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04223252, "avg_duration_per_task_ms": 69942.84, "avg_output_tokens_per_task": 4621.51, "last_run_timestamp": "2026-09-26T08:05:12.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.76, "accuracy_std_dev": 0.027371610829992666, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.009037317142857142, "avg_duration_per_task_ms": 270209.39714285714, "avg_output_tokens_per_task": 6316.0257142857145, "last_run_timestamp": "2026-09-26T08:08:19.666Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.76, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014824000000000002, "avg_duration_per_task_ms": 335920.77, "avg_output_tokens_per_task": 4448.51, "last_run_timestamp": "2026-09-12T08:11:26.740Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.759166625, "accuracy_std_dev": 0.02319920826201565, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0076381949999999995, "avg_duration_per_task_ms": 261034.725, "avg_output_tokens_per_task": 15531.6525, "last_run_timestamp": "2026-09-26T08:24:41.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.758888875, "accuracy_std_dev": 0.0209643399476995, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.6800024841666666, "avg_duration_per_task_ms": 74777.63666666667, "avg_output_tokens_per_task": 5635.594166666667, "last_run_timestamp": "2026-09-26T08:23:22.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.7588888333333333, "accuracy_std_dev": 0.03578052612067383, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.4306518199999999, "avg_duration_per_task_ms": 94862.28333333334, "avg_output_tokens_per_task": 5554.29, "last_run_timestamp": "2026-09-26T08:25:11.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7588406086956522, "accuracy_std_dev": 0.021414040391536566, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.59698552, "avg_duration_per_task_ms": 123337.09304347826, "avg_output_tokens_per_task": 8161.509565217391, "last_run_timestamp": "2026-09-26T08:22:57.486Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7586668, "accuracy_std_dev": 0.01654634971708262, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.27091952399999997, "avg_duration_per_task_ms": 154225.472, "avg_output_tokens_per_task": 23085.08, "last_run_timestamp": "2026-09-01T17:32:40.487Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7586668, "accuracy_std_dev": 0.012927091279943853, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08354813200000001, "avg_duration_per_task_ms": 248803.664, "avg_output_tokens_per_task": 12064.656, "last_run_timestamp": "2026-09-26T08:08:19.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.7585185555555556, "accuracy_std_dev": 0.01743499783711622, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07257207555555555, "avg_duration_per_task_ms": 58870.41333333333, "avg_output_tokens_per_task": 4565.811111111111, "last_run_timestamp": "2026-09-26T08:26:14.308Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.015184156584660873, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.163021, "avg_duration_per_task_ms": 227043.59, "avg_output_tokens_per_task": 11333.075, "last_run_timestamp": "2026-08-29T16:55:33.128Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.022298623526744884, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025212460000000003, "avg_duration_per_task_ms": 129050.295, "avg_output_tokens_per_task": 16100.67, "last_run_timestamp": "2026-09-26T08:22:17.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7583331666666667, "accuracy_std_dev": 0.013844473427047428, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09591996333333334, "avg_duration_per_task_ms": 214673.9, "avg_output_tokens_per_task": 9689.54, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.758, "accuracy_std_dev": 0.03709447398198281, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.079319272, "avg_duration_per_task_ms": 620750.224, "avg_output_tokens_per_task": 10895.356, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.7579964141414142, "accuracy_std_dev": 0.023533015514113357, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.20713992929292932, "avg_duration_per_task_ms": 664979.7727272727, "avg_output_tokens_per_task": 53099.67929292929, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7578125491071429, "accuracy_std_dev": 0.037360230019845994, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.18927937723214286, "avg_duration_per_task_ms": 471319.05133928574, "avg_output_tokens_per_task": 27760.915178571428, "last_run_timestamp": "2026-09-26T08:12:00.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.7577778333333334, "accuracy_std_dev": 0.037646776132079215, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08977579333333333, "avg_duration_per_task_ms": 61527.54, "avg_output_tokens_per_task": 12239.113333333333, "last_run_timestamp": "2026-08-29T16:25:14.949Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.0342828683423336, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08650762222222223, "avg_duration_per_task_ms": 226626.72, "avg_output_tokens_per_task": 9806.055555555555, "last_run_timestamp": "2026-08-29T16:32:40.387Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.7576270508474576, "accuracy_std_dev": 0.017540967140446465, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.11473522711864406, "avg_duration_per_task_ms": 198579.0440677966, "avg_output_tokens_per_task": 7218.661016949152, "last_run_timestamp": "2026-08-15T08:10:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.7575758636363636, "accuracy_std_dev": 0.02000269988924933, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.09427858838383839, "avg_duration_per_task_ms": 374504.8080808081, "avg_output_tokens_per_task": 13062.229797979799, "last_run_timestamp": "2026-08-29T16:33:12.146Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7575, "accuracy_std_dev": 0.023318483371565994, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.34416930749999997, "avg_duration_per_task_ms": 221562.1725, "avg_output_tokens_per_task": 11914.215, "last_run_timestamp": "2026-09-26T08:22:18.965Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.7573335999999999, "accuracy_std_dev": 0.017178865569064807, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.151421216, "avg_duration_per_task_ms": 315167.644, "avg_output_tokens_per_task": 20253, "last_run_timestamp": "2026-09-26T08:10:25.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.031439640936880967, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.081021904, "avg_duration_per_task_ms": 42035.076, "avg_output_tokens_per_task": 4841.128, "last_run_timestamp": "2026-09-26T08:26:16.831Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7573099970760234, "accuracy_std_dev": 0.047450719617777116, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.10973368713450293, "avg_duration_per_task_ms": 2695204.277777778, "avg_output_tokens_per_task": 24270.733918128655, "last_run_timestamp": "2026-08-29T16:45:34.216Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.7572254739884392, "accuracy_std_dev": 0.020048765907361054, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.014727791907514451, "avg_duration_per_task_ms": 418785.12427745666, "avg_output_tokens_per_task": 15970.300578034683, "last_run_timestamp": "2026-08-29T16:29:07.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.756944, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.2442584375, "avg_duration_per_task_ms": 645873.5208333334, "avg_output_tokens_per_task": 12350.729166666666, "last_run_timestamp": "2026-09-26T08:23:04.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7568027346938776, "accuracy_std_dev": 0.038809665680729605, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.3020268469387756, "avg_duration_per_task_ms": 145185.1581632653, "avg_output_tokens_per_task": 20600.117346938776, "last_run_timestamp": "2026-09-01T17:31:47.860Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7567114127516779, "accuracy_std_dev": 0.02249334300085622, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.020425387583892618, "avg_duration_per_task_ms": 519535.90939597314, "avg_output_tokens_per_task": 17284.703020134228, "last_run_timestamp": "2026-09-26T08:07:18.967Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01828589, "avg_duration_per_task_ms": 240722.54, "avg_output_tokens_per_task": 15147.8, "last_run_timestamp": "2026-09-19T08:02:39.158Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.7566669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05416404, "avg_duration_per_task_ms": 1008004.72, "avg_output_tokens_per_task": 38151.6, "last_run_timestamp": "2026-09-26T00:00:04.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.7566667499999999, "accuracy_std_dev": 0.013743584364622665, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.33711718500000004, "avg_duration_per_task_ms": 415294.135, "avg_output_tokens_per_task": 10214.705, "last_run_timestamp": "2026-09-26T08:13:27.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7566665, "accuracy_std_dev": 0.02687425451214104, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.54838445, "avg_duration_per_task_ms": 178864.39333333334, "avg_output_tokens_per_task": 10795.433333333332, "last_run_timestamp": "2026-09-26T08:26:47.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.7566665, "accuracy_std_dev": 0.033720051184965114, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.010374689999999999, "avg_duration_per_task_ms": 242049.23333333334, "avg_output_tokens_per_task": 15360.75, "last_run_timestamp": "2026-08-29T16:54:19.718Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7566665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10920926999999998, "avg_duration_per_task_ms": 101251.93, "avg_output_tokens_per_task": 25100.99, "last_run_timestamp": "2026-09-01T17:23:16.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.7566665, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.42697405000000005, "avg_duration_per_task_ms": 522179.69, "avg_output_tokens_per_task": 37590.96, "last_run_timestamp": "2026-09-01T12:41:24.474Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.7566665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29550982, "avg_duration_per_task_ms": 152697.48, "avg_output_tokens_per_task": 10806.55, "last_run_timestamp": "2026-09-01T17:30:15.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.756, "accuracy_std_dev": 0.012365076271499492, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.0694329, "avg_duration_per_task_ms": 122849.852, "avg_output_tokens_per_task": 4849.072, "last_run_timestamp": "2026-09-26T08:26:14.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7559523794642857, "accuracy_std_dev": 0.05190211941817757, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.07933722767857142, "avg_duration_per_task_ms": 255983.8861607143, "avg_output_tokens_per_task": 9718.457589285714, "last_run_timestamp": "2026-08-29T16:45:34.327Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.7555556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05382074666666667, "avg_duration_per_task_ms": 90716.92666666667, "avg_output_tokens_per_task": 14804.206666666667, "last_run_timestamp": "2026-09-26T08:22:43.308Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.7555556666666666, "accuracy_std_dev": 0.03823241466202323, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0327312, "avg_duration_per_task_ms": 153585.12, "avg_output_tokens_per_task": 15537.24, "last_run_timestamp": "2026-09-26T08:17:02.789Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.7555556666666666, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05866996, "avg_duration_per_task_ms": 356562.5266666667, "avg_output_tokens_per_task": 13311.113333333333, "last_run_timestamp": "2026-09-26T08:12:56.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.7555556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06505471333333333, "avg_duration_per_task_ms": 157291.16, "avg_output_tokens_per_task": 14450.066666666668, "last_run_timestamp": "2026-09-26T08:05:12.747Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.7554439597989949, "accuracy_std_dev": 0.00981512802705599, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.047420793969849245, "avg_duration_per_task_ms": 374713.7035175879, "avg_output_tokens_per_task": 7841.51256281407, "last_run_timestamp": "2026-07-25T08:07:26.285Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7553623043478261, "accuracy_std_dev": 0.032979826821380405, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.19905160782608697, "avg_duration_per_task_ms": 122704.35391304348, "avg_output_tokens_per_task": 7790.118260869565, "last_run_timestamp": "2026-09-26T08:11:40.828Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.7552381428571429, "accuracy_std_dev": 0.009897295726302106, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03504835714285714, "avg_duration_per_task_ms": 271800.3542857143, "avg_output_tokens_per_task": 13667.517142857143, "last_run_timestamp": "2026-09-26T08:10:40.746Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7551515454545454, "accuracy_std_dev": 0.023968159933786534, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.024316309090909087, "avg_duration_per_task_ms": 165689.23272727273, "avg_output_tokens_per_task": 18155.507272727275, "last_run_timestamp": "2026-09-26T08:08:49.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.755, "accuracy_std_dev": 0.02279120200866991, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.346774905, "avg_duration_per_task_ms": 239775.925, "avg_output_tokens_per_task": 12209.34, "last_run_timestamp": "2026-09-29T03:00:06.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.755, "accuracy_std_dev": 0.017240198505237704, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.055826839999999996, "avg_duration_per_task_ms": 122326.96, "avg_output_tokens_per_task": 4308.39, "last_run_timestamp": "2026-09-26T08:26:39.685Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.7546666, "accuracy_std_dev": 0.034615490431886144, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.201248028, "avg_duration_per_task_ms": 612127.586, "avg_output_tokens_per_task": 28603.458, "last_run_timestamp": "2026-09-26T08:13:27.139Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7546664000000001, "accuracy_std_dev": 0.04509740371950474, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.17806932799999997, "avg_duration_per_task_ms": 249010.144, "avg_output_tokens_per_task": 9041.408, "last_run_timestamp": "2026-09-26T08:26:47.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7544445, "accuracy_std_dev": 0.025939233358948773, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.19473402666666667, "avg_duration_per_task_ms": 127608.55, "avg_output_tokens_per_task": 7689.33, "last_run_timestamp": "2026-09-26T08:22:38.150Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7543103448275862, "accuracy_std_dev": 0.016399321098441732, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.04285599137931035, "avg_duration_per_task_ms": 524163.9051724138, "avg_output_tokens_per_task": 22101.97988505747, "last_run_timestamp": "2026-09-26T16:00:25.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.7542953608247424, "accuracy_std_dev": 0.02652500000000002, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.021299783505154636, "avg_duration_per_task_ms": 633495.5257731959, "avg_output_tokens_per_task": 26267.113402061856, "last_run_timestamp": "2026-08-08T08:30:53.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.7542857142857143, "accuracy_std_dev": 0.02838017896205673, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07299003142857143, "avg_duration_per_task_ms": 266110.34571428574, "avg_output_tokens_per_task": 9872.694285714286, "last_run_timestamp": "2026-08-29T17:22:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.754166625, "accuracy_std_dev": 0.02895150279319496, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1985393725, "avg_duration_per_task_ms": 132141.125, "avg_output_tokens_per_task": 7833.895, "last_run_timestamp": "2026-09-26T08:08:19.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.754, "accuracy_std_dev": 0.02048284919633984, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08895154, "avg_duration_per_task_ms": 301180.94, "avg_output_tokens_per_task": 9633.452, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7539393636363636, "accuracy_std_dev": 0.01999089692049736, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.024043370909090908, "avg_duration_per_task_ms": 299110.4909090909, "avg_output_tokens_per_task": 17382.84909090909, "last_run_timestamp": "2026-09-26T08:12:56.930Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.7539393636363636, "accuracy_std_dev": 0.014342041050845328, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.05402639454545455, "avg_duration_per_task_ms": 296059.4763636364, "avg_output_tokens_per_task": 31833.65090909091, "last_run_timestamp": "2026-09-26T16:00:20.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.7538536705202312, "accuracy_std_dev": 0.022320967608497434, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.20567977745664742, "avg_duration_per_task_ms": 493851.28612716764, "avg_output_tokens_per_task": 20258.49710982659, "last_run_timestamp": "2026-09-26T08:08:00.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.7533668080808081, "accuracy_std_dev": 0.00967513458239728, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.0209760505050505, "avg_duration_per_task_ms": 99025.76767676767, "avg_output_tokens_per_task": 14135.353535353535, "last_run_timestamp": "2026-09-26T08:05:09.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7533630986547086, "accuracy_std_dev": 0.021783295924476934, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.07844548430493273, "avg_duration_per_task_ms": 256047.19506726458, "avg_output_tokens_per_task": 9877.659192825113, "last_run_timestamp": "2026-08-29T16:07:02.345Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5044717400000001, "avg_duration_per_task_ms": 153665.59, "avg_output_tokens_per_task": 8278.2, "last_run_timestamp": "2026-09-01T12:41:31.807Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.014142371326973414, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.19823152, "avg_duration_per_task_ms": 359205.05, "avg_output_tokens_per_task": 12583.985, "last_run_timestamp": "2026-09-26T08:10:43.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.7533333999999999, "accuracy_std_dev": 0.01738441192102859, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.019357872, "avg_duration_per_task_ms": 107397.768, "avg_output_tokens_per_task": 16129.496, "last_run_timestamp": "2026-09-26T08:24:48.615Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.7533333999999999, "accuracy_std_dev": 0.0193218356643462, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.037210308, "avg_duration_per_task_ms": 202748.952, "avg_output_tokens_per_task": 17567.136, "last_run_timestamp": "2026-09-26T08:23:54.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021082753333333332, "avg_duration_per_task_ms": 915986.2133333334, "avg_output_tokens_per_task": 13268.026666666667, "last_run_timestamp": "2026-08-15T08:15:52.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.7533332857142857, "accuracy_std_dev": 0.027371645624698595, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.022015817142857143, "avg_duration_per_task_ms": 446424.36, "avg_output_tokens_per_task": 15560.805714285714, "last_run_timestamp": "2026-08-29T16:47:28.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.7533331428571429, "accuracy_std_dev": 0.023904532346270185, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11555654857142857, "avg_duration_per_task_ms": 316732.6342857143, "avg_output_tokens_per_task": 9759.925714285715, "last_run_timestamp": "2026-08-29T16:44:49.542Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.753333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.36418883999999996, "avg_duration_per_task_ms": 138377.24, "avg_output_tokens_per_task": 12769.81, "last_run_timestamp": "2026-09-01T17:31:47.883Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.41880896, "avg_duration_per_task_ms": 447303.41, "avg_output_tokens_per_task": 36551.01, "last_run_timestamp": "2026-09-01T12:59:50.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.6857741799999999, "avg_duration_per_task_ms": 105619.36, "avg_output_tokens_per_task": 7597.08, "last_run_timestamp": "2026-09-01T13:02:16.941Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0196027, "avg_duration_per_task_ms": 76385.02, "avg_output_tokens_per_task": 9950, "last_run_timestamp": "2026-07-18T08:04:02.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10139794, "avg_duration_per_task_ms": 119629.34, "avg_output_tokens_per_task": 9506.02, "last_run_timestamp": "2026-08-01T17:46:04.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09723594, "avg_duration_per_task_ms": 178772.1, "avg_output_tokens_per_task": 8310.58, "last_run_timestamp": "2026-08-01T08:05:58.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7529762410714286, "accuracy_std_dev": 0.03645787970917796, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.10114379464285715, "avg_duration_per_task_ms": 414862.9285714286, "avg_output_tokens_per_task": 9685.97767857143, "last_run_timestamp": "2026-08-29T16:31:41.307Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.752923926900585, "accuracy_std_dev": 0.02558177145020674, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.006783909356725145, "avg_duration_per_task_ms": 266624.59064327483, "avg_output_tokens_per_task": 11932.982456140351, "last_run_timestamp": "2026-08-29T16:33:29.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7525925555555556, "accuracy_std_dev": 0.02463451135257706, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021416855555555555, "avg_duration_per_task_ms": 123014.89111111111, "avg_output_tokens_per_task": 15302.471111111112, "last_run_timestamp": "2026-09-26T08:21:23.410Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.752499875, "accuracy_std_dev": 0.04128993437400179, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.19582053000000002, "avg_duration_per_task_ms": 106281.8925, "avg_output_tokens_per_task": 7612.88, "last_run_timestamp": "2026-09-26T08:23:00.199Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.7524445333333333, "accuracy_std_dev": 0.03730686310384309, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.19597836800000001, "avg_duration_per_task_ms": 104969.73733333334, "avg_output_tokens_per_task": 7646.229333333334, "last_run_timestamp": "2026-09-26T08:12:13.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.752, "accuracy_std_dev": 0.015434478002187193, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.036405735999999994, "avg_duration_per_task_ms": 202702.796, "avg_output_tokens_per_task": 18338.232, "last_run_timestamp": "2026-09-26T08:01:18.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.7516667499999999, "accuracy_std_dev": 0.009860203404975986, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07850260499999999, "avg_duration_per_task_ms": 143951.215, "avg_output_tokens_per_task": 4587.75, "last_run_timestamp": "2026-09-26T08:22:24.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.7516667499999999, "accuracy_std_dev": 0.0172401743375031, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.14700563, "avg_duration_per_task_ms": 224982.975, "avg_output_tokens_per_task": 8550.835, "last_run_timestamp": "2026-09-26T08:08:19.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.751666625, "accuracy_std_dev": 0.02598077013358873, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.54754274, "avg_duration_per_task_ms": 191361.1975, "avg_output_tokens_per_task": 10587.185, "last_run_timestamp": "2026-09-26T08:16:15.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.7516665, "accuracy_std_dev": 0.03378863221188454, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025036024999999996, "avg_duration_per_task_ms": 332301.835, "avg_output_tokens_per_task": 13337.575, "last_run_timestamp": "2026-08-29T17:19:05.764Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7516665, "accuracy_std_dev": 0.03484088434942489, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017483123333333336, "avg_duration_per_task_ms": 78588.8, "avg_output_tokens_per_task": 4208.683333333333, "last_run_timestamp": "2026-09-26T08:25:00.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.7509090909090909, "accuracy_std_dev": 0.04884723887507879, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08554166363636365, "avg_duration_per_task_ms": 195446.1018181818, "avg_output_tokens_per_task": 7986.087272727273, "last_run_timestamp": "2026-09-26T08:09:25.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.75083325, "accuracy_std_dev": 0.025152283418956218, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.014382082499999999, "avg_duration_per_task_ms": 226130.345, "avg_output_tokens_per_task": 15484.9525, "last_run_timestamp": "2026-08-29T16:48:14.107Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.7506668, "accuracy_std_dev": 0.027840878020637227, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08836137599999999, "avg_duration_per_task_ms": 290555.676, "avg_output_tokens_per_task": 12011.224, "last_run_timestamp": "2026-09-26T08:10:43.592Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.7506666, "accuracy_std_dev": 0.029090015088342618, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.008165608, "avg_duration_per_task_ms": 191715.892, "avg_output_tokens_per_task": 4404.68, "last_run_timestamp": "2026-09-26T08:23:40.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7506265313283209, "accuracy_std_dev": 0.029800800965536136, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.06770081704260651, "avg_duration_per_task_ms": 350619.045112782, "avg_output_tokens_per_task": 10643.185463659147, "last_run_timestamp": "2026-08-29T17:16:16.996Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.7505555, "accuracy_std_dev": 0.03274012621004996, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02124632666666667, "avg_duration_per_task_ms": 354797.86666666664, "avg_output_tokens_per_task": 18494.39, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.750303, "accuracy_std_dev": 0.04506919157755715, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11684560363636365, "avg_duration_per_task_ms": 153202.71454545454, "avg_output_tokens_per_task": 8545.56, "last_run_timestamp": "2026-09-26T08:22:30.723Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.7500002567567566, "accuracy_std_dev": 0.007287030686241284, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.009324743243243245, "avg_duration_per_task_ms": 215748.13513513515, "avg_output_tokens_per_task": 3777.3783783783783, "last_run_timestamp": "2026-09-26T08:06:43.931Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2253275, "avg_duration_per_task_ms": 98359.26, "avg_output_tokens_per_task": 5434.86, "last_run_timestamp": "2026-09-01T13:03:25.552Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.75, "accuracy_std_dev": 0.027688836500474354, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.5338496725, "avg_duration_per_task_ms": 189310.245, "avg_output_tokens_per_task": 10232.6975, "last_run_timestamp": "2026-09-26T08:09:25.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.75, "accuracy_std_dev": 0.03766314634582015, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04774656, "avg_duration_per_task_ms": 476512.7733333333, "avg_output_tokens_per_task": 9148.09, "last_run_timestamp": "2026-08-29T16:10:33.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.75, "accuracy_std_dev": 0.020275820303997587, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0059813, "avg_duration_per_task_ms": 146386.56, "avg_output_tokens_per_task": 4218.225, "last_run_timestamp": "2026-09-26T08:22:40.372Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.7499999583333333, "accuracy_std_dev": 0.0368179040090995, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.2266885883333333, "avg_duration_per_task_ms": 100830.78416666666, "avg_output_tokens_per_task": 7512.6875, "last_run_timestamp": "2026-09-26T08:24:10.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.7496296666666666, "accuracy_std_dev": 0.01780858133971749, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021206060000000002, "avg_duration_per_task_ms": 122947.83777777778, "avg_output_tokens_per_task": 16271.093333333334, "last_run_timestamp": "2026-09-26T08:08:19.683Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.7493332, "accuracy_std_dev": 0.01162353169049754, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.172436168, "avg_duration_per_task_ms": 308861.672, "avg_output_tokens_per_task": 10652.584, "last_run_timestamp": "2026-09-26T08:22:58.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.748889, "accuracy_std_dev": 0.01257094435593444, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05290565333333333, "avg_duration_per_task_ms": 112293.75333333333, "avg_output_tokens_per_task": 4801.486666666667, "last_run_timestamp": "2026-09-26T08:17:02.746Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.748889, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01121336, "avg_duration_per_task_ms": 201242.51333333334, "avg_output_tokens_per_task": 4454.833333333333, "last_run_timestamp": "2026-09-26T08:24:26.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7488888888888889, "accuracy_std_dev": 0.06409870860277306, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07423276, "avg_duration_per_task_ms": 214855.35777777777, "avg_output_tokens_per_task": 9268.002222222221, "last_run_timestamp": "2026-08-29T16:58:32.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7485715714285714, "accuracy_std_dev": 0.030386743739697415, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.27195099142857143, "avg_duration_per_task_ms": 529335.9085714286, "avg_output_tokens_per_task": 9553.417142857143, "last_run_timestamp": "2026-08-29T16:51:55.567Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.748333375, "accuracy_std_dev": 0.027233626580651628, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.488940975, "avg_duration_per_task_ms": 194699.49, "avg_output_tokens_per_task": 10000.255, "last_run_timestamp": "2026-09-26T08:11:43.276Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.7483332500000001, "accuracy_std_dev": 0.012802094015726486, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17867437000000003, "avg_duration_per_task_ms": 156388.555, "avg_output_tokens_per_task": 12772.235, "last_run_timestamp": "2026-09-26T08:22:52.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "ModelRun", "accuracy": 0.7482992653061223, "accuracy_std_dev": 0.011944500000000025, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.033130377551020405, "avg_duration_per_task_ms": 119867.04081632652, "avg_output_tokens_per_task": 8697.826530612245, "last_run_timestamp": "2026-07-11T08:05:22.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.7481202957393484, "accuracy_std_dev": 0.01771080940943976, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.01360957142857143, "avg_duration_per_task_ms": 218299.9649122807, "avg_output_tokens_per_task": 16333.967418546366, "last_run_timestamp": "2026-09-26T08:20:57.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.748116, "accuracy_std_dev": 0.022837863998193896, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.37004660695652175, "avg_duration_per_task_ms": 167935.71652173914, "avg_output_tokens_per_task": 11884.72, "last_run_timestamp": "2026-09-26T08:22:24.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7480843850574713, "accuracy_std_dev": 0.025275715672864952, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.006891692528735631, "avg_duration_per_task_ms": 300685.9626436782, "avg_output_tokens_per_task": 13202.612068965518, "last_run_timestamp": "2026-08-29T17:00:01.341Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.7480002, "accuracy_std_dev": 0.014236085605249795, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.14549154800000003, "avg_duration_per_task_ms": 139795.724, "avg_output_tokens_per_task": 19960.988, "last_run_timestamp": "2026-09-26T08:09:25.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.748, "accuracy_std_dev": 0.015434218839967281, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011429584, "avg_duration_per_task_ms": 130555.472, "avg_output_tokens_per_task": 4185.616, "last_run_timestamp": "2026-09-26T08:24:45.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7479998000000001, "accuracy_std_dev": 0.023247453386554, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.025147304000000002, "avg_duration_per_task_ms": 339787.244, "avg_output_tokens_per_task": 4397.436, "last_run_timestamp": "2026-09-26T08:13:28.807Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.747916625, "accuracy_std_dev": 0.029459786032986997, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.56374661625, "avg_duration_per_task_ms": 203094.54, "avg_output_tokens_per_task": 9850.0775, "last_run_timestamp": "2026-09-26T08:23:07.274Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.7478787272727273, "accuracy_std_dev": 0.029242020409898036, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.33650350909090904, "avg_duration_per_task_ms": 209916.4818181818, "avg_output_tokens_per_task": 11777.303636363637, "last_run_timestamp": "2026-09-26T08:09:55.575Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.7477778333333334, "accuracy_std_dev": 0.0267822656461116, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07834820999999999, "avg_duration_per_task_ms": 157847.65, "avg_output_tokens_per_task": 8613.366666666667, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.7477703913043479, "accuracy_std_dev": 0.01921375665647317, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.0067899197324414705, "avg_duration_per_task_ms": 182508.58026755851, "avg_output_tokens_per_task": 12210.97491638796, "last_run_timestamp": "2026-09-26T08:06:10.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.7475668503649635, "accuracy_std_dev": 0.021546232673712428, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.04062066605839416, "avg_duration_per_task_ms": 299491.20802919706, "avg_output_tokens_per_task": 26277.279197080294, "last_run_timestamp": "2026-09-26T16:00:20.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.747500125, "accuracy_std_dev": 0.03323948977661021, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0216457225, "avg_duration_per_task_ms": 167096.8075, "avg_output_tokens_per_task": 13839.2125, "last_run_timestamp": "2026-09-26T08:10:10.199Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7475, "accuracy_std_dev": 0.028565091361660317, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.012148102500000002, "avg_duration_per_task_ms": 152199.3025, "avg_output_tokens_per_task": 16425.225, "last_run_timestamp": "2026-09-26T08:04:53.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.7474747407407407, "accuracy_std_dev": 0.061334593171164214, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.07039795622895623, "avg_duration_per_task_ms": 90683.95286195286, "avg_output_tokens_per_task": 12893.693602693602, "last_run_timestamp": "2026-08-15T14:12:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7466670000000001, "accuracy_std_dev": 0.03999999999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.44994164999999997, "avg_duration_per_task_ms": 128787.73, "avg_output_tokens_per_task": 20836.09, "last_run_timestamp": "2026-09-01T17:32:23.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.18056328, "avg_duration_per_task_ms": 84967.36, "avg_output_tokens_per_task": 6058.98, "last_run_timestamp": "2026-09-29T12:00:06.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.746667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.3847332, "avg_duration_per_task_ms": 155324.72, "avg_output_tokens_per_task": 13079.53, "last_run_timestamp": "2026-09-01T17:31:47.956Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02698528, "avg_duration_per_task_ms": 147907.84, "avg_output_tokens_per_task": 11065.38, "last_run_timestamp": "2026-07-18T08:04:04.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7466667499999999, "accuracy_std_dev": 0.03349941956493424, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.54667351, "avg_duration_per_task_ms": 183079.6925, "avg_output_tokens_per_task": 10525.565, "last_run_timestamp": "2026-09-26T08:07:18.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7466667499999999, "accuracy_std_dev": 0.014907082584714572, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12188539999999999, "avg_duration_per_task_ms": 449678.28, "avg_output_tokens_per_task": 19415.325, "last_run_timestamp": "2026-09-26T08:23:24.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.7466667499999999, "accuracy_std_dev": 0.012018481141038592, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0546571325, "avg_duration_per_task_ms": 218124.0125, "avg_output_tokens_per_task": 9184.585, "last_run_timestamp": "2026-08-22T16:12:21.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.7466667499999999, "accuracy_std_dev": 0.02494415986132826, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0097815, "avg_duration_per_task_ms": 206039.625, "avg_output_tokens_per_task": 4442.965, "last_run_timestamp": "2026-09-26T08:26:29.786Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.2640489466666667, "avg_duration_per_task_ms": 261052.74666666667, "avg_output_tokens_per_task": 16845.666666666668, "last_run_timestamp": "2026-09-26T08:22:21.004Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.42183384, "avg_duration_per_task_ms": 170456.6, "avg_output_tokens_per_task": 28298.04, "last_run_timestamp": "2026-09-01T17:30:52.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.011547101609639251, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04422623666666667, "avg_duration_per_task_ms": 266653.04, "avg_output_tokens_per_task": 13360.296666666667, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.024366269999999995, "avg_duration_per_task_ms": 118191.69, "avg_output_tokens_per_task": 9023.81, "last_run_timestamp": "2026-09-26T16:00:24.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04198429, "avg_duration_per_task_ms": 312370.3, "avg_output_tokens_per_task": 24895.75, "last_run_timestamp": "2026-09-26T16:00:24.612Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.7466665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03717445, "avg_duration_per_task_ms": 350241.71, "avg_output_tokens_per_task": 12501.07, "last_run_timestamp": "2026-09-26T08:10:43.553Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.7466665, "accuracy_std_dev": 0.019999888889274717, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06283585333333332, "avg_duration_per_task_ms": 361191.69, "avg_output_tokens_per_task": 8869.28, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.7466664000000001, "accuracy_std_dev": 0.03126950946593184, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.172299032, "avg_duration_per_task_ms": 325786.904, "avg_output_tokens_per_task": 11139.324, "last_run_timestamp": "2026-09-26T08:08:19.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7466330707070707, "accuracy_std_dev": 0.035448714834779266, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.07755478535353535, "avg_duration_per_task_ms": 198637.78535353535, "avg_output_tokens_per_task": 10041.954545454546, "last_run_timestamp": "2026-08-29T16:44:10.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.7457142857142857, "accuracy_std_dev": 0.01866272796407924, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.041890954285714294, "avg_duration_per_task_ms": 192572.89428571428, "avg_output_tokens_per_task": 12803.885714285714, "last_run_timestamp": "2026-09-26T08:05:12.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.7455555, "accuracy_std_dev": 0.027599586239350292, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11502652333333335, "avg_duration_per_task_ms": 272510.09, "avg_output_tokens_per_task": 12128.78, "last_run_timestamp": "2026-09-26T08:22:52.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7454546363636363, "accuracy_std_dev": 0.026180138836609183, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.35241780545454543, "avg_duration_per_task_ms": 231797.27818181817, "avg_output_tokens_per_task": 10791.974545454545, "last_run_timestamp": "2026-09-26T08:23:01.142Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.745416625, "accuracy_std_dev": 0.01870356936481312, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.03490152, "avg_duration_per_task_ms": 436262.0025, "avg_output_tokens_per_task": 9745.805, "last_run_timestamp": "2026-08-22T16:09:51.985Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.745, "accuracy_std_dev": 0.021015867023513466, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.008970824999999998, "avg_duration_per_task_ms": 224202.575, "avg_output_tokens_per_task": 13976.12, "last_run_timestamp": "2026-08-29T16:33:30.698Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.745, "accuracy_std_dev": 0.009860245661239886, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.45607220499999995, "avg_duration_per_task_ms": 481202.88, "avg_output_tokens_per_task": 36078.37, "last_run_timestamp": "2026-09-01T12:59:55.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.7448276551724138, "accuracy_std_dev": 0.0035438860340341496, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.022715406896551726, "avg_duration_per_task_ms": 289516.0620689655, "avg_output_tokens_per_task": 10103.63448275862, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.7447619999999999, "accuracy_std_dev": 0.020381773461320103, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.2414585142857143, "avg_duration_per_task_ms": 282563.52571428573, "avg_output_tokens_per_task": 10218.734285714285, "last_run_timestamp": "2026-09-29T21:00:05.975Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.7447618571428571, "accuracy_std_dev": 0.035405124090119165, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08685644000000001, "avg_duration_per_task_ms": 256492.48857142858, "avg_output_tokens_per_task": 12140.245714285715, "last_run_timestamp": "2026-09-26T08:23:17.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.744681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.05595836170212766, "avg_duration_per_task_ms": 34218.255319148935, "avg_output_tokens_per_task": 6745.085106382979, "last_run_timestamp": "2026-08-04T03:12:18.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.7441947955056181, "accuracy_std_dev": 0.013929720069329346, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.10463494606741572, "avg_duration_per_task_ms": 943614.8921348314, "avg_output_tokens_per_task": 21227.357303370787, "last_run_timestamp": "2026-09-26T08:24:43.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.74416675, "accuracy_std_dev": 0.021586450870569266, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.32272964499999995, "avg_duration_per_task_ms": 199123.9275, "avg_output_tokens_per_task": 11446.9275, "last_run_timestamp": "2026-09-26T08:10:43.577Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.7441077474747474, "accuracy_std_dev": 0.03625850000000003, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.012172767676767676, "avg_duration_per_task_ms": 161557.24242424243, "avg_output_tokens_per_task": 3853.3636363636365, "last_run_timestamp": "2026-09-26T08:10:43.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.7440306848137535, "accuracy_std_dev": 0.019647453058182128, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.028864432664756447, "avg_duration_per_task_ms": 650673.9828080229, "avg_output_tokens_per_task": 21865.07736389685, "last_run_timestamp": "2026-09-26T16:00:24.615Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.744, "accuracy_std_dev": 0.019137165704461043, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.006426796, "avg_duration_per_task_ms": 131863.972, "avg_output_tokens_per_task": 4262.824, "last_run_timestamp": "2026-09-26T08:07:18.986Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7439998000000001, "accuracy_std_dev": 0.025508095447524134, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.010447856000000002, "avg_duration_per_task_ms": 427694.796, "avg_output_tokens_per_task": 12591.384, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.7438097142857142, "accuracy_std_dev": 0.031240303349150428, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11824320857142855, "avg_duration_per_task_ms": 449478.22285714286, "avg_output_tokens_per_task": 11476.202857142856, "last_run_timestamp": "2026-08-29T16:43:42.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.74338923575419, "accuracy_std_dev": 0.032960776721153426, "run_count": 18, "total_questions": 895, "avg_cost_per_task_usd": 0.11096166927374301, "avg_duration_per_task_ms": 107590.68156424581, "avg_output_tokens_per_task": 8138.489385474861, "last_run_timestamp": "2026-09-26T08:26:47.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.7433335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02746015, "avg_duration_per_task_ms": 758204.27, "avg_output_tokens_per_task": 13329.06, "last_run_timestamp": "2026-07-18T08:03:38.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7433335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0032184600000000002, "avg_duration_per_task_ms": 64941.97, "avg_output_tokens_per_task": 11846.7, "last_run_timestamp": "2026-08-19T18:00:02.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7433332500000001, "accuracy_std_dev": 0.022852157692163335, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.07602055999999999, "avg_duration_per_task_ms": 242604.3475, "avg_output_tokens_per_task": 11060.65, "last_run_timestamp": "2026-08-29T17:19:05.156Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7433330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.34578612, "avg_duration_per_task_ms": 121381.44, "avg_output_tokens_per_task": 6958.96, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7432883288590605, "accuracy_std_dev": 0.0384259139109129, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.058029342281879205, "avg_duration_per_task_ms": 234207.55369127516, "avg_output_tokens_per_task": 7397.003355704698, "last_run_timestamp": "2026-08-15T08:17:17.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.7432794959677419, "accuracy_std_dev": 0.019165579787994942, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.14048193346774193, "avg_duration_per_task_ms": 566039.3870967742, "avg_output_tokens_per_task": 27554.729838709678, "last_run_timestamp": "2026-09-26T08:23:04.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.7429854713804713, "accuracy_std_dev": 0.03977251990661671, "run_count": 12, "total_questions": 594, "avg_cost_per_task_usd": 0.022424602693602694, "avg_duration_per_task_ms": 322809.1447811448, "avg_output_tokens_per_task": 12708.414141414141, "last_run_timestamp": "2026-09-26T08:22:59.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7428572857142857, "accuracy_std_dev": 0.03184420977515506, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07498379428571428, "avg_duration_per_task_ms": 408111.76, "avg_output_tokens_per_task": 11455.622857142856, "last_run_timestamp": "2026-08-29T16:53:19.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.7427776666666667, "accuracy_std_dev": 0.036434422742175517, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3671939166666667, "avg_duration_per_task_ms": 137596.83, "avg_output_tokens_per_task": 5212.336666666667, "last_run_timestamp": "2026-09-01T13:00:30.911Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.7426668, "accuracy_std_dev": 0.03690245640821219, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011645404, "avg_duration_per_task_ms": 585840.916, "avg_output_tokens_per_task": 18794.172, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.7426666000000001, "accuracy_std_dev": 0.01236473120775379, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07381416, "avg_duration_per_task_ms": 152125.172, "avg_output_tokens_per_task": 4690.728, "last_run_timestamp": "2026-09-26T08:27:29.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.7425, "accuracy_std_dev": 0.029988405167664416, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.5165171675, "avg_duration_per_task_ms": 228220.77, "avg_output_tokens_per_task": 10000.7275, "last_run_timestamp": "2026-09-26T08:11:12.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.742499875, "accuracy_std_dev": 0.021327474782762628, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0092532975, "avg_duration_per_task_ms": 294444.1, "avg_output_tokens_per_task": 4179.7575, "last_run_timestamp": "2026-09-26T08:24:45.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7422221666666666, "accuracy_std_dev": 0.009937943254964244, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.34229832999999993, "avg_duration_per_task_ms": 115494.89, "avg_output_tokens_per_task": 7015.326666666667, "last_run_timestamp": "2026-09-26T08:16:17.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7422221666666666, "accuracy_std_dev": 0.029731460662720387, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.34941253333333333, "avg_duration_per_task_ms": 221865.17666666667, "avg_output_tokens_per_task": 10845.116666666667, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7422221666666666, "accuracy_std_dev": 0.01791636261835036, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3279842533333333, "avg_duration_per_task_ms": 196889.34333333332, "avg_output_tokens_per_task": 10978.62, "last_run_timestamp": "2026-09-01T17:31:47.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7421122716049383, "accuracy_std_dev": 0.04949233809146621, "run_count": 5, "total_questions": 243, "avg_cost_per_task_usd": 0.07618280658436215, "avg_duration_per_task_ms": 605963.6255144033, "avg_output_tokens_per_task": 7686.909465020576, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7419047142857143, "accuracy_std_dev": 0.03059492644080661, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1254379457142857, "avg_duration_per_task_ms": 439339.8028571429, "avg_output_tokens_per_task": 9897.565714285714, "last_run_timestamp": "2026-08-29T16:44:15.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.7418180909090909, "accuracy_std_dev": 0.04264882202669216, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.09875620727272726, "avg_duration_per_task_ms": 153243.34545454546, "avg_output_tokens_per_task": 8615.118181818181, "last_run_timestamp": "2026-09-26T08:22:32.817Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.7416666666666667, "accuracy_std_dev": 0.0371060392419108, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.15347501333333333, "avg_duration_per_task_ms": 347171.24333333335, "avg_output_tokens_per_task": 21806.65, "last_run_timestamp": "2026-09-26T08:23:03.682Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.741666625, "accuracy_std_dev": 0.021794446924259765, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0756606775, "avg_duration_per_task_ms": 327680.4725, "avg_output_tokens_per_task": 12938.9875, "last_run_timestamp": "2026-08-29T16:16:50.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.7414969591836733, "accuracy_std_dev": 0.023315275936604284, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.01141882142857143, "avg_duration_per_task_ms": 212540.27551020408, "avg_output_tokens_per_task": 4189.09693877551, "last_run_timestamp": "2026-09-26T08:26:46.238Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.7413600133779265, "accuracy_std_dev": 0.021148441080582975, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.06112101672240802, "avg_duration_per_task_ms": 177751.98996655518, "avg_output_tokens_per_task": 7034.244147157191, "last_run_timestamp": "2026-08-15T08:13:33.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.7411113333333333, "accuracy_std_dev": 0.028588182830595485, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018602686666666667, "avg_duration_per_task_ms": 242873.4, "avg_output_tokens_per_task": 16988.533333333333, "last_run_timestamp": "2026-09-26T08:21:38.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7411111666666667, "accuracy_std_dev": 0.02732958194836176, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.34793083666666663, "avg_duration_per_task_ms": 202714.10666666666, "avg_output_tokens_per_task": 11886.45, "last_run_timestamp": "2026-09-26T08:09:25.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.7411111666666667, "accuracy_std_dev": 0.025361580257130863, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.019523156666666666, "avg_duration_per_task_ms": 217853.99333333335, "avg_output_tokens_per_task": 12622.736666666666, "last_run_timestamp": "2026-08-29T16:56:37.261Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7411111666666667, "accuracy_std_dev": 0.0418846402213535, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.011101129999999999, "avg_duration_per_task_ms": 1049014.8366666667, "avg_output_tokens_per_task": 11369.113333333333, "last_run_timestamp": "2026-08-29T16:44:33.448Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7411110000000001, "accuracy_std_dev": 0.033425712732366185, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03801252333333333, "avg_duration_per_task_ms": 199405.32333333333, "avg_output_tokens_per_task": 13535.723333333333, "last_run_timestamp": "2026-09-26T08:09:22.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.740909090909091, "accuracy_std_dev": 0.034424213616038424, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.3756752018181817, "avg_duration_per_task_ms": 205007.96545454545, "avg_output_tokens_per_task": 11919.866363636364, "last_run_timestamp": "2026-09-26T08:26:37.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7406779661016949, "accuracy_std_dev": 0.0192449897289312, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.02004586440677966, "avg_duration_per_task_ms": 1117850.5423728814, "avg_output_tokens_per_task": 22657.922033898307, "last_run_timestamp": "2026-09-12T08:11:08.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.7406667, "accuracy_std_dev": 0.04386339856201296, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.127795346, "avg_duration_per_task_ms": 805643.47, "avg_output_tokens_per_task": 30964.946, "last_run_timestamp": "2026-09-26T08:01:16.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.7406059090909092, "accuracy_std_dev": 0.025017968199074807, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.024014645454545454, "avg_duration_per_task_ms": 391382.4381818182, "avg_output_tokens_per_task": 12338.783636363636, "last_run_timestamp": "2026-09-26T08:08:19.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7404919865771813, "accuracy_std_dev": 0.019703479706781643, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.13110433892617449, "avg_duration_per_task_ms": 475456.4194630872, "avg_output_tokens_per_task": 10412.023489932886, "last_run_timestamp": "2026-08-22T08:22:21.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7402553306451614, "accuracy_std_dev": 0.04059281224847572, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.12427361491935483, "avg_duration_per_task_ms": 525015.5120967742, "avg_output_tokens_per_task": 25091.802419354837, "last_run_timestamp": "2026-09-26T08:23:03.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.74, "accuracy_std_dev": 0.05789968336207521, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01944233714285714, "avg_duration_per_task_ms": 226107.22, "avg_output_tokens_per_task": 12391.968571428572, "last_run_timestamp": "2026-08-29T16:25:14.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2696536, "avg_duration_per_task_ms": 310259.6, "avg_output_tokens_per_task": 11435.58, "last_run_timestamp": "2026-07-18T08:04:29.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05110574, "avg_duration_per_task_ms": 158677.44, "avg_output_tokens_per_task": 8249.06, "last_run_timestamp": "2026-07-18T16:00:09.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.74, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.054267579999999996, "avg_duration_per_task_ms": 797443.04, "avg_output_tokens_per_task": 42406.53, "last_run_timestamp": "2026-08-29T16:33:37.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.74, "accuracy_std_dev": 0.02380486645100116, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09328288000000001, "avg_duration_per_task_ms": 826540.2925, "avg_output_tokens_per_task": 38601.0775, "last_run_timestamp": "2026-09-26T16:00:20.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02561014, "avg_duration_per_task_ms": 147210.62, "avg_output_tokens_per_task": 10787.18, "last_run_timestamp": "2026-07-18T08:04:42.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.74, "accuracy_std_dev": 0.03187268323457296, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.4139113142857143, "avg_duration_per_task_ms": 36529.922857142854, "avg_output_tokens_per_task": 9198.594285714285, "last_run_timestamp": "2026-08-15T14:16:11.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.74, "accuracy_std_dev": 0.03944045441548562, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1116368375, "avg_duration_per_task_ms": 298791.685, "avg_output_tokens_per_task": 9064.0575, "last_run_timestamp": "2026-08-29T16:12:10.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.74, "accuracy_std_dev": 0.01115542718142162, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.014306968, "avg_duration_per_task_ms": 188774.736, "avg_output_tokens_per_task": 4205.028, "last_run_timestamp": "2026-09-26T08:25:40.376Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.7396135507246376, "accuracy_std_dev": 0.02509467552355785, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.0175928115942029, "avg_duration_per_task_ms": 309043.7826086957, "avg_output_tokens_per_task": 14060.257971014493, "last_run_timestamp": "2026-09-26T08:21:25.884Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.7395366098654709, "accuracy_std_dev": 0.01837246202847226, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.11281150224215246, "avg_duration_per_task_ms": 100432.45515695067, "avg_output_tokens_per_task": 9897.378923766817, "last_run_timestamp": "2026-08-30T00:01:10.606Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.739160162601626, "accuracy_std_dev": 0.016384230901693223, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.02238267073170732, "avg_duration_per_task_ms": 118042.10162601626, "avg_output_tokens_per_task": 3823.4349593495936, "last_run_timestamp": "2026-09-26T08:17:01.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.7391112, "accuracy_std_dev": 0.02063675876746798, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.5042409586666666, "avg_duration_per_task_ms": 734521.768, "avg_output_tokens_per_task": 9377.424, "last_run_timestamp": "2026-09-26T08:10:56.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.7390475714285715, "accuracy_std_dev": 0.014876513626299345, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.007957537142857143, "avg_duration_per_task_ms": 146934.60285714286, "avg_output_tokens_per_task": 13560.128571428571, "last_run_timestamp": "2026-08-29T17:19:58.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.738889, "accuracy_std_dev": 0.017812306747489716, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.022993023333333338, "avg_duration_per_task_ms": 282964.01666666666, "avg_output_tokens_per_task": 13071.093333333334, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.7387387567567568, "accuracy_std_dev": 0.03311066334489372, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.1412797635135135, "avg_duration_per_task_ms": 662200.6148648649, "avg_output_tokens_per_task": 26757.62162162162, "last_run_timestamp": "2026-08-15T14:15:18.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.7386935778894472, "accuracy_std_dev": 0.06597081195877463, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.24299765829145728, "avg_duration_per_task_ms": 432313.4522613065, "avg_output_tokens_per_task": 26613.50502512563, "last_run_timestamp": "2026-08-29T16:13:54.412Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.7386666000000001, "accuracy_std_dev": 0.02777674167068556, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.008449704, "avg_duration_per_task_ms": 262399.948, "avg_output_tokens_per_task": 5465.996, "last_run_timestamp": "2026-09-26T08:30:17.131Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7383335, "accuracy_std_dev": 0.007264793338423339, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.029026915000000004, "avg_duration_per_task_ms": 190934.755, "avg_output_tokens_per_task": 18429.705, "last_run_timestamp": "2026-09-26T08:10:43.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.7383335, "accuracy_std_dev": 0.01092919128984391, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07321903, "avg_duration_per_task_ms": 107189.01, "avg_output_tokens_per_task": 4495.765, "last_run_timestamp": "2026-09-26T08:22:24.359Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.7383335, "accuracy_std_dev": 0.01092919128984391, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.023440455, "avg_duration_per_task_ms": 211636.695, "avg_output_tokens_per_task": 4181.36, "last_run_timestamp": "2026-09-26T08:23:00.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.73833325, "accuracy_std_dev": 0.03345812889175814, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0298002275, "avg_duration_per_task_ms": 280974.21, "avg_output_tokens_per_task": 17047.3125, "last_run_timestamp": "2026-09-26T08:28:05.592Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.7382865983935742, "accuracy_std_dev": 0.027461257286584682, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.07498479919678715, "avg_duration_per_task_ms": 91080.29718875502, "avg_output_tokens_per_task": 4367.955823293173, "last_run_timestamp": "2026-09-26T08:11:26.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7380952857142857, "accuracy_std_dev": 0.034862897001639706, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10877247142857141, "avg_duration_per_task_ms": 324815.4542857143, "avg_output_tokens_per_task": 18549.505714285715, "last_run_timestamp": "2026-09-26T08:13:27.148Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.7380952857142857, "accuracy_std_dev": 0.01771508190952629, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07855077714285713, "avg_duration_per_task_ms": 317722.2542857143, "avg_output_tokens_per_task": 13494.22, "last_run_timestamp": "2026-08-29T16:33:11.313Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.7371364697986577, "accuracy_std_dev": 0.02632276711591866, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.02334386577181208, "avg_duration_per_task_ms": 151084.5889261745, "avg_output_tokens_per_task": 7290.451342281879, "last_run_timestamp": "2026-09-26T16:00:18.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02745565, "avg_duration_per_task_ms": 321964.94, "avg_output_tokens_per_task": 12731.03, "last_run_timestamp": "2026-09-26T08:24:06.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.7366665, "accuracy_std_dev": 0.05666649999999995, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5401649000000001, "avg_duration_per_task_ms": 161301.85, "avg_output_tokens_per_task": 8843.07, "last_run_timestamp": "2026-09-01T13:02:17.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7364864414414414, "accuracy_std_dev": 0.03584140193121722, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.09302163738738738, "avg_duration_per_task_ms": 492123.9324324324, "avg_output_tokens_per_task": 14254.815315315316, "last_run_timestamp": "2026-08-29T16:54:12.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7364862297297298, "accuracy_std_dev": 0.026034309298820803, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.046338074324324324, "avg_duration_per_task_ms": 757120.8243243244, "avg_output_tokens_per_task": 14026.905405405405, "last_run_timestamp": "2026-08-29T16:49:08.522Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.7361687676348548, "accuracy_std_dev": 0.033251517706865664, "run_count": 10, "total_questions": 482, "avg_cost_per_task_usd": 0.057402344398340244, "avg_duration_per_task_ms": 118233.5, "avg_output_tokens_per_task": 7620.255186721992, "last_run_timestamp": "2026-09-26T08:08:33.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.7359389551954242, "accuracy_std_dev": 0.023276928156711292, "run_count": 21, "total_questions": 1049, "avg_cost_per_task_usd": 0.1205376796949476, "avg_duration_per_task_ms": 180044.07244995233, "avg_output_tokens_per_task": 9034.192564346997, "last_run_timestamp": "2026-09-26T08:08:19.667Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7356555409836066, "accuracy_std_dev": 0.012054742658389706, "run_count": 5, "total_questions": 244, "avg_cost_per_task_usd": 0.01272932786885246, "avg_duration_per_task_ms": 60416.67213114754, "avg_output_tokens_per_task": 3637.938524590164, "last_run_timestamp": "2026-09-29T03:00:05.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13620342666666665, "avg_duration_per_task_ms": 101042.81333333334, "avg_output_tokens_per_task": 8769.966666666667, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020994060000000002, "avg_duration_per_task_ms": 93700.16666666667, "avg_output_tokens_per_task": 4193.253333333333, "last_run_timestamp": "2026-09-26T08:26:42.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7355553333333333, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13248254666666667, "avg_duration_per_task_ms": 94960.21333333333, "avg_output_tokens_per_task": 24879.886666666665, "last_run_timestamp": "2026-09-01T17:31:46.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.7355553333333333, "accuracy_std_dev": 0.013698571612479268, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.011050406666666665, "avg_duration_per_task_ms": 621248.44, "avg_output_tokens_per_task": 4366.6466666666665, "last_run_timestamp": "2026-09-26T08:07:18.968Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7350002499999999, "accuracy_std_dev": 0.008660205926391138, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03314667, "avg_duration_per_task_ms": 134389.755, "avg_output_tokens_per_task": 16623.845, "last_run_timestamp": "2026-09-26T08:05:12.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.7348993255033557, "accuracy_std_dev": 0.031569685085691364, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.030765446308724832, "avg_duration_per_task_ms": 431458.02013422817, "avg_output_tokens_per_task": 13334.538590604026, "last_run_timestamp": "2026-09-26T08:22:26.272Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Phala", "accuracy": 0.7346937142857143, "accuracy_std_dev": 0.03541650000000002, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.1668343367346939, "avg_duration_per_task_ms": 415202.8163265306, "avg_output_tokens_per_task": 9046.959183673469, "last_run_timestamp": "2026-07-11T15:34:16.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.7342855714285714, "accuracy_std_dev": 0.03175860195671242, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12320952285714286, "avg_duration_per_task_ms": 247294.2142857143, "avg_output_tokens_per_task": 9957.725714285714, "last_run_timestamp": "2026-08-29T16:46:44.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.7340742222222222, "accuracy_std_dev": 0.04842648211642921, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07961510222222222, "avg_duration_per_task_ms": 198839.9311111111, "avg_output_tokens_per_task": 8248.502222222221, "last_run_timestamp": "2026-09-19T08:12:24.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7339180701754385, "accuracy_std_dev": 0.02156426040350099, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.009527433583959901, "avg_duration_per_task_ms": 516946.6215538847, "avg_output_tokens_per_task": 13232.288220551378, "last_run_timestamp": "2026-09-26T08:07:18.969Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.34495834000000003, "avg_duration_per_task_ms": 101333.53, "avg_output_tokens_per_task": 6956.68, "last_run_timestamp": "2026-09-26T08:22:24.747Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09979016999999998, "avg_duration_per_task_ms": 95640.78, "avg_output_tokens_per_task": 4171.89, "last_run_timestamp": "2026-09-01T12:57:13.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.733333375, "accuracy_std_dev": 0.05196139592749965, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0075224875, "avg_duration_per_task_ms": 325334.9425, "avg_output_tokens_per_task": 15635.5625, "last_run_timestamp": "2026-09-26T08:07:31.852Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008480786666666665, "avg_duration_per_task_ms": 397846.0266666667, "avg_output_tokens_per_task": 19127.326666666668, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.73333325, "accuracy_std_dev": 0.004713927363409425, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.093701365, "avg_duration_per_task_ms": 409506.12, "avg_output_tokens_per_task": 12562.605, "last_run_timestamp": "2026-09-26T08:23:24.418Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.7333332, "accuracy_std_dev": 0.01633001326882499, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.016486864, "avg_duration_per_task_ms": 85790.632, "avg_output_tokens_per_task": 4187.072, "last_run_timestamp": "2026-09-26T08:05:55.055Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.7333331666666667, "accuracy_std_dev": 0.012171581962597242, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3451316866666666, "avg_duration_per_task_ms": 86668.75666666667, "avg_output_tokens_per_task": 6980.716666666666, "last_run_timestamp": "2026-09-26T08:17:01.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.34216192, "avg_duration_per_task_ms": 118971.98, "avg_output_tokens_per_task": 7077.54, "last_run_timestamp": "2026-09-26T08:22:25.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.13542124, "avg_duration_per_task_ms": 450545.7, "avg_output_tokens_per_task": 22302.62, "last_run_timestamp": "2026-09-28T06:00:03.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.046736959999999994, "avg_duration_per_task_ms": 211644.04, "avg_output_tokens_per_task": 28167.02, "last_run_timestamp": "2026-09-28T18:00:05.897Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2412942, "avg_duration_per_task_ms": 207134.84, "avg_output_tokens_per_task": 12538.34, "last_run_timestamp": "2026-08-01T08:06:12.346Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0268046, "avg_duration_per_task_ms": 1050187.54, "avg_output_tokens_per_task": 22343.12, "last_run_timestamp": "2026-09-28T18:00:05.487Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.7329933877551019, "accuracy_std_dev": 0.02075268657909138, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.05900951530612245, "avg_duration_per_task_ms": 460991.887755102, "avg_output_tokens_per_task": 12569.994897959185, "last_run_timestamp": "2026-09-26T08:23:24.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.7328571428571429, "accuracy_std_dev": 0.034013600054887995, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0738926457142857, "avg_duration_per_task_ms": 257306.79142857142, "avg_output_tokens_per_task": 11384.062857142857, "last_run_timestamp": "2026-08-29T17:08:03.684Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.7328571428571429, "accuracy_std_dev": 0.02503509736024079, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06745117142857142, "avg_duration_per_task_ms": 76562.92857142857, "avg_output_tokens_per_task": 8496.594285714285, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7326621543624161, "accuracy_std_dev": 0.030789543369281885, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.09465771812080537, "avg_duration_per_task_ms": 390427.067114094, "avg_output_tokens_per_task": 11230.369127516778, "last_run_timestamp": "2026-08-29T16:33:38.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.732381, "accuracy_std_dev": 0.026530205265481322, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07102172, "avg_duration_per_task_ms": 255607.41142857142, "avg_output_tokens_per_task": 7988.711428571429, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.7316890269058296, "accuracy_std_dev": 0.028135970753590752, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.03146767488789237, "avg_duration_per_task_ms": 485485.01569506724, "avg_output_tokens_per_task": 14361.15470852018, "last_run_timestamp": "2026-08-29T16:11:34.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.73166675, "accuracy_std_dev": 0.03210208110368392, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.00461813, "avg_duration_per_task_ms": 254807.5, "avg_output_tokens_per_task": 4301.06, "last_run_timestamp": "2026-09-26T08:25:43.420Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.7315434328859061, "accuracy_std_dev": 0.019791500921937582, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.007604364093959733, "avg_duration_per_task_ms": 318961.6610738255, "avg_output_tokens_per_task": 13646.53187919463, "last_run_timestamp": "2026-09-26T08:05:12.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7314287142857142, "accuracy_std_dev": 0.02301532369478017, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07236491428571429, "avg_duration_per_task_ms": 89556.96, "avg_output_tokens_per_task": 10594.671428571428, "last_run_timestamp": "2026-09-26T16:00:24.624Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.7314287142857142, "accuracy_std_dev": 0.009897460680314293, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.39489586285714284, "avg_duration_per_task_ms": 142184.54857142858, "avg_output_tokens_per_task": 11046.07142857143, "last_run_timestamp": "2026-09-26T08:21:22.720Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012396426666666667, "avg_duration_per_task_ms": 337807.9, "avg_output_tokens_per_task": 15691.053333333333, "last_run_timestamp": "2026-09-26T08:22:46.339Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.025903600000000002, "avg_duration_per_task_ms": 117172.05333333333, "avg_output_tokens_per_task": 15158.646666666667, "last_run_timestamp": "2026-09-26T08:07:18.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.035416115465514666, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3332058333333333, "avg_duration_per_task_ms": 117249.60666666667, "avg_output_tokens_per_task": 11086.966666666667, "last_run_timestamp": "2026-09-01T17:31:47.546Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.730909, "accuracy_std_dev": 0.02416641341351107, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06488479090909091, "avg_duration_per_task_ms": 624903.3563636363, "avg_output_tokens_per_task": 8185.998181818181, "last_run_timestamp": "2026-09-26T08:22:17.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.730909, "accuracy_std_dev": 0.028180123769003633, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06749359090909092, "avg_duration_per_task_ms": 291533.3, "avg_output_tokens_per_task": 9954.196363636363, "last_run_timestamp": "2026-09-26T08:23:16.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.7307692064777328, "accuracy_std_dev": 0.04749195792721122, "run_count": 5, "total_questions": 247, "avg_cost_per_task_usd": 0.15509178542510121, "avg_duration_per_task_ms": 774237.7125506073, "avg_output_tokens_per_task": 29578.534412955465, "last_run_timestamp": "2026-08-29T16:30:43.592Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7305263684210526, "accuracy_std_dev": 0.033390586105116714, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.35496904315789474, "avg_duration_per_task_ms": 206890.78210526315, "avg_output_tokens_per_task": 10297.44, "last_run_timestamp": "2026-09-26T08:17:01.551Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.7304761428571429, "accuracy_std_dev": 0.040293813213262725, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.006170451428571428, "avg_duration_per_task_ms": 328862.59714285715, "avg_output_tokens_per_task": 15818.588571428572, "last_run_timestamp": "2026-09-26T08:13:27.139Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7300306912442397, "accuracy_std_dev": 0.022919556859094893, "run_count": 9, "total_questions": 434, "avg_cost_per_task_usd": 0.018301880184331796, "avg_duration_per_task_ms": 93142.05069124424, "avg_output_tokens_per_task": 5748.317972350231, "last_run_timestamp": "2026-09-26T16:00:25.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.7300002, "accuracy_std_dev": 0.01633001326882499, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.06427955199999999, "avg_duration_per_task_ms": 445840.884, "avg_output_tokens_per_task": 8867.492, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7300001666666667, "accuracy_std_dev": 0.03605552816336059, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11662036, "avg_duration_per_task_ms": 82058.93666666666, "avg_output_tokens_per_task": 9160.97, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.73, "accuracy_std_dev": 0.030489986214056124, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.49638783, "avg_duration_per_task_ms": 192253.77333333335, "avg_output_tokens_per_task": 9911.976666666667, "last_run_timestamp": "2026-09-26T08:22:57.969Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.73, "accuracy_std_dev": 0.0247205998233255, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.36644244, "avg_duration_per_task_ms": 230988.18, "avg_output_tokens_per_task": 10810.57375, "last_run_timestamp": "2026-09-26T08:22:54.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.73, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23651355, "avg_duration_per_task_ms": 142691.29, "avg_output_tokens_per_task": 22898.27, "last_run_timestamp": "2026-09-01T17:31:43.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.73, "accuracy_std_dev": 0.026874047787782167, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.40094411999999996, "avg_duration_per_task_ms": 212371.995, "avg_output_tokens_per_task": 8595.31, "last_run_timestamp": "2026-09-26T08:23:04.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.73, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.30719247000000005, "avg_duration_per_task_ms": 162813.34, "avg_output_tokens_per_task": 10978.12, "last_run_timestamp": "2026-09-01T17:32:02.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.73, "accuracy_std_dev": 0.01374388753228139, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.243089465, "avg_duration_per_task_ms": 228014.985, "avg_output_tokens_per_task": 12589.82, "last_run_timestamp": "2026-09-26T08:11:43.368Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.7295868703170029, "accuracy_std_dev": 0.031298136757478015, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.07363949279538905, "avg_duration_per_task_ms": 211055.38616714696, "avg_output_tokens_per_task": 8076.533141210374, "last_run_timestamp": "2026-09-26T08:14:46.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.7295848835341366, "accuracy_std_dev": 0.03127469483608115, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.04057755020080321, "avg_duration_per_task_ms": 229097.8734939759, "avg_output_tokens_per_task": 9654.106425702812, "last_run_timestamp": "2026-09-26T08:25:16.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.7294807939698492, "accuracy_std_dev": 0.09894662195850853, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.011416806532663316, "avg_duration_per_task_ms": 215331.50753768845, "avg_output_tokens_per_task": 13145.422110552763, "last_run_timestamp": "2026-08-29T17:11:34.630Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.7293334, "accuracy_std_dev": 0.03935030820260496, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.051760342, "avg_duration_per_task_ms": 152743.692, "avg_output_tokens_per_task": 8580.186, "last_run_timestamp": "2026-09-26T08:30:17.801Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Phala", "accuracy": 0.729167, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.044939625000000004, "avg_duration_per_task_ms": 321431, "avg_output_tokens_per_task": 27916.854166666668, "last_run_timestamp": "2026-07-11T08:03:26.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.72916675, "accuracy_std_dev": 0.019346895388860176, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.059703584999999997, "avg_duration_per_task_ms": 1171969.47, "avg_output_tokens_per_task": 26770.295, "last_run_timestamp": "2026-09-26T08:10:43.579Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.7290909090909091, "accuracy_std_dev": 0.024826342686218476, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.02280834909090909, "avg_duration_per_task_ms": 86499.97454545455, "avg_output_tokens_per_task": 7271.118181818182, "last_run_timestamp": "2026-09-26T16:00:24.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.7288886666666667, "accuracy_std_dev": 0.02572412999932599, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.041228259999999996, "avg_duration_per_task_ms": 125095.52, "avg_output_tokens_per_task": 23895.02666666667, "last_run_timestamp": "2026-09-26T08:16:16.958Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.72874888252149, "accuracy_std_dev": 0.05217086698179985, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.06682716905444126, "avg_duration_per_task_ms": 1514284.8567335242, "avg_output_tokens_per_task": 8343.650429799427, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7286667, "accuracy_std_dev": 0.04099317937913574, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.106204026, "avg_duration_per_task_ms": 169828.326, "avg_output_tokens_per_task": 9261.106, "last_run_timestamp": "2026-09-26T08:22:58.695Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.7285714285714285, "accuracy_std_dev": 0.03504452237962912, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04105087428571428, "avg_duration_per_task_ms": 218582.77142857143, "avg_output_tokens_per_task": 20493.322857142855, "last_run_timestamp": "2026-09-26T08:07:18.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.7283335, "accuracy_std_dev": 0.02279116544299565, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.345777245, "avg_duration_per_task_ms": 85393.745, "avg_output_tokens_per_task": 6868.74, "last_run_timestamp": "2026-09-26T08:25:13.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.7283270805860806, "accuracy_std_dev": 0.03258021639071963, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.09055074542124542, "avg_duration_per_task_ms": 216630.95054945056, "avg_output_tokens_per_task": 8399.408424908424, "last_run_timestamp": "2026-09-26T08:23:18.761Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.7281478888888889, "accuracy_std_dev": 0.026625571215341294, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.045681291111111105, "avg_duration_per_task_ms": 180676.11555555556, "avg_output_tokens_per_task": 8297.222222222223, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.728, "accuracy_std_dev": 0.029333386364345997, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.032369690000000007, "avg_duration_per_task_ms": 162941.07, "avg_output_tokens_per_task": 8635.816, "last_run_timestamp": "2026-09-26T08:12:13.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.7276787589285714, "accuracy_std_dev": 0.023663327303270934, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.019942993303571428, "avg_duration_per_task_ms": 481307.24776785716, "avg_output_tokens_per_task": 14518.11830357143, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.7275, "accuracy_std_dev": 0.011273272229038026, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.25819418250000004, "avg_duration_per_task_ms": 292629.7475, "avg_output_tokens_per_task": 10249.3025, "last_run_timestamp": "2026-09-26T08:16:17.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7273567338262477, "accuracy_std_dev": 0.034097962965258875, "run_count": 11, "total_questions": 541, "avg_cost_per_task_usd": 0.10647174121996304, "avg_duration_per_task_ms": 340266.1959334566, "avg_output_tokens_per_task": 15129.060998151572, "last_run_timestamp": "2026-09-26T08:12:00.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7272349799498746, "accuracy_std_dev": 0.02832567897437543, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.07201908771929824, "avg_duration_per_task_ms": 227357.91979949875, "avg_output_tokens_per_task": 11495.062656641605, "last_run_timestamp": "2026-08-29T16:55:12.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.7272108979591837, "accuracy_std_dev": 0.019828664901097113, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.020857016326530614, "avg_duration_per_task_ms": 240921.06530612244, "avg_output_tokens_per_task": 10264.273469387756, "last_run_timestamp": "2026-09-26T08:13:28.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7271692397260274, "accuracy_std_dev": 0.01097209078627325, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.12186630821917807, "avg_duration_per_task_ms": 191102.53424657535, "avg_output_tokens_per_task": 9680.431506849314, "last_run_timestamp": "2026-09-26T16:00:20.148Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7271506693548386, "accuracy_std_dev": 0.030421326219611147, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.13253313306451614, "avg_duration_per_task_ms": 725346.5120967742, "avg_output_tokens_per_task": 29741.665322580644, "last_run_timestamp": "2026-08-15T08:15:35.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.029598939999999997, "avg_duration_per_task_ms": 159012.9, "avg_output_tokens_per_task": 16200.32, "last_run_timestamp": "2026-09-26T08:25:06.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.3914632, "avg_duration_per_task_ms": 307698.14, "avg_output_tokens_per_task": 11120.02, "last_run_timestamp": "2026-09-29T12:00:07.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.27023132, "avg_duration_per_task_ms": 228964.68, "avg_output_tokens_per_task": 14058.24, "last_run_timestamp": "2026-08-01T08:05:59.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06664494, "avg_duration_per_task_ms": 229624.28, "avg_output_tokens_per_task": 13080.68, "last_run_timestamp": "2026-09-23T06:00:08.551Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.72666675, "accuracy_std_dev": 0.01414201777284626, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.19420406499999998, "avg_duration_per_task_ms": 374699.79, "avg_output_tokens_per_task": 9514.245, "last_run_timestamp": "2026-09-26T08:17:01.582Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.7266666363636364, "accuracy_std_dev": 0.025897973870460233, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.028845547272727274, "avg_duration_per_task_ms": 194541.59636363637, "avg_output_tokens_per_task": 9348.309090909092, "last_run_timestamp": "2026-09-26T16:00:24.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.7266666, "accuracy_std_dev": 0.028906085481088594, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.078362018, "avg_duration_per_task_ms": 221052.738, "avg_output_tokens_per_task": 8631.572, "last_run_timestamp": "2026-09-26T08:23:24.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.7266665454545455, "accuracy_std_dev": 0.021835047354544654, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07702561272727272, "avg_duration_per_task_ms": 268603.45636363636, "avg_output_tokens_per_task": 9013.23090909091, "last_run_timestamp": "2026-09-26T08:23:24.553Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.7266665, "accuracy_std_dev": 0.020000166668055548, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.46161753000000005, "avg_duration_per_task_ms": 792082.91, "avg_output_tokens_per_task": 9376.13, "last_run_timestamp": "2026-09-26T08:24:06.679Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.7266665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20674873, "avg_duration_per_task_ms": 122950.24, "avg_output_tokens_per_task": 5001.27, "last_run_timestamp": "2026-09-01T13:00:51.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.7266665, "accuracy_std_dev": 0.022607791184677924, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.022968040000000002, "avg_duration_per_task_ms": 247273.5, "avg_output_tokens_per_task": 4817.96, "last_run_timestamp": "2026-09-26T08:11:43.349Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.7264310101010101, "accuracy_std_dev": 0.027838133791617557, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.04808676262626262, "avg_duration_per_task_ms": 349379.2727272727, "avg_output_tokens_per_task": 10200.30303030303, "last_run_timestamp": "2026-08-29T16:33:11.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.7261538461538461, "accuracy_std_dev": 0.03891341591641658, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.04461130461538462, "avg_duration_per_task_ms": 117488.42923076924, "avg_output_tokens_per_task": 8622.856923076923, "last_run_timestamp": "2026-09-26T08:17:01.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.7260000999999999, "accuracy_std_dev": 0.02958217406293864, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.009236828, "avg_duration_per_task_ms": 354448.192, "avg_output_tokens_per_task": 13566.026, "last_run_timestamp": "2026-09-26T08:23:24.666Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.7259998000000001, "accuracy_std_dev": 0.02502449821994441, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.07057247200000001, "avg_duration_per_task_ms": 194517.306, "avg_output_tokens_per_task": 8403.624, "last_run_timestamp": "2026-09-26T08:23:24.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.72583325, "accuracy_std_dev": 0.022714969979894317, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05062893500000001, "avg_duration_per_task_ms": 399514.3275, "avg_output_tokens_per_task": 14703.53, "last_run_timestamp": "2026-08-29T16:13:54.386Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7252010563002681, "accuracy_std_dev": 0.03163209639632926, "run_count": 15, "total_questions": 746, "avg_cost_per_task_usd": 0.1270538310991957, "avg_duration_per_task_ms": 103835.95710455764, "avg_output_tokens_per_task": 9115.029490616622, "last_run_timestamp": "2026-09-26T08:26:14.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.725, "accuracy_std_dev": 0.025000000000000022, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00440039, "avg_duration_per_task_ms": 1062673.02, "avg_output_tokens_per_task": 13355.81, "last_run_timestamp": "2026-09-26T08:07:18.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.7248498828828829, "accuracy_std_dev": 0.09386597077919659, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.07434012837837839, "avg_duration_per_task_ms": 150951.06756756757, "avg_output_tokens_per_task": 13940.813063063064, "last_run_timestamp": "2026-08-29T16:57:02.331Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.7244446666666666, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02107129333333333, "avg_duration_per_task_ms": 190752.33333333334, "avg_output_tokens_per_task": 8303.633333333333, "last_run_timestamp": "2026-09-26T16:00:20.149Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7240537802197802, "accuracy_std_dev": 0.10194063410640787, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.14525350366300366, "avg_duration_per_task_ms": 610254.7802197802, "avg_output_tokens_per_task": 27583.227106227107, "last_run_timestamp": "2026-09-26T08:05:12.733Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.7238232103746397, "accuracy_std_dev": 0.022993657910877263, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.38125144668587896, "avg_duration_per_task_ms": 402514.99711815565, "avg_output_tokens_per_task": 9102.43227665706, "last_run_timestamp": "2026-09-26T08:24:54.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.7237036666666666, "accuracy_std_dev": 0.02535886411318755, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05174337333333333, "avg_duration_per_task_ms": 273069.79333333333, "avg_output_tokens_per_task": 8633.697777777777, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.7236365454545454, "accuracy_std_dev": 0.030664462731500284, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.022053703636363636, "avg_duration_per_task_ms": 116837.80363636363, "avg_output_tokens_per_task": 6700.861818181818, "last_run_timestamp": "2026-09-26T16:00:25.697Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7234432490842492, "accuracy_std_dev": 0.039259986936420826, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.03470968315018315, "avg_duration_per_task_ms": 178672.7087912088, "avg_output_tokens_per_task": 8952.778388278388, "last_run_timestamp": "2026-09-26T08:24:54.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02063872, "avg_duration_per_task_ms": 638594.61, "avg_output_tokens_per_task": 13867.61, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.7233335, "accuracy_std_dev": 0.016666800000300002, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.052515885000000005, "avg_duration_per_task_ms": 281245.84, "avg_output_tokens_per_task": 8472.745, "last_run_timestamp": "2026-08-01T17:38:23.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7233335, "accuracy_std_dev": 0.06333349999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02959498, "avg_duration_per_task_ms": 615820.2, "avg_output_tokens_per_task": 25536.79, "last_run_timestamp": "2026-09-26T08:17:01.598Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09717974, "avg_duration_per_task_ms": 146982.06, "avg_output_tokens_per_task": 51201.48, "last_run_timestamp": "2026-09-01T17:23:14.324Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.722508731958763, "accuracy_std_dev": 0.02841921157751388, "run_count": 4, "total_questions": 194, "avg_cost_per_task_usd": 0.14483782474226806, "avg_duration_per_task_ms": 621637.6597938144, "avg_output_tokens_per_task": 28411.036082474227, "last_run_timestamp": "2026-08-22T08:19:48.101Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.722499875, "accuracy_std_dev": 0.03558044972817764, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.29646921, "avg_duration_per_task_ms": 366904.835, "avg_output_tokens_per_task": 9370.25, "last_run_timestamp": "2026-09-26T08:26:14.256Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.1329602666666667, "avg_duration_per_task_ms": 430783.74, "avg_output_tokens_per_task": 33596.9, "last_run_timestamp": "2026-09-26T08:22:24.836Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.7222221428571428, "accuracy_std_dev": 0.024312166707227047, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.046834047619047615, "avg_duration_per_task_ms": 58108.6462585034, "avg_output_tokens_per_task": 13243.625850340137, "last_run_timestamp": "2026-09-29T21:00:05.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.722222, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.019033145833333334, "avg_duration_per_task_ms": 65993.54166666667, "avg_output_tokens_per_task": 3355.9791666666665, "last_run_timestamp": "2026-09-12T08:12:22.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.7220459047619048, "accuracy_std_dev": 0.03375849235312024, "run_count": 19, "total_questions": 945, "avg_cost_per_task_usd": 0.11261768253968255, "avg_duration_per_task_ms": 146602.55555555556, "avg_output_tokens_per_task": 8336.625396825397, "last_run_timestamp": "2026-09-26T08:14:43.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7214815555555555, "accuracy_std_dev": 0.022394489084549912, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05446545555555556, "avg_duration_per_task_ms": 331631.70444444445, "avg_output_tokens_per_task": 11480.095555555556, "last_run_timestamp": "2026-08-29T17:03:20.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.7214813333333334, "accuracy_std_dev": 0.02490029941363937, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06093124888888888, "avg_duration_per_task_ms": 320605.1911111111, "avg_output_tokens_per_task": 9726.764444444445, "last_run_timestamp": "2026-08-30T00:01:02.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7211113333333333, "accuracy_std_dev": 0.019309173922142694, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04666514, "avg_duration_per_task_ms": 136735.63333333333, "avg_output_tokens_per_task": 27417.226666666666, "last_run_timestamp": "2026-09-26T08:23:22.508Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.7211113333333333, "accuracy_std_dev": 0.011331263428624743, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012200653333333334, "avg_duration_per_task_ms": 38919.36666666667, "avg_output_tokens_per_task": 3634.8866666666668, "last_run_timestamp": "2026-09-26T08:23:23.813Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.7210882448979592, "accuracy_std_dev": 0.0076780598851532815, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.009489710204081632, "avg_duration_per_task_ms": 145499.9755102041, "avg_output_tokens_per_task": 16260.81224489796, "last_run_timestamp": "2026-09-26T08:06:43.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.720833375, "accuracy_std_dev": 0.01898462645759391, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.048760994999999994, "avg_duration_per_task_ms": 383295.01, "avg_output_tokens_per_task": 9253.3225, "last_run_timestamp": "2026-08-29T16:11:23.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.7207407777777778, "accuracy_std_dev": 0.023819327953845365, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.058247844444444453, "avg_duration_per_task_ms": 312927.7555555556, "avg_output_tokens_per_task": 10357.577777777778, "last_run_timestamp": "2026-08-30T00:01:09.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.720606, "accuracy_std_dev": 0.021919006936695594, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07318695090909091, "avg_duration_per_task_ms": 256701.00727272726, "avg_output_tokens_per_task": 8429.969090909091, "last_run_timestamp": "2026-09-26T08:05:12.739Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.72, "accuracy_std_dev": 0.0354337331437149, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.330923255, "avg_duration_per_task_ms": 203933.605, "avg_output_tokens_per_task": 10272.2525, "last_run_timestamp": "2026-09-26T08:10:43.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Io Net", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12671924, "avg_duration_per_task_ms": 566714.78, "avg_output_tokens_per_task": 16714.02, "last_run_timestamp": "2026-09-28T18:00:05.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0037487199999999997, "avg_duration_per_task_ms": 165993.86, "avg_output_tokens_per_task": 15330.68, "last_run_timestamp": "2026-09-23T18:00:03.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15154258, "avg_duration_per_task_ms": 105755.46, "avg_output_tokens_per_task": 13133.32, "last_run_timestamp": "2026-08-04T03:12:01.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.72, "accuracy_std_dev": 0.01845912401316305, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13026846666666667, "avg_duration_per_task_ms": 109723.80666666667, "avg_output_tokens_per_task": 4025.5433333333335, "last_run_timestamp": "2026-09-01T13:00:52.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14489558, "avg_duration_per_task_ms": 248203.88, "avg_output_tokens_per_task": 4971.24, "last_run_timestamp": "2026-09-23T15:00:07.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.402901, "avg_duration_per_task_ms": 196016.4, "avg_output_tokens_per_task": 16256.52, "last_run_timestamp": "2026-09-27T00:00:03.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.028463379999999996, "avg_duration_per_task_ms": 583244.64, "avg_output_tokens_per_task": 43866.54, "last_run_timestamp": "2026-08-08T08:27:52.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.72, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.037352619999999996, "avg_duration_per_task_ms": 206767.2, "avg_output_tokens_per_task": 11957.14, "last_run_timestamp": "2026-09-26T08:10:43.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7192983846153846, "accuracy_std_dev": 0.06419322678515234, "run_count": 10, "total_questions": 494, "avg_cost_per_task_usd": 0.14071399392712552, "avg_duration_per_task_ms": 1258098.6842105263, "avg_output_tokens_per_task": 26322.05870445344, "last_run_timestamp": "2026-09-26T08:12:28.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.719166875, "accuracy_std_dev": 0.03169958486493749, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0506060625, "avg_duration_per_task_ms": 962184.9925, "avg_output_tokens_per_task": 8508.9975, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.7191666250000001, "accuracy_std_dev": 0.03978245617711878, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06181229, "avg_duration_per_task_ms": 550251.335, "avg_output_tokens_per_task": 13930.4, "last_run_timestamp": "2026-08-29T17:14:29.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.7191666250000001, "accuracy_std_dev": 0.020666060140587385, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0268572725, "avg_duration_per_task_ms": 236107.135, "avg_output_tokens_per_task": 10157.88, "last_run_timestamp": "2026-08-29T17:16:15.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.7190475714285715, "accuracy_std_dev": 0.02815552369991236, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3010150485714286, "avg_duration_per_task_ms": 421383.48, "avg_output_tokens_per_task": 9299.894285714287, "last_run_timestamp": "2026-09-26T08:07:18.968Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7189780218978102, "accuracy_std_dev": 0.03683748767175827, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.10041807299270071, "avg_duration_per_task_ms": 119245.32116788322, "avg_output_tokens_per_task": 9093.193430656935, "last_run_timestamp": "2026-09-26T08:22:36.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.7188806165413533, "accuracy_std_dev": 0.09675689930043425, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.16023803007518797, "avg_duration_per_task_ms": 435488.2756892231, "avg_output_tokens_per_task": 22759.51127819549, "last_run_timestamp": "2026-08-29T17:19:03.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.7188550454545455, "accuracy_std_dev": 0.03234119038253501, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.0716665202020202, "avg_duration_per_task_ms": 390250.71717171714, "avg_output_tokens_per_task": 11350.588383838383, "last_run_timestamp": "2026-08-29T16:47:08.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.7186897976878613, "accuracy_std_dev": 0.027015736117292515, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.02169187283236994, "avg_duration_per_task_ms": 314303.23988439306, "avg_output_tokens_per_task": 7575.85549132948, "last_run_timestamp": "2026-08-29T16:48:46.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7185185555555555, "accuracy_std_dev": 0.01771600069059424, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.10858120444444444, "avg_duration_per_task_ms": 333763.2088888889, "avg_output_tokens_per_task": 15783.317777777778, "last_run_timestamp": "2026-08-29T17:17:21.072Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.7185184444444445, "accuracy_std_dev": 0.03648484962925207, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06869702888888889, "avg_duration_per_task_ms": 618804.8311111111, "avg_output_tokens_per_task": 23662.906666666666, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.7183906609195403, "accuracy_std_dev": 0.025974738950584678, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.06343264655172413, "avg_duration_per_task_ms": 497586.9425287356, "avg_output_tokens_per_task": 14530.419540229885, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7177420000000001, "accuracy_std_dev": 0.05717499919930039, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.2799884032258065, "avg_duration_per_task_ms": 377640.3185483871, "avg_output_tokens_per_task": 9988.213709677419, "last_run_timestamp": "2026-09-26T08:10:43.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.717499875, "accuracy_std_dev": 0.027474714386311185, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.35365771499999993, "avg_duration_per_task_ms": 238504.995, "avg_output_tokens_per_task": 10803.1675, "last_run_timestamp": "2026-09-26T08:10:56.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.7173334, "accuracy_std_dev": 0.025508314986294175, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.327251336, "avg_duration_per_task_ms": 427459.224, "avg_output_tokens_per_task": 9077.78, "last_run_timestamp": "2026-09-26T08:23:18.418Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.7173334, "accuracy_std_dev": 0.050526018820405805, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.171608576, "avg_duration_per_task_ms": 290919.704, "avg_output_tokens_per_task": 9956.296, "last_run_timestamp": "2026-09-26T08:22:17.301Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.717083375, "accuracy_std_dev": 0.015629102796845852, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.03730813999999999, "avg_duration_per_task_ms": 202282.19375, "avg_output_tokens_per_task": 10076.16875, "last_run_timestamp": "2026-08-29T16:54:26.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.716667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015077799999999999, "avg_duration_per_task_ms": 476878.72, "avg_output_tokens_per_task": 26097.54, "last_run_timestamp": "2026-09-28T09:00:02.615Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.7166667999999999, "accuracy_std_dev": 0.03303193740851419, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.11440428, "avg_duration_per_task_ms": 101112.032, "avg_output_tokens_per_task": 9922.216, "last_run_timestamp": "2026-09-26T08:27:14.783Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17924758000000002, "avg_duration_per_task_ms": 326369.44, "avg_output_tokens_per_task": 9562.9, "last_run_timestamp": "2026-09-28T18:00:05.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7162963333333333, "accuracy_std_dev": 0.030325222992676492, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.12511306444444442, "avg_duration_per_task_ms": 317088.20666666667, "avg_output_tokens_per_task": 9945.955555555556, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.716111, "accuracy_std_dev": 0.031056479812324726, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.006504103333333333, "avg_duration_per_task_ms": 381723.9266666667, "avg_output_tokens_per_task": 12559.69, "last_run_timestamp": "2026-09-26T08:05:12.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7155646528925619, "accuracy_std_dev": 0.07922982153103719, "run_count": 5, "total_questions": 242, "avg_cost_per_task_usd": 0.11734726446280992, "avg_duration_per_task_ms": 254637.0785123967, "avg_output_tokens_per_task": 42006.561983471074, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7155556666666666, "accuracy_std_dev": 0.040855070557058414, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012743326666666667, "avg_duration_per_task_ms": 1224143.2066666668, "avg_output_tokens_per_task": 18669.653333333332, "last_run_timestamp": "2026-08-29T09:20:05.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.715488212121212, "accuracy_std_dev": 0.013403360380889571, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04391666666666666, "avg_duration_per_task_ms": 58944.35858585859, "avg_output_tokens_per_task": 8110.530303030303, "last_run_timestamp": "2026-08-08T08:31:07.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.715238, "accuracy_std_dev": 0.036811758695907394, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0628034857142857, "avg_duration_per_task_ms": 75005.29714285715, "avg_output_tokens_per_task": 9080.965714285714, "last_run_timestamp": "2026-09-26T08:24:48.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.715, "accuracy_std_dev": 0.04499999999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.16922785, "avg_duration_per_task_ms": 299708.78, "avg_output_tokens_per_task": 9901.85, "last_run_timestamp": "2026-08-29T08:38:56.982Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.7144443333333333, "accuracy_std_dev": 0.0015716626723173276, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04586062666666667, "avg_duration_per_task_ms": 212784.84, "avg_output_tokens_per_task": 30659.346666666668, "last_run_timestamp": "2026-09-26T08:22:27.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.7142854897959183, "accuracy_std_dev": 0.039861000000000035, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.11979946938775511, "avg_duration_per_task_ms": 677914.6734693878, "avg_output_tokens_per_task": 54287.663265306124, "last_run_timestamp": "2026-09-01T17:30:15.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.7140255173041894, "accuracy_std_dev": 0.0345616737838867, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.06628586703096538, "avg_duration_per_task_ms": 272000.2349726776, "avg_output_tokens_per_task": 8738.27504553734, "last_run_timestamp": "2026-09-26T08:25:04.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.7133335, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.028499629999999998, "avg_duration_per_task_ms": 176126.96, "avg_output_tokens_per_task": 22001.83, "last_run_timestamp": "2026-09-26T08:08:19.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7133335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06446003, "avg_duration_per_task_ms": 348420.47, "avg_output_tokens_per_task": 34152.4, "last_run_timestamp": "2026-09-01T12:41:25.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.7133333333333334, "accuracy_std_dev": 0.04268744285956807, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04181986333333333, "avg_duration_per_task_ms": 299500.48, "avg_output_tokens_per_task": 13842.47, "last_run_timestamp": "2026-09-26T08:13:28.082Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.7133333333333334, "accuracy_std_dev": 0.019625871060640555, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12069053999999999, "avg_duration_per_task_ms": 78204.86666666667, "avg_output_tokens_per_task": 4256.786666666667, "last_run_timestamp": "2026-09-01T12:59:40.225Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7133330000000001, "accuracy_std_dev": 0.02828427124746193, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.031477000000000005, "avg_duration_per_task_ms": 184080.18666666668, "avg_output_tokens_per_task": 16016.74, "last_run_timestamp": "2026-09-26T08:01:16.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.7133330000000001, "accuracy_std_dev": 0.040000000000000036, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07565915000000001, "avg_duration_per_task_ms": 95308.62, "avg_output_tokens_per_task": 3758.94, "last_run_timestamp": "2026-09-01T13:00:41.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.043553119999999994, "avg_duration_per_task_ms": 152550.44, "avg_output_tokens_per_task": 34185.24, "last_run_timestamp": "2026-09-26T08:22:24.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14862698, "avg_duration_per_task_ms": 506333.72, "avg_output_tokens_per_task": 12832.14, "last_run_timestamp": "2026-09-26T16:00:25.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.57503108, "avg_duration_per_task_ms": 657829.88, "avg_output_tokens_per_task": 51958.96, "last_run_timestamp": "2026-09-01T13:00:31.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0723164, "avg_duration_per_task_ms": 346522.38, "avg_output_tokens_per_task": 59152.72, "last_run_timestamp": "2026-09-12T16:00:24.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.7130758729674798, "accuracy_std_dev": 0.03492932464076567, "run_count": 20, "total_questions": 984, "avg_cost_per_task_usd": 0.09219651422764227, "avg_duration_per_task_ms": 75102.03760162601, "avg_output_tokens_per_task": 7773.928861788618, "last_run_timestamp": "2026-09-26T08:12:44.279Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.712, "accuracy_std_dev": 0.03249611258904671, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.08037907, "avg_duration_per_task_ms": 74455.182, "avg_output_tokens_per_task": 8504.276, "last_run_timestamp": "2026-09-26T08:11:40.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.712, "accuracy_std_dev": 0.023626914974240687, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.34044856000000007, "avg_duration_per_task_ms": 188002.144, "avg_output_tokens_per_task": 11667.488, "last_run_timestamp": "2026-09-26T08:22:45.792Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7117119324324325, "accuracy_std_dev": 0.01247848271581482, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.6902402702702702, "avg_duration_per_task_ms": 688360.5945945946, "avg_output_tokens_per_task": 56108.03378378379, "last_run_timestamp": "2026-09-01T13:02:58.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.7116669999999999, "accuracy_std_dev": 0.008660254037844347, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.013152875000000001, "avg_duration_per_task_ms": 1125576.36, "avg_output_tokens_per_task": 15885.82, "last_run_timestamp": "2026-09-26T08:11:57.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.7115385708502023, "accuracy_std_dev": 0.03244045710975723, "run_count": 10, "total_questions": 494, "avg_cost_per_task_usd": 0.021128163967611337, "avg_duration_per_task_ms": 99825.94129554655, "avg_output_tokens_per_task": 6537.080971659919, "last_run_timestamp": "2026-09-26T16:00:20.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.7114285714285714, "accuracy_std_dev": 0.0324087457766623, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.034869005714285715, "avg_duration_per_task_ms": 406092.0742857143, "avg_output_tokens_per_task": 5418.051428571429, "last_run_timestamp": "2026-09-30T00:35:04.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7111871198630138, "accuracy_std_dev": 0.09721728114340919, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.03006977739726028, "avg_duration_per_task_ms": 1645574.0684931506, "avg_output_tokens_per_task": 44781.23630136986, "last_run_timestamp": "2026-09-26T16:00:24.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.711111, "accuracy_std_dev": 0.016629528997138404, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03228578, "avg_duration_per_task_ms": 111202.65333333334, "avg_output_tokens_per_task": 22255.406666666666, "last_run_timestamp": "2026-09-26T08:17:01.712Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.711111, "accuracy_std_dev": 0.07430303651040564, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.040132593333333334, "avg_duration_per_task_ms": 322574.2266666667, "avg_output_tokens_per_task": 8636.586666666666, "last_run_timestamp": "2026-07-18T16:00:10.793Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.7106666, "accuracy_std_dev": 0.1560397473582933, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.0609269, "avg_duration_per_task_ms": 167712.206, "avg_output_tokens_per_task": 20248.218, "last_run_timestamp": "2026-09-26T16:00:25.686Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.7105555, "accuracy_std_dev": 0.03132382736166404, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.10523619666666666, "avg_duration_per_task_ms": 90620.79166666667, "avg_output_tokens_per_task": 10589.016666666666, "last_run_timestamp": "2026-09-26T08:22:18.077Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.71, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.009475860000000001, "avg_duration_per_task_ms": 101117.88, "avg_output_tokens_per_task": 5437.48, "last_run_timestamp": "2026-09-01T17:32:44.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7094593108108108, "accuracy_std_dev": 0.025596061967072654, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.29917547972972974, "avg_duration_per_task_ms": 177661.23648648648, "avg_output_tokens_per_task": 4943.472972972973, "last_run_timestamp": "2026-09-01T13:00:41.783Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.709333, "accuracy_std_dev": 0.034409301068170535, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.23672154, "avg_duration_per_task_ms": 252134.184, "avg_output_tokens_per_task": 20169.304, "last_run_timestamp": "2026-09-26T08:14:47.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7091838979591837, "accuracy_std_dev": 0.022986000000000006, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.021414438775510206, "avg_duration_per_task_ms": 174657.33673469388, "avg_output_tokens_per_task": 20207.65306122449, "last_run_timestamp": "2026-09-26T08:07:31.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.7091665, "accuracy_std_dev": 0.012774768970513724, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01202336, "avg_duration_per_task_ms": 319678.925, "avg_output_tokens_per_task": 17962.82, "last_run_timestamp": "2026-09-26T08:14:04.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.708889, "accuracy_std_dev": 0.019115994210782406, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01979013333333333, "avg_duration_per_task_ms": 164929.67333333334, "avg_output_tokens_per_task": 14302.1, "last_run_timestamp": "2026-09-26T08:17:02.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7088886666666667, "accuracy_std_dev": 0.0339208519579853, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08842117999999999, "avg_duration_per_task_ms": 151404.06666666668, "avg_output_tokens_per_task": 46789.46333333333, "last_run_timestamp": "2026-09-01T17:31:47.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7088886666666667, "accuracy_std_dev": 0.025724129999325985, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.062267260000000005, "avg_duration_per_task_ms": 262899.92, "avg_output_tokens_per_task": 8890.053333333333, "last_run_timestamp": "2026-09-26T08:07:15.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.7087542626262627, "accuracy_std_dev": 0.01740072369903045, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.09113491919191918, "avg_duration_per_task_ms": 1081759.01010101, "avg_output_tokens_per_task": 17133.217171717173, "last_run_timestamp": "2026-08-01T17:38:07.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.708333375, "accuracy_std_dev": 0.028037521983662796, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04000606, "avg_duration_per_task_ms": 322622.4375, "avg_output_tokens_per_task": 9103.8975, "last_run_timestamp": "2026-08-29T16:33:11.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.708, "accuracy_std_dev": 0.024367428169587355, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.117385436, "avg_duration_per_task_ms": 671239.06, "avg_output_tokens_per_task": 25704.02, "last_run_timestamp": "2026-09-26T08:22:25.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.7077728430493274, "accuracy_std_dev": 0.03674680352169054, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.0593315, "avg_duration_per_task_ms": 484083.41704035876, "avg_output_tokens_per_task": 14009.72197309417, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7077207463235294, "accuracy_std_dev": 0.07487438084226404, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.04842610477941176, "avg_duration_per_task_ms": 275262.8474264706, "avg_output_tokens_per_task": 27885.965073529413, "last_run_timestamp": "2026-09-26T16:00:24.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.707206945945946, "accuracy_std_dev": 0.06900418229514953, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.23642254054054054, "avg_duration_per_task_ms": 438110.88513513515, "avg_output_tokens_per_task": 23582.5, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openrouter/auto-beta", "model_slug": "openrouter/auto-beta", "model_name": "Auto Router (Beta)", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2407015, "avg_duration_per_task_ms": 149076.32, "avg_output_tokens_per_task": 17938.72, "last_run_timestamp": "2026-07-22T00:25:39.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.3541182, "avg_duration_per_task_ms": 1646927.84, "avg_output_tokens_per_task": 24534.74, "last_run_timestamp": "2026-09-10T00:00:02.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02431272, "avg_duration_per_task_ms": 304327.62, "avg_output_tokens_per_task": 14297.9, "last_run_timestamp": "2026-09-12T08:11:25.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08614948, "avg_duration_per_task_ms": 52133.28, "avg_output_tokens_per_task": 7649.3, "last_run_timestamp": "2026-07-18T08:04:46.262Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00934721, "avg_duration_per_task_ms": 103901.71, "avg_output_tokens_per_task": 5395.91, "last_run_timestamp": "2026-09-01T17:30:15.713Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1838298, "avg_duration_per_task_ms": 329811.9, "avg_output_tokens_per_task": 9540.84, "last_run_timestamp": "2026-08-15T00:16:18.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06360505, "avg_duration_per_task_ms": 327814.04, "avg_output_tokens_per_task": 33622.71, "last_run_timestamp": "2026-09-01T13:01:01.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.042424746666666666, "avg_duration_per_task_ms": 131868.91333333333, "avg_output_tokens_per_task": 15613.653333333334, "last_run_timestamp": "2026-09-26T08:23:01.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7063491754385965, "accuracy_std_dev": 0.03950815373122365, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.13163428320802006, "avg_duration_per_task_ms": 73963.29824561403, "avg_output_tokens_per_task": 10257.41604010025, "last_run_timestamp": "2026-08-29T17:11:32.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.706237370221328, "accuracy_std_dev": 0.032130056570911926, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.37140224144869216, "avg_duration_per_task_ms": 217046.72837022133, "avg_output_tokens_per_task": 8219.961770623742, "last_run_timestamp": "2026-09-26T08:16:17.130Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.705833375, "accuracy_std_dev": 0.13289250276721548, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.03976693, "avg_duration_per_task_ms": 263266.415, "avg_output_tokens_per_task": 10599.815, "last_run_timestamp": "2026-08-29T17:02:01.052Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7055556666666667, "accuracy_std_dev": 0.03430073605462263, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.038495376666666664, "avg_duration_per_task_ms": 114809.37333333334, "avg_output_tokens_per_task": 5174.493333333333, "last_run_timestamp": "2026-09-01T17:31:46.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.7044674329896906, "accuracy_std_dev": 0.02710499999999999, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.11629732989690722, "avg_duration_per_task_ms": 255742.56701030929, "avg_output_tokens_per_task": 22402.164948453606, "last_run_timestamp": "2026-07-18T08:00:15.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.704, "accuracy_std_dev": 0.02370180490173691, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.008245141999999999, "avg_duration_per_task_ms": 308896.162, "avg_output_tokens_per_task": 13728.714, "last_run_timestamp": "2026-09-26T08:22:46.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7039998000000001, "accuracy_std_dev": 0.028782815862246673, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.059492744, "avg_duration_per_task_ms": 531413.344, "avg_output_tokens_per_task": 14170.264, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7037787044534413, "accuracy_std_dev": 0.09864976994620919, "run_count": 10, "total_questions": 494, "avg_cost_per_task_usd": 0.16661966194331984, "avg_duration_per_task_ms": 961787.6214574899, "avg_output_tokens_per_task": 43098.3036437247, "last_run_timestamp": "2026-09-26T08:11:26.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.7029631111111111, "accuracy_std_dev": 0.042758122148362894, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06687537777777779, "avg_duration_per_task_ms": 328702.3044444444, "avg_output_tokens_per_task": 9886.144444444444, "last_run_timestamp": "2026-08-30T00:01:09.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.4077153666666667, "avg_duration_per_task_ms": 153641.3, "avg_output_tokens_per_task": 23369.493333333332, "last_run_timestamp": "2026-09-01T17:32:03.083Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.701666625, "accuracy_std_dev": 0.02598062579566502, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04259596, "avg_duration_per_task_ms": 192678.315, "avg_output_tokens_per_task": 8418.055, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.7000002, "accuracy_std_dev": 0.02828436552868031, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.142434276, "avg_duration_per_task_ms": 459844.396, "avg_output_tokens_per_task": 11247.26, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.31292506, "avg_duration_per_task_ms": 261751.42, "avg_output_tokens_per_task": 14948.8, "last_run_timestamp": "2026-08-01T08:06:08.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.9826790200000001, "avg_duration_per_task_ms": 401624.28, "avg_output_tokens_per_task": 9843.34, "last_run_timestamp": "2026-08-26T00:00:04.572Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": 0.01440159456449179, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.017548586666666668, "avg_duration_per_task_ms": 296799.9066666667, "avg_output_tokens_per_task": 10272.013333333334, "last_run_timestamp": "2026-08-01T17:39:06.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.7, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00948338, "avg_duration_per_task_ms": 87248.24, "avg_output_tokens_per_task": 5625.9, "last_run_timestamp": "2026-09-26T08:10:43.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0112321, "avg_duration_per_task_ms": 596563.34, "avg_output_tokens_per_task": 4097.46, "last_run_timestamp": "2026-09-12T09:21:58.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.6986666, "accuracy_std_dev": 0.007483243510671025, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.04888268, "avg_duration_per_task_ms": 227205.572, "avg_output_tokens_per_task": 6367.684, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.6983967174348699, "accuracy_std_dev": 0.025880910808740845, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.03737082565130261, "avg_duration_per_task_ms": 243092.8116232465, "avg_output_tokens_per_task": 9769.018036072144, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.69833325, "accuracy_std_dev": 0.009859949857250788, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.14240881, "avg_duration_per_task_ms": 514302.935, "avg_output_tokens_per_task": 58927.42, "last_run_timestamp": "2026-09-01T17:31:48.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.698333, "accuracy_std_dev": 0.02291287847477922, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05212012666666667, "avg_duration_per_task_ms": 385762.69666666666, "avg_output_tokens_per_task": 9391.48, "last_run_timestamp": "2026-08-22T16:05:40.004Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.698312293670886, "accuracy_std_dev": 0.0558284276286228, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.057643389873417716, "avg_duration_per_task_ms": 901330.3670886076, "avg_output_tokens_per_task": 61107.00506329114, "last_run_timestamp": "2026-09-26T16:00:25.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.6974275704697986, "accuracy_std_dev": 0.019759375021746, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.020142030201342278, "avg_duration_per_task_ms": 437469.0536912752, "avg_output_tokens_per_task": 10039.570469798658, "last_run_timestamp": "2026-08-29T16:32:40.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.6966669999999999, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12995234, "avg_duration_per_task_ms": 694377.26, "avg_output_tokens_per_task": 58360.09, "last_run_timestamp": "2026-09-01T17:23:14.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.69666675, "accuracy_std_dev": 0.019720195515955205, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.27175437, "avg_duration_per_task_ms": 364436.225, "avg_output_tokens_per_task": 35416.46, "last_run_timestamp": "2026-09-01T13:04:22.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03611107, "avg_duration_per_task_ms": 366038.43, "avg_output_tokens_per_task": 14231.82, "last_run_timestamp": "2026-07-18T16:00:09.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.6965619147982063, "accuracy_std_dev": 0.03306560984534301, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.025591688340807175, "avg_duration_per_task_ms": 227729.04260089685, "avg_output_tokens_per_task": 9657.32735426009, "last_run_timestamp": "2026-08-29T16:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.6962962222222222, "accuracy_std_dev": 0.07160319018774045, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05209685777777778, "avg_duration_per_task_ms": 312714.1022222222, "avg_output_tokens_per_task": 8152.073333333334, "last_run_timestamp": "2026-08-29T16:33:36.567Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.695652, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.32337578260869565, "avg_duration_per_task_ms": 302938.26086956525, "avg_output_tokens_per_task": 52094.65217391304, "last_run_timestamp": "2026-07-04T08:06:25.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.6945686116071429, "accuracy_std_dev": 0.2043857173195383, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.06466310491071428, "avg_duration_per_task_ms": 392031.1339285714, "avg_output_tokens_per_task": 8293.89732142857, "last_run_timestamp": "2026-08-29T17:36:31.188Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.694166625, "accuracy_std_dev": 0.06141286114475023, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.3036356975, "avg_duration_per_task_ms": 394520.48, "avg_output_tokens_per_task": 9370.85, "last_run_timestamp": "2026-09-26T08:08:19.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.6936023333333333, "accuracy_std_dev": 0.000481077111802244, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.08194677777777779, "avg_duration_per_task_ms": 693284.398989899, "avg_output_tokens_per_task": 14499.550505050505, "last_run_timestamp": "2026-08-01T17:38:16.661Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020424833333333333, "avg_duration_per_task_ms": 454611.69333333336, "avg_output_tokens_per_task": 10363.686666666666, "last_run_timestamp": "2026-08-01T17:38:51.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6933331428571429, "accuracy_std_dev": 0.02519760633715926, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.11111235000000001, "avg_duration_per_task_ms": 77728.44714285714, "avg_output_tokens_per_task": 9319.027142857143, "last_run_timestamp": "2026-09-26T08:24:41.403Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015592, "avg_duration_per_task_ms": 1084747.56, "avg_output_tokens_per_task": 17745.48, "last_run_timestamp": "2026-07-31T03:44:20.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02187568, "avg_duration_per_task_ms": 236451.76, "avg_output_tokens_per_task": 11760.74, "last_run_timestamp": "2026-07-18T08:04:03.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6921965086705202, "accuracy_std_dev": 0.017490253563464578, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.02071824566473988, "avg_duration_per_task_ms": 811870.6213872832, "avg_output_tokens_per_task": 19057.85838150289, "last_run_timestamp": "2026-08-29T16:48:06.159Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.69166675, "accuracy_std_dev": 0.015184321229067198, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.043835795, "avg_duration_per_task_ms": 455940.16, "avg_output_tokens_per_task": 10146.67, "last_run_timestamp": "2026-08-01T17:38:21.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6909547336683417, "accuracy_std_dev": 0.03444369764353668, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.045872309045226135, "avg_duration_per_task_ms": 459922.040201005, "avg_output_tokens_per_task": 9154.939698492462, "last_run_timestamp": "2026-08-29T16:33:38.223Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6900003333333334, "accuracy_std_dev": 0.012472191289246483, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04195019333333334, "avg_duration_per_task_ms": 246231.66666666666, "avg_output_tokens_per_task": 31170.286666666667, "last_run_timestamp": "2026-09-24T09:00:05.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.69, "accuracy_std_dev": 0.032088096328701056, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.18925239666666666, "avg_duration_per_task_ms": 1912773.68, "avg_output_tokens_per_task": 13075.366666666667, "last_run_timestamp": "2026-08-29T16:50:22.450Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.68999975, "accuracy_std_dev": 0.03605553586742955, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.113432665, "avg_duration_per_task_ms": 675363.205, "avg_output_tokens_per_task": 27403.155, "last_run_timestamp": "2026-08-15T08:15:24.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.6873795664739885, "accuracy_std_dev": 0.030644933065213705, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.03333745375722543, "avg_duration_per_task_ms": 810141.4335260115, "avg_output_tokens_per_task": 20074.78901734104, "last_run_timestamp": "2026-08-29T17:42:21.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1282767, "avg_duration_per_task_ms": 113186.12, "avg_output_tokens_per_task": 4988.34, "last_run_timestamp": "2026-09-23T15:00:14.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.6866667, "accuracy_std_dev": 0.04862554427057861, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.029370520000000004, "avg_duration_per_task_ms": 173382.156, "avg_output_tokens_per_task": 10540.91, "last_run_timestamp": "2026-08-01T17:38:12.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.6866666666666666, "accuracy_std_dev": 0.005443174461857887, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.009258993333333333, "avg_duration_per_task_ms": 105180.14666666667, "avg_output_tokens_per_task": 8673.2, "last_run_timestamp": "2026-09-23T06:00:08.557Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6866663333333333, "accuracy_std_dev": 0.024944382578492918, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013524920000000001, "avg_duration_per_task_ms": 73418.06, "avg_output_tokens_per_task": 5390.6866666666665, "last_run_timestamp": "2026-09-01T17:31:44.289Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.6860002, "accuracy_std_dev": 0.0747409975258559, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.11085599800000001, "avg_duration_per_task_ms": 65942.592, "avg_output_tokens_per_task": 8834.634, "last_run_timestamp": "2026-09-26T08:22:29.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.6858332500000001, "accuracy_std_dev": 0.009538074291359878, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.11306765, "avg_duration_per_task_ms": 138784.995, "avg_output_tokens_per_task": 8808.785, "last_run_timestamp": "2026-08-01T17:38:46.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.6857144285714285, "accuracy_std_dev": 0.042077516201333307, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.030182771428571425, "avg_duration_per_task_ms": 323904.1228571429, "avg_output_tokens_per_task": 10162.908571428572, "last_run_timestamp": "2026-08-29T16:50:26.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.6857143795918367, "accuracy_std_dev": 0.056091523474496544, "run_count": 10, "total_questions": 490, "avg_cost_per_task_usd": 0.030057089795918368, "avg_duration_per_task_ms": 947954.6224489796, "avg_output_tokens_per_task": 40436.48163265306, "last_run_timestamp": "2026-09-26T16:00:18.485Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.6851852777777777, "accuracy_std_dev": 0.03997262094623929, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.035166996666666665, "avg_duration_per_task_ms": 166784.71777777778, "avg_output_tokens_per_task": 10260.99888888889, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.6850458295739348, "accuracy_std_dev": 0.02015963904958062, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.006078426065162907, "avg_duration_per_task_ms": 231835.2882205514, "avg_output_tokens_per_task": 9389.711779448622, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.685, "accuracy_std_dev": 0.025110843165851677, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0074074375, "avg_duration_per_task_ms": 254406.3375, "avg_output_tokens_per_task": 11400.4, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.683673530612245, "accuracy_std_dev": 0.03777750000000002, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.6821915918367347, "avg_duration_per_task_ms": 803506.224489796, "avg_output_tokens_per_task": 61161.97959183674, "last_run_timestamp": "2026-09-01T12:56:59.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.683333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.05961793, "avg_duration_per_task_ms": 370273.51, "avg_output_tokens_per_task": 10007.47, "last_run_timestamp": "2026-07-18T08:04:08.785Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.6820807398843931, "accuracy_std_dev": 0.018427811270036426, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.015085965317919076, "avg_duration_per_task_ms": 243924.5809248555, "avg_output_tokens_per_task": 11146.326589595375, "last_run_timestamp": "2026-08-30T00:01:09.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DeepInfra", "accuracy": 0.6819729591836734, "accuracy_std_dev": 0.005416499999999991, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.019869846938775512, "avg_duration_per_task_ms": 223014.25510204083, "avg_output_tokens_per_task": 10886.510204081633, "last_run_timestamp": "2026-07-11T15:34:15.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33301777, "avg_duration_per_task_ms": 171524.99, "avg_output_tokens_per_task": 8948.71, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.65240112, "avg_duration_per_task_ms": 90576.24, "avg_output_tokens_per_task": 3843.06, "last_run_timestamp": "2026-09-26T09:24:36.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.68, "accuracy_std_dev": 0, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.088178175, "avg_duration_per_task_ms": 74028.285, "avg_output_tokens_per_task": 3643.675, "last_run_timestamp": "2026-09-19T08:08:39.705Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06724904000000001, "avg_duration_per_task_ms": 102008.58, "avg_output_tokens_per_task": 2510.66, "last_run_timestamp": "2026-08-22T08:23:03.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.6788891666666665, "accuracy_std_dev": 0.08738317986587706, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.057002019999999994, "avg_duration_per_task_ms": 515207.3233333333, "avg_output_tokens_per_task": 18064.933333333334, "last_run_timestamp": "2026-08-29T17:31:24.151Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.6785186666666666, "accuracy_std_dev": 0.019570227381861923, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.00874342888888889, "avg_duration_per_task_ms": 244185.2288888889, "avg_output_tokens_per_task": 10575.206666666667, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.6773809285714286, "accuracy_std_dev": 0.1326003459074697, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.12869523000000002, "avg_duration_per_task_ms": 133454.41285714286, "avg_output_tokens_per_task": 8415.124285714286, "last_run_timestamp": "2026-09-26T08:16:17.531Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6770371111111111, "accuracy_std_dev": 0.2038422884816399, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04420765333333334, "avg_duration_per_task_ms": 387239.0911111111, "avg_output_tokens_per_task": 8342.586666666666, "last_run_timestamp": "2026-08-30T00:01:10.585Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.675675972972973, "accuracy_std_dev": 0.02284471405428887, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04042408108108108, "avg_duration_per_task_ms": 518660.3783783784, "avg_output_tokens_per_task": 20990.527027027027, "last_run_timestamp": "2026-09-26T08:11:26.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.6755553333333334, "accuracy_std_dev": 0.00314285394011385, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09428396666666666, "avg_duration_per_task_ms": 1235635.52, "avg_output_tokens_per_task": 16258.493333333334, "last_run_timestamp": "2026-07-18T16:00:08.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.6744185058139535, "accuracy_std_dev": 0.09615398664493471, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.0726462965116279, "avg_duration_per_task_ms": 198176.1191860465, "avg_output_tokens_per_task": 25038.38081395349, "last_run_timestamp": "2026-08-29T16:10:33.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.6741667499999999, "accuracy_std_dev": 0.04671857083845245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09568332249999999, "avg_duration_per_task_ms": 277611.575, "avg_output_tokens_per_task": 10319.5475, "last_run_timestamp": "2026-08-29T16:43:32.997Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.6740741111111112, "accuracy_std_dev": 0.19675003227188467, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.2889698533333333, "avg_duration_per_task_ms": 475612.9288888889, "avg_output_tokens_per_task": 22340.875555555554, "last_run_timestamp": "2026-08-29T16:58:32.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14733292, "avg_duration_per_task_ms": 130030.77, "avg_output_tokens_per_task": 5038.48, "last_run_timestamp": "2026-09-23T15:13:09.897Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1116277, "avg_duration_per_task_ms": 79582.44, "avg_output_tokens_per_task": 4280.42, "last_run_timestamp": "2026-08-01T08:05:57.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.039595319999999996, "avg_duration_per_task_ms": 304692.26, "avg_output_tokens_per_task": 31675.6, "last_run_timestamp": "2026-09-24T09:00:04.937Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6717170404040403, "accuracy_std_dev": 0.038697517759541104, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.02239533585858586, "avg_duration_per_task_ms": 432351.29545454547, "avg_output_tokens_per_task": 20630.38888888889, "last_run_timestamp": "2026-08-29T16:12:10.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.6707315609756098, "accuracy_std_dev": 0.021850282996794323, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.016168723577235772, "avg_duration_per_task_ms": 272417.73170731706, "avg_output_tokens_per_task": 10469.817073170732, "last_run_timestamp": "2026-08-01T17:38:31.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.6704762857142856, "accuracy_std_dev": 0.09919401452235525, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08830733714285714, "avg_duration_per_task_ms": 86182.73714285715, "avg_output_tokens_per_task": 8229.205714285714, "last_run_timestamp": "2026-08-29T16:33:39.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.67, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03314721, "avg_duration_per_task_ms": 151834.99, "avg_output_tokens_per_task": 17524.36, "last_run_timestamp": "2026-09-01T17:29:39.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.67, "accuracy_std_dev": 0.031446692107437925, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07852493499999999, "avg_duration_per_task_ms": 68196.455, "avg_output_tokens_per_task": 3640.76, "last_run_timestamp": "2026-09-19T08:12:48.272Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.67, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014700359999999999, "avg_duration_per_task_ms": 1773933.08, "avg_output_tokens_per_task": 23989.78, "last_run_timestamp": "2026-07-18T08:04:08.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.6694631409395974, "accuracy_std_dev": 0.06606770680752856, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.024180788590604026, "avg_duration_per_task_ms": 455932.432885906, "avg_output_tokens_per_task": 17012.89932885906, "last_run_timestamp": "2026-08-29T16:44:07.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6681159130434783, "accuracy_std_dev": 0.032598105721055856, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.12360180173913042, "avg_duration_per_task_ms": 117551.10695652173, "avg_output_tokens_per_task": 12432.393913043477, "last_run_timestamp": "2026-09-26T08:11:26.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.668, "accuracy_std_dev": 0.06598329755021341, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.06998024000000001, "avg_duration_per_task_ms": 716919.092, "avg_output_tokens_per_task": 18456.574, "last_run_timestamp": "2026-09-26T08:15:33.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.667619, "accuracy_std_dev": 0.045694607868825315, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.019704325714285712, "avg_duration_per_task_ms": 356821.1142857143, "avg_output_tokens_per_task": 19482.165714285715, "last_run_timestamp": "2026-08-29T17:51:56.583Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.6674998750000001, "accuracy_std_dev": 0.034710984293295034, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11729118250000001, "avg_duration_per_task_ms": 121864.08, "avg_output_tokens_per_task": 12614.4, "last_run_timestamp": "2026-09-26T08:10:43.582Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.6666665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11778575999999999, "avg_duration_per_task_ms": 148387.42, "avg_output_tokens_per_task": 58241.15, "last_run_timestamp": "2026-09-01T17:32:23.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6666663333333335, "accuracy_std_dev": 0.016996731711975913, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020487560000000002, "avg_duration_per_task_ms": 154828.78666666665, "avg_output_tokens_per_task": 15481.913333333334, "last_run_timestamp": "2026-08-29T16:48:14.218Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.6655251643835617, "accuracy_std_dev": 0.050919510401766024, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.0655772808219178, "avg_duration_per_task_ms": 50910.90410958904, "avg_output_tokens_per_task": 3110.958904109589, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.6633333333333333, "accuracy_std_dev": 0.016555205069369833, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01703684, "avg_duration_per_task_ms": 144424.58666666667, "avg_output_tokens_per_task": 21218.593333333334, "last_run_timestamp": "2026-09-26T08:04:53.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6622221666666667, "accuracy_std_dev": 0.0198761101108564, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.31710583, "avg_duration_per_task_ms": 192864.10333333333, "avg_output_tokens_per_task": 8814.376666666667, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.6614868263254112, "accuracy_std_dev": 0.20509144592998726, "run_count": 11, "total_questions": 547, "avg_cost_per_task_usd": 0.09529280438756856, "avg_duration_per_task_ms": 265157.61608775135, "avg_output_tokens_per_task": 8754.009140767825, "last_run_timestamp": "2026-09-26T08:24:45.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.6608527965116279, "accuracy_std_dev": 0.03853886998862316, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.024110087209302324, "avg_duration_per_task_ms": 240569.14244186046, "avg_output_tokens_per_task": 17453.627906976744, "last_run_timestamp": "2026-08-29T17:44:03.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.660833375, "accuracy_std_dev": 0.08579283793962278, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0750140775, "avg_duration_per_task_ms": 256748.24, "avg_output_tokens_per_task": 24550.89, "last_run_timestamp": "2026-08-29T17:00:54.109Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00507128, "avg_duration_per_task_ms": 72920.3, "avg_output_tokens_per_task": 17941.8, "last_run_timestamp": "2026-09-28T18:00:05.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.66, "accuracy_std_dev": 0.01911622670810208, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02949636222222222, "avg_duration_per_task_ms": 349016.0333333333, "avg_output_tokens_per_task": 10678.31111111111, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00819156, "avg_duration_per_task_ms": 91056.4, "avg_output_tokens_per_task": 8317.36, "last_run_timestamp": "2026-09-23T06:00:08.542Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.658750125, "accuracy_std_dev": 0.021339771978851493, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.113134535, "avg_duration_per_task_ms": 123266.8825, "avg_output_tokens_per_task": 12095.085, "last_run_timestamp": "2026-09-26T08:08:19.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.65833325, "accuracy_std_dev": 0.020480390351199366, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.022493505000000004, "avg_duration_per_task_ms": 261094.5125, "avg_output_tokens_per_task": 10075.545, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.6570371111111111, "accuracy_std_dev": 0.027596789142718296, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.029809600000000006, "avg_duration_per_task_ms": 233052.69555555555, "avg_output_tokens_per_task": 11101.757777777777, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.6570371111111111, "accuracy_std_dev": 0.023330540692141164, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.022303715555555555, "avg_duration_per_task_ms": 285586.0155555556, "avg_output_tokens_per_task": 10395.377777777778, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.6566665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.008976079999999999, "avg_duration_per_task_ms": 368613.48, "avg_output_tokens_per_task": 25363.16, "last_run_timestamp": "2026-07-18T16:00:10.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6565315, "accuracy_std_dev": 0.011837568509716087, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.014823912162162164, "avg_duration_per_task_ms": 271525.86486486485, "avg_output_tokens_per_task": 19985.412162162163, "last_run_timestamp": "2026-09-19T08:01:36.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6555556666666666, "accuracy_std_dev": 0.03189478845969805, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023210153333333334, "avg_duration_per_task_ms": 207735.58, "avg_output_tokens_per_task": 33693.153333333335, "last_run_timestamp": "2026-09-26T08:25:43.443Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.655102163265306, "accuracy_std_dev": 0.015608943341559031, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.012813330612244898, "avg_duration_per_task_ms": 278688.9183673469, "avg_output_tokens_per_task": 16201.359183673469, "last_run_timestamp": "2026-08-29T16:33:39.228Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.6549999166666667, "accuracy_std_dev": 0.02926570645242861, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.12179273749999998, "avg_duration_per_task_ms": 120187.11666666667, "avg_output_tokens_per_task": 12516.864166666666, "last_run_timestamp": "2026-09-26T08:13:57.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.6533335, "accuracy_std_dev": 0.053333500000000034, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.22863909999999998, "avg_duration_per_task_ms": 717866.38, "avg_output_tokens_per_task": 11422.01, "last_run_timestamp": "2026-09-26T08:13:25.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00923216, "avg_duration_per_task_ms": 145753.66, "avg_output_tokens_per_task": 8192.38, "last_run_timestamp": "2026-09-23T06:00:08.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.6525, "accuracy_std_dev": 0.054102510725011654, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021760159999999997, "avg_duration_per_task_ms": 206554.95, "avg_output_tokens_per_task": 8661.865, "last_run_timestamp": "2026-08-29T16:37:21.285Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.65166675, "accuracy_std_dev": 0.04506177549972369, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04474220500000001, "avg_duration_per_task_ms": 212693.005, "avg_output_tokens_per_task": 3236.785, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.6514084084507042, "accuracy_std_dev": 0.140951893455415, "run_count": 3, "total_questions": 142, "avg_cost_per_task_usd": 0.04356793661971831, "avg_duration_per_task_ms": 110031.35915492958, "avg_output_tokens_per_task": 6698.830985915493, "last_run_timestamp": "2026-08-01T08:06:01.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.651111, "accuracy_std_dev": 0.09803010371649448, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.010906386666666667, "avg_duration_per_task_ms": 303175.14, "avg_output_tokens_per_task": 4777.48, "last_run_timestamp": "2026-09-26T08:13:57.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.6493332, "accuracy_std_dev": 0.024801501748079685, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.024958387999999998, "avg_duration_per_task_ms": 169231.86, "avg_output_tokens_per_task": 11298.596, "last_run_timestamp": "2026-08-01T17:38:10.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.6481482020202021, "accuracy_std_dev": 0.0054059166891471805, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.009114631313131312, "avg_duration_per_task_ms": 604721.6666666666, "avg_output_tokens_per_task": 19746.29292929293, "last_run_timestamp": "2026-08-22T16:10:30.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6476509932885907, "accuracy_std_dev": 0.013206828848070314, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.018146744966442953, "avg_duration_per_task_ms": 194939.0268456376, "avg_output_tokens_per_task": 22002.395973154362, "last_run_timestamp": "2026-09-26T08:13:57.279Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.6474073333333333, "accuracy_std_dev": 0.028011374332581395, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021467380000000005, "avg_duration_per_task_ms": 230369.31333333332, "avg_output_tokens_per_task": 10678.271111111111, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.6466667777777777, "accuracy_std_dev": 0.17752752803424773, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06551144888888888, "avg_duration_per_task_ms": 209379.12444444443, "avg_output_tokens_per_task": 9860.064444444444, "last_run_timestamp": "2026-08-29T17:03:33.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.6461902857142857, "accuracy_std_dev": 0.024196977005487303, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.022573365714285715, "avg_duration_per_task_ms": 279162.21142857143, "avg_output_tokens_per_task": 11143.031428571428, "last_run_timestamp": "2026-08-15T14:15:19.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.645833375, "accuracy_std_dev": 0.02526261098905604, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11683335, "avg_duration_per_task_ms": 147559.335, "avg_output_tokens_per_task": 12607.1725, "last_run_timestamp": "2026-09-26T08:22:25.227Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.6455556666666666, "accuracy_std_dev": 0.09964144484545016, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11177372666666668, "avg_duration_per_task_ms": 710817.72, "avg_output_tokens_per_task": 87845.5, "last_run_timestamp": "2026-09-29T19:45:03.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.6433335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.034682500000000005, "avg_duration_per_task_ms": 159195.12, "avg_output_tokens_per_task": 18151.16, "last_run_timestamp": "2026-09-01T17:31:43.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.6433333333333333, "accuracy_std_dev": 0.14455548467015086, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10531282, "avg_duration_per_task_ms": 1453976.54, "avg_output_tokens_per_task": 18878.353333333333, "last_run_timestamp": "2026-08-08T08:21:31.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.6429628888888889, "accuracy_std_dev": 0.020872536285455455, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.024601560000000005, "avg_duration_per_task_ms": 160050.17111111112, "avg_output_tokens_per_task": 10753.557777777778, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.6418009435483871, "accuracy_std_dev": 0.15603995157958747, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.04030671370967741, "avg_duration_per_task_ms": 276664.12096774194, "avg_output_tokens_per_task": 11942.516129032258, "last_run_timestamp": "2026-08-29T16:15:12.662Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.64166675, "accuracy_std_dev": 0.00985994985725082, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0329834, "avg_duration_per_task_ms": 238262.615, "avg_output_tokens_per_task": 16910.1, "last_run_timestamp": "2026-09-01T17:31:47.440Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.6415410502512563, "accuracy_std_dev": 0.10999376007351223, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.09234448994974874, "avg_duration_per_task_ms": 238857.3718592965, "avg_output_tokens_per_task": 26390.881909547737, "last_run_timestamp": "2026-08-29T16:12:10.428Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "WandB", "accuracy": 0.64, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08159856, "avg_duration_per_task_ms": 189065.52, "avg_output_tokens_per_task": 12650.4, "last_run_timestamp": "2026-07-04T16:00:23.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.64, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.036191219999999996, "avg_duration_per_task_ms": 572703.13, "avg_output_tokens_per_task": 26589.6, "last_run_timestamp": "2026-07-11T15:34:14.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.64, "accuracy_std_dev": 0.025647213408087847, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.218415728, "avg_duration_per_task_ms": 1219489.072, "avg_output_tokens_per_task": 31443.888, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6397003505617977, "accuracy_std_dev": 0.15332848831846185, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.01602385617977528, "avg_duration_per_task_ms": 268837.0134831461, "avg_output_tokens_per_task": 8576.319101123596, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.6386664000000001, "accuracy_std_dev": 0.04139773692172073, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.02512048, "avg_duration_per_task_ms": 174653.388, "avg_output_tokens_per_task": 11467.556, "last_run_timestamp": "2026-08-01T17:38:10.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.637681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.04568002173913044, "avg_duration_per_task_ms": 70889.17391304347, "avg_output_tokens_per_task": 8168.934782608696, "last_run_timestamp": "2026-08-01T17:41:39.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.6360152471264368, "accuracy_std_dev": 0.13554242758288354, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.04763444827586207, "avg_duration_per_task_ms": 2222096.040229885, "avg_output_tokens_per_task": 71368.38505747127, "last_run_timestamp": "2026-09-26T08:08:19.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.6356786532663317, "accuracy_std_dev": 0.023893543066067047, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.018825286432160804, "avg_duration_per_task_ms": 163374.6633165829, "avg_output_tokens_per_task": 9394.713567839195, "last_run_timestamp": "2026-08-01T17:40:52.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.6333333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07877431333333335, "avg_duration_per_task_ms": 1233888.1733333333, "avg_output_tokens_per_task": 25898.94, "last_run_timestamp": "2026-07-18T16:00:08.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6333332500000001, "accuracy_std_dev": 0.03299835012826401, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.102073965, "avg_duration_per_task_ms": 160807.645, "avg_output_tokens_per_task": 12456.285, "last_run_timestamp": "2026-09-01T17:31:46.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6331093959731544, "accuracy_std_dev": 0.01453205911080741, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.012315053691275168, "avg_duration_per_task_ms": 1064929.322147651, "avg_output_tokens_per_task": 21067.724832214764, "last_run_timestamp": "2026-08-29T17:18:10.895Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6319998, "accuracy_std_dev": 0.0363685871784979, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.09467958400000001, "avg_duration_per_task_ms": 55836.916, "avg_output_tokens_per_task": 3550.436, "last_run_timestamp": "2026-08-29T16:15:31.797Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.6316667499999999, "accuracy_std_dev": 0.030867157124320685, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021079455, "avg_duration_per_task_ms": 289742.9075, "avg_output_tokens_per_task": 22034.425, "last_run_timestamp": "2026-08-29T16:25:26.695Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.6308725570469799, "accuracy_std_dev": 0.09289222799567248, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.06755993288590603, "avg_duration_per_task_ms": 587099.4328859061, "avg_output_tokens_per_task": 38157.05369127517, "last_run_timestamp": "2026-08-29T16:07:02.358Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.6303702222222223, "accuracy_std_dev": 0.02063465195989924, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.029002144444444447, "avg_duration_per_task_ms": 221656.39333333334, "avg_output_tokens_per_task": 10431.462222222222, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.63, "accuracy_std_dev": 0.043332999999999955, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08937484999999999, "avg_duration_per_task_ms": 3066082.31, "avg_output_tokens_per_task": 80584.37, "last_run_timestamp": "2026-09-29T01:28:53.566Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.63, "accuracy_std_dev": 0.023333000000000048, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07672618, "avg_duration_per_task_ms": 231628.06, "avg_output_tokens_per_task": 10580.58, "last_run_timestamp": "2026-08-01T17:40:48.165Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.63, "accuracy_std_dev": 0.050000000000000044, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09203941, "avg_duration_per_task_ms": 133131.18, "avg_output_tokens_per_task": 12511.49, "last_run_timestamp": "2026-09-01T17:23:15.998Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.6295238571428571, "accuracy_std_dev": 0.055874856932839716, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0383867, "avg_duration_per_task_ms": 389225.1657142857, "avg_output_tokens_per_task": 20771.2, "last_run_timestamp": "2026-08-29T16:50:48.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.62946425, "accuracy_std_dev": 0.027834597690954414, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.03242941071428571, "avg_duration_per_task_ms": 84991.91294642857, "avg_output_tokens_per_task": 21655.93526785714, "last_run_timestamp": "2026-08-29T17:17:57.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.625850418367347, "accuracy_std_dev": 0.1514466649820573, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.0701549693877551, "avg_duration_per_task_ms": 396505.55102040817, "avg_output_tokens_per_task": 11691.816326530612, "last_run_timestamp": "2026-08-01T17:38:10.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.622650703264095, "accuracy_std_dev": 0.0769236199560503, "run_count": 7, "total_questions": 337, "avg_cost_per_task_usd": 0.14786436795252222, "avg_duration_per_task_ms": 364083.8011869436, "avg_output_tokens_per_task": 11314.480712166172, "last_run_timestamp": "2026-08-29T16:44:17.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.6224298747663553, "accuracy_std_dev": 0.06407885487325574, "run_count": 11, "total_questions": 535, "avg_cost_per_task_usd": 0.022028269158878504, "avg_duration_per_task_ms": 247156.75327102805, "avg_output_tokens_per_task": 24158.08785046729, "last_run_timestamp": "2026-09-26T16:00:24.609Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.6222782361809045, "accuracy_std_dev": 0.013777638925424605, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.10842231658291457, "avg_duration_per_task_ms": 712075.7864321608, "avg_output_tokens_per_task": 34125.979899497484, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.6211111666666667, "accuracy_std_dev": 0.013005359772246037, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10838059000000001, "avg_duration_per_task_ms": 725500.3766666667, "avg_output_tokens_per_task": 33770.87, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.620246288590604, "accuracy_std_dev": 0.024043751131473293, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.19689856375838927, "avg_duration_per_task_ms": 175265.28187919463, "avg_output_tokens_per_task": 13638.724832214764, "last_run_timestamp": "2026-09-01T13:00:41.540Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.6180555, "accuracy_std_dev": 0.00694450000000002, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.025814302083333334, "avg_duration_per_task_ms": 76875.0625, "avg_output_tokens_per_task": 17301.635416666668, "last_run_timestamp": "2026-08-29T16:33:38.021Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.6176750230547552, "accuracy_std_dev": 0.22454644005875757, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.04794541498559077, "avg_duration_per_task_ms": 857866.4149855907, "avg_output_tokens_per_task": 11712.968299711816, "last_run_timestamp": "2026-08-29T17:09:01.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.6173332, "accuracy_std_dev": 0.03309246574312648, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.09146958399999999, "avg_duration_per_task_ms": 147054.108, "avg_output_tokens_per_task": 3898.968, "last_run_timestamp": "2026-08-01T17:38:24.442Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.6163948885869566, "accuracy_std_dev": 0.028746044099319138, "run_count": 15, "total_questions": 736, "avg_cost_per_task_usd": 0.019574407608695648, "avg_duration_per_task_ms": 331626.9605978261, "avg_output_tokens_per_task": 20514.514945652172, "last_run_timestamp": "2026-08-29T16:49:23.763Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6159998000000001, "accuracy_std_dev": 0.03440923906976152, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045486667999999994, "avg_duration_per_task_ms": 195084.972, "avg_output_tokens_per_task": 3358.316, "last_run_timestamp": "2026-09-19T08:08:42.956Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6158248141414141, "accuracy_std_dev": 0.2285695922821756, "run_count": 10, "total_questions": 495, "avg_cost_per_task_usd": 0.020359361616161618, "avg_duration_per_task_ms": 340617.86262626265, "avg_output_tokens_per_task": 10409.925252525252, "last_run_timestamp": "2026-09-26T08:14:40.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.61401515, "accuracy_std_dev": 0.08680383555799967, "run_count": 9, "total_questions": 440, "avg_cost_per_task_usd": 0.021259502272727274, "avg_duration_per_task_ms": 807630.4068181819, "avg_output_tokens_per_task": 27761.55, "last_run_timestamp": "2026-09-26T16:00:20.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.6133335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.22014136, "avg_duration_per_task_ms": 192160.83, "avg_output_tokens_per_task": 15089.79, "last_run_timestamp": "2026-09-01T12:57:05.201Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.60958325, "accuracy_std_dev": 0.03034222282871678, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06603929000000001, "avg_duration_per_task_ms": 26897.2, "avg_output_tokens_per_task": 20840.3025, "last_run_timestamp": "2026-08-29T16:55:05.300Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.608333, "accuracy_std_dev": 0.0050000000000000044, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03168119, "avg_duration_per_task_ms": 488065.11, "avg_output_tokens_per_task": 10898.63, "last_run_timestamp": "2026-07-18T16:00:09.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.608, "accuracy_std_dev": 0.11488351975457575, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.058990216, "avg_duration_per_task_ms": 582295.388, "avg_output_tokens_per_task": 35784.112, "last_run_timestamp": "2026-08-15T14:08:35.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.6066666666666667, "accuracy_std_dev": 0.15241273398753646, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017056396666666668, "avg_duration_per_task_ms": 233868.72666666665, "avg_output_tokens_per_task": 8308.71, "last_run_timestamp": "2026-08-29T16:33:39.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.6045386517857143, "accuracy_std_dev": 0.02551152635115024, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.019703399553571425, "avg_duration_per_task_ms": 94813.06026785714, "avg_output_tokens_per_task": 4259.450892857143, "last_run_timestamp": "2026-08-29T16:08:08.875Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.604, "accuracy_std_dev": 0.19783738552457672, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.131018624, "avg_duration_per_task_ms": 356662.58, "avg_output_tokens_per_task": 7644.116, "last_run_timestamp": "2026-09-26T08:24:48.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6033335000000001, "accuracy_std_dev": 0.028867224784346696, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04717542499999999, "avg_duration_per_task_ms": 204909.125, "avg_output_tokens_per_task": 3753.575, "last_run_timestamp": "2026-09-19T08:01:34.089Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.6023054178674352, "accuracy_std_dev": 0.20058517615217727, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.010320801152737752, "avg_duration_per_task_ms": 214339.75504322766, "avg_output_tokens_per_task": 8030.982708933718, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.6021870316742081, "accuracy_std_dev": 0.025520253734854576, "run_count": 9, "total_questions": 442, "avg_cost_per_task_usd": 0.04226960180995475, "avg_duration_per_task_ms": 82823.46606334842, "avg_output_tokens_per_task": 26213.19230769231, "last_run_timestamp": "2026-08-29T17:17:47.080Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.6004564109589041, "accuracy_std_dev": 0.09338877055873593, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.485756647260274, "avg_duration_per_task_ms": 2654805.914383562, "avg_output_tokens_per_task": 83086.6404109589, "last_run_timestamp": "2026-09-26T08:17:02.590Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.5993149931506849, "accuracy_std_dev": 0.055256651291194496, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.0717366712328767, "avg_duration_per_task_ms": 725826.4726027397, "avg_output_tokens_per_task": 48647.558219178085, "last_run_timestamp": "2026-08-29T17:09:45.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.5935375204081632, "accuracy_std_dev": 0.22137617923387215, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.04782920408163266, "avg_duration_per_task_ms": 235539.23979591837, "avg_output_tokens_per_task": 7215.7295918367345, "last_run_timestamp": "2026-08-01T17:38:27.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.5933335000000001, "accuracy_std_dev": 0.05333349999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08864457999999999, "avg_duration_per_task_ms": 125444.95, "avg_output_tokens_per_task": 11722.21, "last_run_timestamp": "2026-09-01T17:31:29.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.593333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04552234, "avg_duration_per_task_ms": 32685.39, "avg_output_tokens_per_task": 3125.98, "last_run_timestamp": "2026-09-26T08:22:32.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.5909962011494253, "accuracy_std_dev": 0.028655941146289218, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.16123717816091956, "avg_duration_per_task_ms": 636124.5057471264, "avg_output_tokens_per_task": 30137.091954022988, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.5887097016129033, "accuracy_std_dev": 0.03627146285111756, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.0998679556451613, "avg_duration_per_task_ms": 111335.03629032258, "avg_output_tokens_per_task": 3689.3790322580644, "last_run_timestamp": "2026-08-01T17:38:23.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.5866666666666667, "accuracy_std_dev": 0.0066666666750000055, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029609283333333326, "avg_duration_per_task_ms": 75747.02, "avg_output_tokens_per_task": 8199.643333333333, "last_run_timestamp": "2026-09-01T17:32:36.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.5857141428571428, "accuracy_std_dev": 0.041622460359860544, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.011210711428571428, "avg_duration_per_task_ms": 363659.66, "avg_output_tokens_per_task": 19214.505714285715, "last_run_timestamp": "2026-08-29T16:33:38.124Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.5855555555555556, "accuracy_std_dev": 0.04898963357262684, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04505041111111111, "avg_duration_per_task_ms": 84460.94888888889, "avg_output_tokens_per_task": 4504.664444444445, "last_run_timestamp": "2026-08-29T16:44:17.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.5811856778350516, "accuracy_std_dev": 0.03980730883718937, "run_count": 8, "total_questions": 388, "avg_cost_per_task_usd": 0.07594913402061856, "avg_duration_per_task_ms": 92087.82731958762, "avg_output_tokens_per_task": 5709.461340206186, "last_run_timestamp": "2026-08-29T16:11:10.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.5803722081218273, "accuracy_std_dev": 0.06797104807149512, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.05299261167512691, "avg_duration_per_task_ms": 433958.16751269036, "avg_output_tokens_per_task": 10198.129441624365, "last_run_timestamp": "2026-08-30T00:01:02.157Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.58, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10846708, "avg_duration_per_task_ms": 79571.2, "avg_output_tokens_per_task": 4199.82, "last_run_timestamp": "2026-08-01T08:05:35.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.5766665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.028599199999999998, "avg_duration_per_task_ms": 78841.33, "avg_output_tokens_per_task": 8172.09, "last_run_timestamp": "2026-09-01T17:31:45.195Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.573333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012256119999999999, "avg_duration_per_task_ms": 120534.1, "avg_output_tokens_per_task": 16750, "last_run_timestamp": "2026-08-07T00:00:02.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5726744011627907, "accuracy_std_dev": 0.020765505569357568, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.13094280523255813, "avg_duration_per_task_ms": 1202301.4127906978, "avg_output_tokens_per_task": 25694.97965116279, "last_run_timestamp": "2026-08-29T16:31:42.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.569727612244898, "accuracy_std_dev": 0.026944500000000038, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.0024336428571428573, "avg_duration_per_task_ms": 317617.1530612245, "avg_output_tokens_per_task": 5820.755102040816, "last_run_timestamp": "2026-08-29T16:57:41.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.5648772953020134, "accuracy_std_dev": 0.0424873867714486, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1486826308724832, "avg_duration_per_task_ms": 18002.88590604027, "avg_output_tokens_per_task": 2546.3288590604025, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.5639731717171717, "accuracy_std_dev": 0.09459911998137191, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.04424948484848485, "avg_duration_per_task_ms": 3137109.6868686867, "avg_output_tokens_per_task": 95124.10606060606, "last_run_timestamp": "2026-08-29T17:24:25.648Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5604491347150259, "accuracy_std_dev": 0.10678459157041047, "run_count": 8, "total_questions": 386, "avg_cost_per_task_usd": 0.07120025388601035, "avg_duration_per_task_ms": 510444.22279792745, "avg_output_tokens_per_task": 29300.067357512955, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.5562962222222222, "accuracy_std_dev": 0.09823697755233152, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04116750222222223, "avg_duration_per_task_ms": 96293.60222222222, "avg_output_tokens_per_task": 4303.308888888889, "last_run_timestamp": "2026-08-29T16:43:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.5533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03619409, "avg_duration_per_task_ms": 60452.63, "avg_output_tokens_per_task": 6631.82, "last_run_timestamp": "2026-09-01T17:31:47.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.55, "accuracy_std_dev": 0.036667000000000005, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23514412, "avg_duration_per_task_ms": 166142.51, "avg_output_tokens_per_task": 13675.21, "last_run_timestamp": "2026-09-01T13:01:57.044Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.5382882432432433, "accuracy_std_dev": 0.2423313405275696, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.02003441891891892, "avg_duration_per_task_ms": 116748.24662162163, "avg_output_tokens_per_task": 20889.20608108108, "last_run_timestamp": "2026-08-29T17:06:47.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.5355556666666667, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09782582666666667, "avg_duration_per_task_ms": 375218.6066666667, "avg_output_tokens_per_task": 16827.4, "last_run_timestamp": "2026-08-15T14:16:11.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.5342466164383561, "accuracy_std_dev": 0.014081561995113387, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.1389923630136986, "avg_duration_per_task_ms": 68151.21917808219, "avg_output_tokens_per_task": 2403.2671232876714, "last_run_timestamp": "2026-09-01T13:01:05.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.529239730994152, "accuracy_std_dev": 0.016446009428504185, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.02731367543859649, "avg_duration_per_task_ms": 1767438.9385964912, "avg_output_tokens_per_task": 142501.04678362573, "last_run_timestamp": "2026-08-29T16:33:29.988Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.526667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03416076, "avg_duration_per_task_ms": 1845834.04, "avg_output_tokens_per_task": 105339.92, "last_run_timestamp": "2026-07-18T16:00:08.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.5234740915492958, "accuracy_std_dev": 0.24433860528145954, "run_count": 3, "total_questions": 142, "avg_cost_per_task_usd": 0.12058070422535211, "avg_duration_per_task_ms": 209796.21126760563, "avg_output_tokens_per_task": 11881.387323943662, "last_run_timestamp": "2026-07-18T16:00:09.881Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.5228572857142858, "accuracy_std_dev": 0.021925379532237367, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.028223582857142857, "avg_duration_per_task_ms": 445218.9428571428, "avg_output_tokens_per_task": 57366.497142857144, "last_run_timestamp": "2026-09-26T16:00:18.487Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.5183044971098266, "accuracy_std_dev": 0.03913581029998911, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.1411618005780347, "avg_duration_per_task_ms": 1537742.4248554914, "avg_output_tokens_per_task": 30943.121387283238, "last_run_timestamp": "2026-08-29T16:25:50.849Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5151512828282828, "accuracy_std_dev": 0.04224499999999998, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.026153939393939396, "avg_duration_per_task_ms": 213641.58585858587, "avg_output_tokens_per_task": 54161.38383838384, "last_run_timestamp": "2026-09-01T17:31:30.190Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.5119049598214286, "accuracy_std_dev": 0.02807387800381059, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.06541727232142856, "avg_duration_per_task_ms": 304953.5691964286, "avg_output_tokens_per_task": 4005.4308035714284, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.5104164999999999, "accuracy_std_dev": 0.003472500000000045, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.024820281250000003, "avg_duration_per_task_ms": 480070.4583333333, "avg_output_tokens_per_task": 9578.8125, "last_run_timestamp": "2026-08-08T16:03:43.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.5077973684210526, "accuracy_std_dev": 0.01607029428505164, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.021449652046783625, "avg_duration_per_task_ms": 1262349.283625731, "avg_output_tokens_per_task": 119951.4649122807, "last_run_timestamp": "2026-08-29T17:16:15.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.5072072378378378, "accuracy_std_dev": 0.016622828858597505, "run_count": 15, "total_questions": 740, "avg_cost_per_task_usd": 0.04104409594594594, "avg_duration_per_task_ms": 771120.0337837838, "avg_output_tokens_per_task": 212846.60270270272, "last_run_timestamp": "2026-08-29T16:44:16.523Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.5044443333333334, "accuracy_std_dev": 0.01571379829604824, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.022218646666666668, "avg_duration_per_task_ms": 1815315.58, "avg_output_tokens_per_task": 146379.76666666666, "last_run_timestamp": "2026-07-18T16:00:10.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.5033669747474748, "accuracy_std_dev": 0.027292446564651823, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.025276146464646466, "avg_duration_per_task_ms": 577739.3712121212, "avg_output_tokens_per_task": 53598.469696969696, "last_run_timestamp": "2026-09-26T16:00:25.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.5028571428571429, "accuracy_std_dev": 0.010605205587400014, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01953936, "avg_duration_per_task_ms": 356098.7342857143, "avg_output_tokens_per_task": 10947.26, "last_run_timestamp": "2026-08-29T16:44:22.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.49666650000000007, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.13482986, "avg_duration_per_task_ms": 58783.54, "avg_output_tokens_per_task": 2896.5, "last_run_timestamp": "2026-09-01T13:00:51.538Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.49666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23109727999999996, "avg_duration_per_task_ms": 80168.37, "avg_output_tokens_per_task": 2746.03, "last_run_timestamp": "2026-08-01T17:41:27.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.49523814285714285, "accuracy_std_dev": 0.023289352137395136, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06688620571428572, "avg_duration_per_task_ms": 308997.38285714283, "avg_output_tokens_per_task": 4074.4114285714286, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.493333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14235776, "avg_duration_per_task_ms": 443941.74, "avg_output_tokens_per_task": 8833.66, "last_run_timestamp": "2026-08-06T00:00:02.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.49319713265306125, "accuracy_std_dev": 0.03082350135347216, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.05686764285714286, "avg_duration_per_task_ms": 2675498.8775510206, "avg_output_tokens_per_task": 42954.67857142857, "last_run_timestamp": "2026-08-15T14:12:30.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.4926666, "accuracy_std_dev": 0.017688487155209202, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.033023068, "avg_duration_per_task_ms": 659102.76, "avg_output_tokens_per_task": 61671.012, "last_run_timestamp": "2026-09-26T16:00:20.149Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.4915826313131313, "accuracy_std_dev": 0.024392630759296454, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.023856613636363638, "avg_duration_per_task_ms": 839892.5429292929, "avg_output_tokens_per_task": 22724.651515151516, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.49028390582959647, "accuracy_std_dev": 0.024789823060240374, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.02451580269058296, "avg_duration_per_task_ms": 275409.5044843049, "avg_output_tokens_per_task": 18530.174887892375, "last_run_timestamp": "2026-08-29T16:33:30.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.48857142857142855, "accuracy_std_dev": 0.018418205526498126, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.034540077142857145, "avg_duration_per_task_ms": 44715.205714285716, "avg_output_tokens_per_task": 3650.385714285714, "last_run_timestamp": "2026-08-29T16:42:36.232Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Chutes", "accuracy": 0.48666699999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058167340000000005, "avg_duration_per_task_ms": 169001.24, "avg_output_tokens_per_task": 5056.06, "last_run_timestamp": "2026-07-04T16:00:23.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.48666650000000006, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12261337999999998, "avg_duration_per_task_ms": 50813.54, "avg_output_tokens_per_task": 2655.2, "last_run_timestamp": "2026-09-01T12:53:49.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.48333325000000005, "accuracy_std_dev": 0.025166081671319035, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021045685, "avg_duration_per_task_ms": 161568.115, "avg_output_tokens_per_task": 3811.995, "last_run_timestamp": "2026-08-29T16:46:44.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.4829628888888889, "accuracy_std_dev": 0.029666526102455164, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02247696444444445, "avg_duration_per_task_ms": 720948.6155555556, "avg_output_tokens_per_task": 22403.7, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.48243642152466365, "accuracy_std_dev": 0.023113682722310937, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.028583977578475337, "avg_duration_per_task_ms": 385123.331838565, "avg_output_tokens_per_task": 66962.30269058296, "last_run_timestamp": "2026-09-26T16:00:20.148Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4822222222222222, "accuracy_std_dev": 0.04036851236016555, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05617918666666666, "avg_duration_per_task_ms": 128221.7311111111, "avg_output_tokens_per_task": 3794.2533333333336, "last_run_timestamp": "2026-08-29T16:33:38.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.4805680986547085, "accuracy_std_dev": 0.032856388856151955, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.023212632286995515, "avg_duration_per_task_ms": 56208.64349775785, "avg_output_tokens_per_task": 3761.609865470852, "last_run_timestamp": "2026-08-29T16:12:10.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.47916700000000007, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.04362447916666667, "avg_duration_per_task_ms": 66953.94791666667, "avg_output_tokens_per_task": 2828.96875, "last_run_timestamp": "2026-08-15T14:08:14.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.47716860273972606, "accuracy_std_dev": 0.06078806821700747, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.026057061643835613, "avg_duration_per_task_ms": 2288262.0684931506, "avg_output_tokens_per_task": 135316.74657534246, "last_run_timestamp": "2026-08-29T17:10:29.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.47701135057471267, "accuracy_std_dev": 0.009988678401274195, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.1616835948275862, "avg_duration_per_task_ms": 2419122.9655172415, "avg_output_tokens_per_task": 43666.33620689655, "last_run_timestamp": "2026-08-29T16:20:58.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.47619069387755103, "accuracy_std_dev": 0.04472200000000001, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.042756918367346936, "avg_duration_per_task_ms": 697970.0612244898, "avg_output_tokens_per_task": 99743.38775510204, "last_run_timestamp": "2026-09-01T17:27:02.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.4737037777777778, "accuracy_std_dev": 0.04287349816683645, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0567969311111111, "avg_duration_per_task_ms": 70294.79555555555, "avg_output_tokens_per_task": 3592.2155555555555, "last_run_timestamp": "2026-08-29T17:01:46.593Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.47333325000000004, "accuracy_std_dev": 0.025603927651583064, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02492969, "avg_duration_per_task_ms": 126362.985, "avg_output_tokens_per_task": 3798.445, "last_run_timestamp": "2026-08-29T16:22:50.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.023663, "avg_duration_per_task_ms": 175642.6, "avg_output_tokens_per_task": 45106.7, "last_run_timestamp": "2026-08-01T17:39:14.289Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Phala", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.023599600000000002, "avg_duration_per_task_ms": 409207.06, "avg_output_tokens_per_task": 7009.96, "last_run_timestamp": "2026-07-04T16:00:25.388Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "OpenAI", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09545477999999999, "avg_duration_per_task_ms": 59370.42, "avg_output_tokens_per_task": 2482.38, "last_run_timestamp": "2026-08-01T08:06:06.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Together", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05687784, "avg_duration_per_task_ms": 126514.22, "avg_output_tokens_per_task": 3833.8, "last_run_timestamp": "2026-07-04T16:00:25.259Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.469999875, "accuracy_std_dev": 0.031798003956370814, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.113410725, "avg_duration_per_task_ms": 103888.355, "avg_output_tokens_per_task": 2755.44, "last_run_timestamp": "2026-08-29T16:33:30.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.46952399999999994, "accuracy_std_dev": 0.034523096873170074, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.019040825714285718, "avg_duration_per_task_ms": 89246.73142857142, "avg_output_tokens_per_task": 3672.6485714285714, "last_run_timestamp": "2026-08-29T16:43:11.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.46666633333333335, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03296042, "avg_duration_per_task_ms": 474531.4533333333, "avg_output_tokens_per_task": 36019.21333333333, "last_run_timestamp": "2026-07-18T16:00:08.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.46583349999999996, "accuracy_std_dev": 0.051363408397905215, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.127061305, "avg_duration_per_task_ms": 112391.425, "avg_output_tokens_per_task": 3072.22, "last_run_timestamp": "2026-08-29T16:25:26.800Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.4652775, "accuracy_std_dev": 0.006944499999999992, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.027522666666666668, "avg_duration_per_task_ms": 29612.604166666668, "avg_output_tokens_per_task": 1399.6145833333333, "last_run_timestamp": "2026-07-18T16:00:08.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.46444433333333335, "accuracy_std_dev": 0.008314853589945066, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.033371, "avg_duration_per_task_ms": 305705.87333333335, "avg_output_tokens_per_task": 3594.2266666666665, "last_run_timestamp": "2026-08-15T08:15:42.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.464166625, "accuracy_std_dev": 0.03843292588047877, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05392122499999999, "avg_duration_per_task_ms": 149701.2725, "avg_output_tokens_per_task": 3823.17, "last_run_timestamp": "2026-08-29T17:03:47.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.46333349999999995, "accuracy_std_dev": 0.0166665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20313888, "avg_duration_per_task_ms": 52334.34, "avg_output_tokens_per_task": 2078.1, "last_run_timestamp": "2026-09-01T12:41:31.668Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.463333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26064434, "avg_duration_per_task_ms": 31467.4, "avg_output_tokens_per_task": 1819.56, "last_run_timestamp": "2026-08-01T08:06:06.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.46279764285714287, "accuracy_std_dev": 0.03220534007479711, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.022562319196428572, "avg_duration_per_task_ms": 190962.13169642858, "avg_output_tokens_per_task": 3688.5491071428573, "last_run_timestamp": "2026-08-29T16:22:50.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 156315.84, "avg_output_tokens_per_task": 4524, "last_run_timestamp": "2026-09-01T12:59:40.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09873096, "avg_duration_per_task_ms": 65244.1, "avg_output_tokens_per_task": 2748.04, "last_run_timestamp": "2026-08-01T08:05:59.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.46, "accuracy_std_dev": 0.013333000000000012, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02811872, "avg_duration_per_task_ms": 579611.05, "avg_output_tokens_per_task": 20031.27, "last_run_timestamp": "2026-09-01T13:02:37.014Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.45920115624999996, "accuracy_std_dev": 0.020597873615982785, "run_count": 4, "total_questions": 192, "avg_cost_per_task_usd": 0.08208188020833333, "avg_duration_per_task_ms": 47993.442708333336, "avg_output_tokens_per_task": 3348.3072916666665, "last_run_timestamp": "2026-08-15T14:15:58.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.45777799999999996, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00622334, "avg_duration_per_task_ms": 39170.17333333333, "avg_output_tokens_per_task": 1174.36, "last_run_timestamp": "2026-08-15T08:16:13.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.4576988578680203, "accuracy_std_dev": 0.05672366902457362, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.16923886294416243, "avg_duration_per_task_ms": 401189.9695431472, "avg_output_tokens_per_task": 63938.149746192896, "last_run_timestamp": "2026-08-29T17:20:10.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.45542626162790695, "accuracy_std_dev": 0.018912652487751393, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.046210372093023255, "avg_duration_per_task_ms": 174189.27034883722, "avg_output_tokens_per_task": 1898.7906976744187, "last_run_timestamp": "2026-08-29T16:23:01.234Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.45333349999999994, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.2293187, "avg_duration_per_task_ms": 78949.14, "avg_output_tokens_per_task": 2726.37, "last_run_timestamp": "2026-07-18T08:04:08.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.4533333333333333, "accuracy_std_dev": 0.02553153460592084, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.027975164444444443, "avg_duration_per_task_ms": 208757.2288888889, "avg_output_tokens_per_task": 3694.0333333333333, "last_run_timestamp": "2026-08-29T16:33:39.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.44891123618090456, "accuracy_std_dev": 0.023521463575517047, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.020988829145728645, "avg_duration_per_task_ms": 492653.43467336684, "avg_output_tokens_per_task": 21696.05025125628, "last_run_timestamp": "2026-08-22T16:08:38.678Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.448889, "accuracy_std_dev": 0.0031423825355930196, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08242190666666667, "avg_duration_per_task_ms": 81216.82666666666, "avg_output_tokens_per_task": 4095.7466666666664, "last_run_timestamp": "2026-08-15T08:11:06.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.4488886666666667, "accuracy_std_dev": 0.05137597555494419, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0494734, "avg_duration_per_task_ms": 36454.72, "avg_output_tokens_per_task": 2419.86, "last_run_timestamp": "2026-08-29T16:58:32.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.44666675, "accuracy_std_dev": 0.031972270973259, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.3239053575, "avg_duration_per_task_ms": 100810.8825, "avg_output_tokens_per_task": 2994.355, "last_run_timestamp": "2026-08-29T16:12:10.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.4466665, "accuracy_std_dev": 0.006666499999999992, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020371870000000004, "avg_duration_per_task_ms": 172937.47, "avg_output_tokens_per_task": 42830.02, "last_run_timestamp": "2026-08-01T17:38:24.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.4466665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20957163, "avg_duration_per_task_ms": 47193.8, "avg_output_tokens_per_task": 2215.92, "last_run_timestamp": "2026-09-01T12:58:37.172Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.4463601206896552, "accuracy_std_dev": 0.026366188567007943, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.01648494540229885, "avg_duration_per_task_ms": 206287.91091954024, "avg_output_tokens_per_task": 2854.1609195402298, "last_run_timestamp": "2026-08-29T16:11:10.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.44500016666666664, "accuracy_std_dev": 0.029674109553709534, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.028000296666666664, "avg_duration_per_task_ms": 79011.23, "avg_output_tokens_per_task": 2803.3, "last_run_timestamp": "2026-08-29T17:27:31.024Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.445, "accuracy_std_dev": 0.001667000000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06940407, "avg_duration_per_task_ms": 279300.93, "avg_output_tokens_per_task": 3446.49, "last_run_timestamp": "2026-09-01T12:58:16.471Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.4444444444444444, "accuracy_std_dev": 0.02196410423850697, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.0480205202020202, "avg_duration_per_task_ms": 850576.6565656565, "avg_output_tokens_per_task": 104088.0202020202, "last_run_timestamp": "2026-09-01T17:31:48.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.4444443333333334, "accuracy_std_dev": 0.01617809828818648, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09676157666666667, "avg_duration_per_task_ms": 911223.4233333333, "avg_output_tokens_per_task": 20700.716666666667, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.44, "accuracy_std_dev": 0.013332999999999984, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02857548, "avg_duration_per_task_ms": 74491.22, "avg_output_tokens_per_task": 2449.6, "last_run_timestamp": "2026-09-01T17:30:53.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.44, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.7291186999999999, "avg_duration_per_task_ms": 35507.54, "avg_output_tokens_per_task": 3272.36, "last_run_timestamp": "2026-08-01T08:05:57.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.44, "accuracy_std_dev": 0.02788886675718467, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03487082, "avg_duration_per_task_ms": 281485.155, "avg_output_tokens_per_task": 21754.5, "last_run_timestamp": "2026-08-01T17:38:29.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.43965518965517236, "accuracy_std_dev": 0.020525836824522862, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.045893882183908045, "avg_duration_per_task_ms": 128881.75862068965, "avg_output_tokens_per_task": 2035.2413793103449, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.43902436585365856, "accuracy_std_dev": 0.046630122309082575, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.15270590650406504, "avg_duration_per_task_ms": 175421.88617886178, "avg_output_tokens_per_task": 6311.357723577235, "last_run_timestamp": "2026-08-30T00:01:09.486Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.4387170412371134, "accuracy_std_dev": 0.022793011768351766, "run_count": 6, "total_questions": 291, "avg_cost_per_task_usd": 0.031083333333333338, "avg_duration_per_task_ms": 487106.5326460481, "avg_output_tokens_per_task": 4378.945017182131, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.4383965367088608, "accuracy_std_dev": 0.03960836597472182, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.05240046329113924, "avg_duration_per_task_ms": 422067.2658227848, "avg_output_tokens_per_task": 20533.427848101266, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.4379670089686099, "accuracy_std_dev": 0.04940897449394402, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.03162573766816144, "avg_duration_per_task_ms": 87326.9730941704, "avg_output_tokens_per_task": 3019.251121076233, "last_run_timestamp": "2026-08-29T17:17:42.808Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.4373334, "accuracy_std_dev": 0.019136942750606727, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.077255652, "avg_duration_per_task_ms": 788193.684, "avg_output_tokens_per_task": 19596.964, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.43666675, "accuracy_std_dev": 0.011055139582452123, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.21359883, "avg_duration_per_task_ms": 47801.905, "avg_output_tokens_per_task": 2174.32, "last_run_timestamp": "2026-09-01T13:02:23.243Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.43666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014376880000000002, "avg_duration_per_task_ms": 93606.58, "avg_output_tokens_per_task": 10159.34, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.4353740544217687, "accuracy_std_dev": 0.0333418611528944, "run_count": 6, "total_questions": 294, "avg_cost_per_task_usd": 0.03948419727891156, "avg_duration_per_task_ms": 53655.58163265306, "avg_output_tokens_per_task": 2073.0680272108843, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.43348282959641254, "accuracy_std_dev": 0.021733670805150856, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.024745609865470853, "avg_duration_per_task_ms": 267535.8094170404, "avg_output_tokens_per_task": 3614.85201793722, "last_run_timestamp": "2026-08-29T16:17:35.870Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.4333336666666667, "accuracy_std_dev": 0.0339934634239519, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03350491333333334, "avg_duration_per_task_ms": 439900.87333333335, "avg_output_tokens_per_task": 22781.906666666666, "last_run_timestamp": "2026-08-01T17:38:32.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.43333333333333335, "accuracy_std_dev": 0.014401440259764139, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03429177333333333, "avg_duration_per_task_ms": 515166.3933333333, "avg_output_tokens_per_task": 24569.606666666667, "last_run_timestamp": "2026-08-01T17:38:35.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4311113333333333, "accuracy_std_dev": 0.006285236475706817, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023601506666666664, "avg_duration_per_task_ms": 216917.58666666667, "avg_output_tokens_per_task": 3146.213333333333, "last_run_timestamp": "2026-08-15T08:02:48.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.43, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02958018, "avg_duration_per_task_ms": 73337.54, "avg_output_tokens_per_task": 2441.2, "last_run_timestamp": "2026-09-01T17:31:46.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.43, "accuracy_std_dev": 0.0033330000000000026, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23500686999999998, "avg_duration_per_task_ms": 47776.24, "avg_output_tokens_per_task": 2493.12, "last_run_timestamp": "2026-09-01T13:00:52.000Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.42481477777777776, "accuracy_std_dev": 0.01055379582875573, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.024998100000000002, "avg_duration_per_task_ms": 230644.6577777778, "avg_output_tokens_per_task": 3644.9155555555553, "last_run_timestamp": "2026-08-29T16:50:29.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.423333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02960835, "avg_duration_per_task_ms": 524604.61, "avg_output_tokens_per_task": 20056.49, "last_run_timestamp": "2026-09-01T12:41:25.851Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.4222224444444444, "accuracy_std_dev": 0.03624331355991708, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05411629777777777, "avg_duration_per_task_ms": 135677.24, "avg_output_tokens_per_task": 3556.682222222222, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.4193990491803279, "accuracy_std_dev": 0.02746729167500866, "run_count": 10, "total_questions": 488, "avg_cost_per_task_usd": 0.05026936680327868, "avg_duration_per_task_ms": 211126.46106557376, "avg_output_tokens_per_task": 4020.3565573770493, "last_run_timestamp": "2026-09-26T08:09:25.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.41851855555555556, "accuracy_std_dev": 0.03580157095594522, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0414691511111111, "avg_duration_per_task_ms": 80026.80444444444, "avg_output_tokens_per_task": 3600.311111111111, "last_run_timestamp": "2026-08-29T16:10:58.039Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4183674081632653, "accuracy_std_dev": 0.008472500000000022, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.048346357142857135, "avg_duration_per_task_ms": 818319.5204081633, "avg_output_tokens_per_task": 111719.33673469388, "last_run_timestamp": "2026-09-01T17:32:02.570Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.41666675, "accuracy_std_dev": 0.013743584364622648, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.027539789999999998, "avg_duration_per_task_ms": 73803.845, "avg_output_tokens_per_task": 2931.665, "last_run_timestamp": "2026-09-01T17:32:02.600Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.413333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06003468, "avg_duration_per_task_ms": 392895.94, "avg_output_tokens_per_task": 17058.96, "last_run_timestamp": "2026-08-01T08:06:29.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.41306208651399495, "accuracy_std_dev": 0.03034144422624565, "run_count": 8, "total_questions": 393, "avg_cost_per_task_usd": 0.01169636641221374, "avg_duration_per_task_ms": 132148.3969465649, "avg_output_tokens_per_task": 3534.709923664122, "last_run_timestamp": "2026-08-29T16:17:15.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.41228081754385965, "accuracy_std_dev": 0.14934019625078246, "run_count": 6, "total_questions": 285, "avg_cost_per_task_usd": 0.03923479298245614, "avg_duration_per_task_ms": 212298.04912280702, "avg_output_tokens_per_task": 88632.91228070176, "last_run_timestamp": "2026-08-29T17:06:38.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41142871428571426, "accuracy_std_dev": 0.022737753675420126, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014496525714285714, "avg_duration_per_task_ms": 78866.04571428572, "avg_output_tokens_per_task": 9061.385714285714, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.40851855555555555, "accuracy_std_dev": 0.0222838565040111, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06348359111111111, "avg_duration_per_task_ms": 98874.61111111111, "avg_output_tokens_per_task": 2927.2933333333335, "last_run_timestamp": "2026-08-29T16:15:12.659Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.40778087608069163, "accuracy_std_dev": 0.020533700034383498, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.030645242074927952, "avg_duration_per_task_ms": 639129.7435158502, "avg_output_tokens_per_task": 7195.42939481268, "last_run_timestamp": "2026-08-30T00:01:10.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.40666674999999997, "accuracy_std_dev": 0.024266872711321913, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02623856, "avg_duration_per_task_ms": 288008.1325, "avg_output_tokens_per_task": 4171.565, "last_run_timestamp": "2026-08-29T16:17:08.788Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.40666650000000004, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06123743, "avg_duration_per_task_ms": 234369.43, "avg_output_tokens_per_task": 8023.68, "last_run_timestamp": "2026-09-01T12:41:31.874Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.4051851111111111, "accuracy_std_dev": 0.14120402210076063, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03654666, "avg_duration_per_task_ms": 265671.77777777775, "avg_output_tokens_per_task": 3253.0866666666666, "last_run_timestamp": "2026-08-29T16:33:29.143Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.4051851111111111, "accuracy_std_dev": 0.039475118659858095, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05917665777777779, "avg_duration_per_task_ms": 299711.4088888889, "avg_output_tokens_per_task": 7407.624444444445, "last_run_timestamp": "2026-08-29T16:11:10.099Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.40379423577235773, "accuracy_std_dev": 0.19339763406453553, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.01662922357723577, "avg_duration_per_task_ms": 1820667.5203252032, "avg_output_tokens_per_task": 111556.55284552845, "last_run_timestamp": "2026-08-29T17:12:32.860Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.4, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.1315643, "avg_duration_per_task_ms": 71996.84, "avg_output_tokens_per_task": 3249.82, "last_run_timestamp": "2026-08-01T08:06:02.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.4, "accuracy_std_dev": 0.019626323972325204, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020620846666666668, "avg_duration_per_task_ms": 194089.08666666667, "avg_output_tokens_per_task": 3830.9066666666668, "last_run_timestamp": "2026-08-15T08:02:44.458Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.3977775555555556, "accuracy_std_dev": 0.01859228567092104, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07825311555555554, "avg_duration_per_task_ms": 125269.89111111111, "avg_output_tokens_per_task": 3131.8733333333334, "last_run_timestamp": "2026-08-29T16:33:39.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.3971632127659574, "accuracy_std_dev": 0.3419385, "run_count": 2, "total_questions": 94, "avg_cost_per_task_usd": 0.030152606382978724, "avg_duration_per_task_ms": 146561.8404255319, "avg_output_tokens_per_task": 4856.648936170212, "last_run_timestamp": "2026-08-15T14:23:29.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.39666650000000003, "accuracy_std_dev": 0.02333349999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0319114, "avg_duration_per_task_ms": 79638.86, "avg_output_tokens_per_task": 5369.64, "last_run_timestamp": "2026-07-18T08:03:14.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.39062490277777784, "accuracy_std_dev": 0.02073837202707419, "run_count": 6, "total_questions": 288, "avg_cost_per_task_usd": 0.010834395833333335, "avg_duration_per_task_ms": 31261.70486111111, "avg_output_tokens_per_task": 586.9097222222222, "last_run_timestamp": "2026-08-15T14:15:11.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.3903702222222223, "accuracy_std_dev": 0.023541163785910455, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03730226666666667, "avg_duration_per_task_ms": 195629.94666666666, "avg_output_tokens_per_task": 3811.4333333333334, "last_run_timestamp": "2026-08-29T16:31:33.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.39, "accuracy_std_dev": 0.014782396941407474, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.028387520000000006, "avg_duration_per_task_ms": 78569.82333333333, "avg_output_tokens_per_task": 4258.516666666666, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.39, "accuracy_std_dev": 0.023094082938060125, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02572764, "avg_duration_per_task_ms": 216676.0775, "avg_output_tokens_per_task": 4467.8225, "last_run_timestamp": "2026-08-29T16:10:20.675Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.38666666666666666, "accuracy_std_dev": 0.012957518435256032, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.012331926666666666, "avg_duration_per_task_ms": 31907.722222222223, "avg_output_tokens_per_task": 547.9311111111111, "last_run_timestamp": "2026-08-29T16:10:09.208Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.385926, "accuracy_std_dev": 0.0079780288153013, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01442735111111111, "avg_duration_per_task_ms": 79203.29111111112, "avg_output_tokens_per_task": 598.6177777777777, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.38333349999999994, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07100836, "avg_duration_per_task_ms": 283936.54, "avg_output_tokens_per_task": 7506.19, "last_run_timestamp": "2026-09-01T12:58:37.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.37916662500000003, "accuracy_std_dev": 0.02933276396343131, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04733663, "avg_duration_per_task_ms": 70203.2925, "avg_output_tokens_per_task": 4327.02, "last_run_timestamp": "2026-08-29T16:25:50.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.3771427142857142, "accuracy_std_dev": 0.008624165809999659, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03271254285714286, "avg_duration_per_task_ms": 138753.44, "avg_output_tokens_per_task": 4451.42, "last_run_timestamp": "2026-08-29T16:13:32.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.3744443333333334, "accuracy_std_dev": 0.01940465134674902, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06861020666666667, "avg_duration_per_task_ms": 17591.08, "avg_output_tokens_per_task": 793.6233333333333, "last_run_timestamp": "2026-08-29T16:18:42.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.3712121363636363, "accuracy_std_dev": 0.203715753368511, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.01041270707070707, "avg_duration_per_task_ms": 88573.35858585859, "avg_output_tokens_per_task": 2290.378787878788, "last_run_timestamp": "2026-08-29T17:44:16.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.37016358035714286, "accuracy_std_dev": 0.060232280320582955, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.05735951562499999, "avg_duration_per_task_ms": 340422.8660714286, "avg_output_tokens_per_task": 3670.198660714286, "last_run_timestamp": "2026-08-29T16:13:54.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.3688886666666667, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07234757333333333, "avg_duration_per_task_ms": 16758.733333333334, "avg_output_tokens_per_task": 864.72, "last_run_timestamp": "2026-07-18T16:00:09.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.36666699999999997, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.80181, "avg_duration_per_task_ms": 63430.68, "avg_output_tokens_per_task": 3064.6, "last_run_timestamp": "2026-08-01T08:06:07.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.3639454693877551, "accuracy_std_dev": 0.261389, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.004816234693877552, "avg_duration_per_task_ms": 133358.81632653062, "avg_output_tokens_per_task": 12437.69387755102, "last_run_timestamp": "2026-07-11T15:34:14.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.362963, "accuracy_std_dev": 0.09661485400174123, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05752990444444444, "avg_duration_per_task_ms": 257852.61777777778, "avg_output_tokens_per_task": 10247.41111111111, "last_run_timestamp": "2026-08-29T16:11:34.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3616235461254612, "accuracy_std_dev": 0.028902948389451646, "run_count": 11, "total_questions": 542, "avg_cost_per_task_usd": 0.07057725645756457, "avg_duration_per_task_ms": 374277.7712177122, "avg_output_tokens_per_task": 7489.30258302583, "last_run_timestamp": "2026-09-26T08:14:46.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.358859, "accuracy_std_dev": 0.037673584399498986, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.021177574324324325, "avg_duration_per_task_ms": 100388.78153153154, "avg_output_tokens_per_task": 1092.0855855855855, "last_run_timestamp": "2026-08-29T16:14:38.085Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.3555554444444445, "accuracy_std_dev": 0.3294587300632867, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021335997777777782, "avg_duration_per_task_ms": 65350.72222222222, "avg_output_tokens_per_task": 11586.551111111112, "last_run_timestamp": "2026-09-26T16:00:24.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.35542168674698793, "accuracy_std_dev": 0.02435901384703412, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.08032603012048192, "avg_duration_per_task_ms": 238054.84738955824, "avg_output_tokens_per_task": 7629.79718875502, "last_run_timestamp": "2026-09-26T08:01:16.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.35266659999999994, "accuracy_std_dev": 0.014047424441512414, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.039413411999999995, "avg_duration_per_task_ms": 97771.824, "avg_output_tokens_per_task": 4606.912, "last_run_timestamp": "2026-08-01T17:38:36.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.34641463372093023, "accuracy_std_dev": 0.07304902258240625, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.00352575, "avg_duration_per_task_ms": 248041.55523255814, "avg_output_tokens_per_task": 3801.1162790697676, "last_run_timestamp": "2026-08-29T16:33:36.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.34058, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.11353834782608695, "avg_duration_per_task_ms": 397172.5, "avg_output_tokens_per_task": 14067.434782608696, "last_run_timestamp": "2026-08-01T08:05:57.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.3370835, "accuracy_std_dev": 0.06602697155140465, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.004872619999999999, "avg_duration_per_task_ms": 111751.1125, "avg_output_tokens_per_task": 3307.625, "last_run_timestamp": "2026-08-29T17:02:26.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.3322147852348994, "accuracy_std_dev": 0.07110040643109339, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.03765286577181208, "avg_duration_per_task_ms": 95171.10067114094, "avg_output_tokens_per_task": 6163.89932885906, "last_run_timestamp": "2026-07-25T08:07:19.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.3309091818181818, "accuracy_std_dev": 0.022966516131961975, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08758338363636364, "avg_duration_per_task_ms": 154847.89454545453, "avg_output_tokens_per_task": 8333.869090909091, "last_run_timestamp": "2026-09-26T08:09:25.338Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.3219998, "accuracy_std_dev": 0.024908747237868067, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.03118176, "avg_duration_per_task_ms": 78513.244, "avg_output_tokens_per_task": 3895.62, "last_run_timestamp": "2026-08-01T17:38:37.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.32068464732142854, "accuracy_std_dev": 0.18170803102970257, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.12890572098214284, "avg_duration_per_task_ms": 772274.0200892857, "avg_output_tokens_per_task": 5070.225446428572, "last_run_timestamp": "2026-09-26T08:05:39.969Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Groq", "accuracy": 0.32, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07498856, "avg_duration_per_task_ms": 19652.8, "avg_output_tokens_per_task": 2138.96, "last_run_timestamp": "2026-07-04T16:00:20.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.3143632967479675, "accuracy_std_dev": 0.23029092136912385, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.008914227642276422, "avg_duration_per_task_ms": 59356.63821138212, "avg_output_tokens_per_task": 1974.3902439024391, "last_run_timestamp": "2026-08-08T16:03:49.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.31282059999999995, "accuracy_std_dev": 0.030993226130882217, "run_count": 8, "total_questions": 390, "avg_cost_per_task_usd": 0.015985989743589744, "avg_duration_per_task_ms": 332807.73846153845, "avg_output_tokens_per_task": 4072.7410256410258, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.3081481111111111, "accuracy_std_dev": 0.03755203650771211, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.15946950444444447, "avg_duration_per_task_ms": 119529.95333333334, "avg_output_tokens_per_task": 1809.4622222222222, "last_run_timestamp": "2026-08-29T17:05:59.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.27, "accuracy_std_dev": 0.03, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02140288, "avg_duration_per_task_ms": 65620.09, "avg_output_tokens_per_task": 3949.38, "last_run_timestamp": "2026-09-01T17:29:05.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.266667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00574078, "avg_duration_per_task_ms": 359216.92, "avg_output_tokens_per_task": 2727.92, "last_run_timestamp": "2026-07-11T08:03:53.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.266667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02052575, "avg_duration_per_task_ms": 64186.84, "avg_output_tokens_per_task": 3663.19, "last_run_timestamp": "2026-09-01T17:31:47.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.26148144444444443, "accuracy_std_dev": 0.06338407627246437, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.015817097777777775, "avg_duration_per_task_ms": 25942.504444444443, "avg_output_tokens_per_task": 1328.6466666666668, "last_run_timestamp": "2026-08-29T16:17:15.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.26041658333333334, "accuracy_std_dev": 0.01347799999999999, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.07850963541666667, "avg_duration_per_task_ms": 588191.84375, "avg_output_tokens_per_task": 5667.5625, "last_run_timestamp": "2026-09-01T12:59:55.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.25, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08593323, "avg_duration_per_task_ms": 547480.87, "avg_output_tokens_per_task": 5216.43, "last_run_timestamp": "2026-09-01T12:57:03.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.23299353061224487, "accuracy_std_dev": 0.02687500000000001, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.028852255102040817, "avg_duration_per_task_ms": 377353.28571428574, "avg_output_tokens_per_task": 5486.489795918367, "last_run_timestamp": "2026-07-18T08:04:09.749Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.2316665, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02960246, "avg_duration_per_task_ms": 418677.18, "avg_output_tokens_per_task": 5444.6, "last_run_timestamp": "2026-07-18T08:04:25.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.21166649999999998, "accuracy_std_dev": 0.046278414695730456, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0049267899999999995, "avg_duration_per_task_ms": 45675.635, "avg_output_tokens_per_task": 1348.715, "last_run_timestamp": "2026-08-22T16:04:58.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.20642976649746192, "accuracy_std_dev": 0.1484845071697297, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.17611084517766495, "avg_duration_per_task_ms": 524637.7563451777, "avg_output_tokens_per_task": 9406.852791878173, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.19841299319727893, "accuracy_std_dev": 0.016383133956060492, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.011659061224489795, "avg_duration_per_task_ms": 134707.1224489796, "avg_output_tokens_per_task": 5676.0204081632655, "last_run_timestamp": "2026-09-01T13:00:50.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.17333325, "accuracy_std_dev": 0.11522926663368771, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0127158975, "avg_duration_per_task_ms": 62680.3375, "avg_output_tokens_per_task": 1280.3475, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01877496, "avg_duration_per_task_ms": 63137.36, "avg_output_tokens_per_task": 2797.32, "last_run_timestamp": "2026-08-01T08:06:16.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.016978299999999998, "avg_duration_per_task_ms": 57790.52, "avg_output_tokens_per_task": 2492.08, "last_run_timestamp": "2026-08-01T08:06:07.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.1366665, "accuracy_std_dev": 0.003333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00694106, "avg_duration_per_task_ms": 56167.4, "avg_output_tokens_per_task": 2292.83, "last_run_timestamp": "2026-09-01T17:31:40.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.125, "accuracy_std_dev": 0.020750046855368784, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.00834746, "avg_duration_per_task_ms": 82234.105, "avg_output_tokens_per_task": 2517.12, "last_run_timestamp": "2026-09-01T17:31:43.416Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.1033335, "accuracy_std_dev": 0.003333499999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0070183400000000005, "avg_duration_per_task_ms": 61321.36, "avg_output_tokens_per_task": 2324.09, "last_run_timestamp": "2026-09-01T17:31:46.301Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.013333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0036388199999999996, "avg_duration_per_task_ms": 4217.8, "avg_output_tokens_per_task": 129.92, "last_run_timestamp": "2026-07-04T16:00:25.000Z" } ] }