{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-09-27", "retrieved_at": "2026-09-27T01:45:11.774Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's newest 10,000 non-zero-score runs: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2121, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2003321212121212, "avg_duration_per_task_ms": 124514.04545454546, "avg_output_tokens_per_task": 16571.29797979798, "last_run_timestamp": "2026-09-01T17:50:10.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra", "model_slug": "sakana/fugu-ultra", "model_name": "Fugu Ultra", "provider_name": "auto-routing", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 1.148739303030303, "avg_duration_per_task_ms": 328779.10101010103, "avg_output_tokens_per_task": 33271.606060606064, "last_run_timestamp": "2026-09-15T01:07:47.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.116425398989899, "avg_duration_per_task_ms": 44572.09595959596, "avg_output_tokens_per_task": 2176.570707070707, "last_run_timestamp": "2026-09-06T03:17:58.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.945286, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2012271818181818, "avg_duration_per_task_ms": 134249.20707070708, "avg_output_tokens_per_task": 16644.886363636364, "last_run_timestamp": "2026-09-01T17:53:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02981064646464646, "avg_duration_per_task_ms": 28082.027777777777, "avg_output_tokens_per_task": 7803.272727272727, "last_run_timestamp": "2026-09-01T13:28:27.192Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.20185933333333334, "avg_duration_per_task_ms": 126427.48484848485, "avg_output_tokens_per_task": 16698.565656565657, "last_run_timestamp": "2026-09-01T17:51:04.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11967219191919193, "avg_duration_per_task_ms": 38197.17171717172, "avg_output_tokens_per_task": 2017.3383838383838, "last_run_timestamp": "2026-09-26T10:54:50.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9436415317919074, "accuracy_std_dev": 0.0020135000000000014, "run_count": 2, "total_questions": 346, "avg_cost_per_task_usd": 0.09933485549132949, "avg_duration_per_task_ms": 198116.95664739885, "avg_output_tokens_per_task": 1767.9046242774566, "last_run_timestamp": "2026-09-06T03:05:46.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.942761, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.030509156565656566, "avg_duration_per_task_ms": 68136.79292929293, "avg_output_tokens_per_task": 7993.409090909091, "last_run_timestamp": "2026-09-01T13:25:46.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.9377104999999999, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.30815713636363634, "avg_duration_per_task_ms": 160641.4873737374, "avg_output_tokens_per_task": 10142.272727272728, "last_run_timestamp": "2026-09-01T13:20:32.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.93771, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.4446258484848485, "avg_duration_per_task_ms": 116119.66666666667, "avg_output_tokens_per_task": 11825.727272727272, "last_run_timestamp": "2026-08-01T08:45:01.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.9351849999999999, "accuracy_std_dev": 0.006162785766193721, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1365803898989899, "avg_duration_per_task_ms": 176670.50606060607, "avg_output_tokens_per_task": 8896.30606060606, "last_run_timestamp": "2026-09-26T09:17:14.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.9335082598425197, "accuracy_std_dev": 0.011397999999999964, "run_count": 2, "total_questions": 381, "avg_cost_per_task_usd": 0.0213603280839895, "avg_duration_per_task_ms": 53235.17585301837, "avg_output_tokens_per_task": 7688.425196850394, "last_run_timestamp": "2026-09-01T13:29:18.783Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra-v2", "model_slug": "sakana/fugu-ultra-v2", "model_name": "Fugu Ultra V2", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.5025571414141414, "avg_duration_per_task_ms": 194915.41919191918, "avg_output_tokens_per_task": 13806.227272727272, "last_run_timestamp": "2026-09-15T01:07:48.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.12205839393939394, "avg_duration_per_task_ms": 394338.13636363635, "avg_output_tokens_per_task": 20077.80808080808, "last_run_timestamp": "2026-09-01T13:22:57.685Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9324968387096775, "accuracy_std_dev": 0.005080239036370865, "run_count": 3, "total_questions": 558, "avg_cost_per_task_usd": 0.0715788476702509, "avg_duration_per_task_ms": 64320.11827956989, "avg_output_tokens_per_task": 11534.012544802867, "last_run_timestamp": "2026-09-01T17:51:06.380Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.930976, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.32996825757575754, "avg_duration_per_task_ms": 166479.3383838384, "avg_output_tokens_per_task": 10868.631313131313, "last_run_timestamp": "2026-09-01T13:22:14.451Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9299660000000001, "accuracy_std_dev": 0.009009433123121587, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.32134650707070705, "avg_duration_per_task_ms": 152235.298989899, "avg_output_tokens_per_task": 9994.927272727273, "last_run_timestamp": "2026-09-01T13:24:49.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.9294545733558178, "accuracy_std_dev": 0.004334827511626685, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.02681522428330523, "avg_duration_per_task_ms": 26517.369308600337, "avg_output_tokens_per_task": 2356.256323777403, "last_run_timestamp": "2026-09-23T15:41:43.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9292930000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14407252272727272, "avg_duration_per_task_ms": 58692.59848484849, "avg_output_tokens_per_task": 15884.012626262625, "last_run_timestamp": "2026-09-01T17:55:40.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.92887175, "accuracy_std_dev": 0.005239941859171739, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06947331565656567, "avg_duration_per_task_ms": 67244.06060606061, "avg_output_tokens_per_task": 11635.200757575758, "last_run_timestamp": "2026-09-01T17:56:37.544Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06683859343434344, "avg_duration_per_task_ms": 59600.07575757576, "avg_output_tokens_per_task": 11674.472222222223, "last_run_timestamp": "2026-09-01T17:57:17.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9283648630849219, "accuracy_std_dev": 0.017830944261311557, "run_count": 4, "total_questions": 577, "avg_cost_per_task_usd": 0.09151112998266897, "avg_duration_per_task_ms": 252153.69497400348, "avg_output_tokens_per_task": 7044.975736568457, "last_run_timestamp": "2026-09-26T11:09:52.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.9279571938392187, "accuracy_std_dev": 0.00835201542708194, "run_count": 22, "total_questions": 3993, "avg_cost_per_task_usd": 0.15530522213874282, "avg_duration_per_task_ms": 142386.56699223642, "avg_output_tokens_per_task": 8728.15927873779, "last_run_timestamp": "2026-09-26T11:50:40.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.927609, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14183102777777779, "avg_duration_per_task_ms": 79740.97727272728, "avg_output_tokens_per_task": 15646.176767676769, "last_run_timestamp": "2026-09-01T18:06:42.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.92613625, "accuracy_std_dev": 0.007949293313716653, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.09945460101010102, "avg_duration_per_task_ms": 230595.33712121213, "avg_output_tokens_per_task": 9430.997474747475, "last_run_timestamp": "2026-09-26T08:53:13.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9259257500000001, "accuracy_std_dev": 0.004610545920766882, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.1489979318181818, "avg_duration_per_task_ms": 84816.23358585859, "avg_output_tokens_per_task": 15665.375, "last_run_timestamp": "2026-09-01T17:51:30.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.925347078125, "accuracy_std_dev": 0.005810499999999996, "run_count": 2, "total_questions": 384, "avg_cost_per_task_usd": 0.3411057265625, "avg_duration_per_task_ms": 143482.140625, "avg_output_tokens_per_task": 10207.984375, "last_run_timestamp": "2026-09-01T13:21:58.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.026419161616161615, "avg_duration_per_task_ms": 21367.934343434343, "avg_output_tokens_per_task": 2243.222222222222, "last_run_timestamp": "2026-09-23T15:27:41.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9240362551020408, "accuracy_std_dev": 0.010411205213614818, "run_count": 3, "total_questions": 588, "avg_cost_per_task_usd": 0.11667940476190476, "avg_duration_per_task_ms": 366799.1649659864, "avg_output_tokens_per_task": 19230.027210884353, "last_run_timestamp": "2026-09-01T13:27:43.904Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.922559, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.4976751313131313, "avg_duration_per_task_ms": 91843.67676767676, "avg_output_tokens_per_task": 12034.212121212122, "last_run_timestamp": "2026-08-01T08:41:32.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9224121127739175, "accuracy_std_dev": 0.007223723392406446, "run_count": 10, "total_questions": 1871, "avg_cost_per_task_usd": 0.15054908498129344, "avg_duration_per_task_ms": 122412.57936932122, "avg_output_tokens_per_task": 9889.19080705505, "last_run_timestamp": "2026-09-26T08:55:05.759Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9215687647058822, "accuracy_std_dev": 0.0038094199325123296, "run_count": 3, "total_questions": 561, "avg_cost_per_task_usd": 0.10569379500891266, "avg_duration_per_task_ms": 54609.55971479501, "avg_output_tokens_per_task": 3484.6114081996434, "last_run_timestamp": "2026-09-01T13:22:25.850Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9212118, "accuracy_std_dev": 0.008675973568424486, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1201016404040404, "avg_duration_per_task_ms": 61346.69292929293, "avg_output_tokens_per_task": 3579.7959595959596, "last_run_timestamp": "2026-09-01T13:28:53.908Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.920875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.4329253232323232, "avg_duration_per_task_ms": 113725.84848484848, "avg_output_tokens_per_task": 11493.873737373737, "last_run_timestamp": "2026-08-01T08:53:39.678Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.9205012222222222, "accuracy_std_dev": 0.00850227984821038, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1576766610549944, "avg_duration_per_task_ms": 236410.12177328844, "avg_output_tokens_per_task": 12595.254208754208, "last_run_timestamp": "2026-09-26T08:55:21.775Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.9200335, "accuracy_std_dev": 0.007575500000000013, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07599968055555556, "avg_duration_per_task_ms": 289463.85101010103, "avg_output_tokens_per_task": 24687.122474747473, "last_run_timestamp": "2026-09-26T09:34:25.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.919865, "accuracy_std_dev": 0.007939302840930074, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.16831854444444444, "avg_duration_per_task_ms": 345204.0070707071, "avg_output_tokens_per_task": 9582.316161616161, "last_run_timestamp": "2026-09-26T09:45:25.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.9192883359550563, "accuracy_std_dev": 0.004632004101898027, "run_count": 9, "total_questions": 1780, "avg_cost_per_task_usd": 0.19511707022471914, "avg_duration_per_task_ms": 338842.3713483146, "avg_output_tokens_per_task": 44188.06966292135, "last_run_timestamp": "2026-09-26T08:56:52.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191915, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07126612626262625, "avg_duration_per_task_ms": 64525.257575757576, "avg_output_tokens_per_task": 3520.8156565656564, "last_run_timestamp": "2026-09-01T13:21:08.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9188843954372623, "accuracy_std_dev": 0.012617923846160294, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.13335884410646387, "avg_duration_per_task_ms": 335309.8307984791, "avg_output_tokens_per_task": 10503.390367553866, "last_run_timestamp": "2026-09-26T08:52:39.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.9186867999999999, "accuracy_std_dev": 0.0068919500984844775, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13311589696969697, "avg_duration_per_task_ms": 546514.5777777778, "avg_output_tokens_per_task": 36924.677777777775, "last_run_timestamp": "2026-09-26T09:34:49.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.9179851756329115, "accuracy_std_dev": 0.013506707135345767, "run_count": 10, "total_questions": 1896, "avg_cost_per_task_usd": 0.12114161814345992, "avg_duration_per_task_ms": 120275.13765822785, "avg_output_tokens_per_task": 8135.254219409283, "last_run_timestamp": "2026-09-26T09:59:38.434Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.9175826670976084, "accuracy_std_dev": 0.014195212289272039, "run_count": 8, "total_questions": 1547, "avg_cost_per_task_usd": 0.008065983839689723, "avg_duration_per_task_ms": 489574.17323852616, "avg_output_tokens_per_task": 20043.881060116353, "last_run_timestamp": "2026-09-26T09:23:12.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.917508, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.025625949494949497, "avg_duration_per_task_ms": 37548.47474747475, "avg_output_tokens_per_task": 2404.0757575757575, "last_run_timestamp": "2026-09-23T15:32:55.695Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.9164982, "accuracy_std_dev": 0.005280767383629021, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.09286130606060607, "avg_duration_per_task_ms": 636505.3202020202, "avg_output_tokens_per_task": 40298.016161616164, "last_run_timestamp": "2026-09-26T09:33:50.566Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.9157045, "accuracy_std_dev": 0.012329586256237483, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.029113009740259738, "avg_duration_per_task_ms": 215302.51695526697, "avg_output_tokens_per_task": 24148.466810966813, "last_run_timestamp": "2026-09-26T16:05:15.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.914983, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07349445707070708, "avg_duration_per_task_ms": 64386.71464646464, "avg_output_tokens_per_task": 3569.0353535353534, "last_run_timestamp": "2026-09-01T13:22:37.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9147023333333334, "accuracy_std_dev": 0.011850896290529652, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1591313367003367, "avg_duration_per_task_ms": 538081.3703703703, "avg_output_tokens_per_task": 39867.37205387205, "last_run_timestamp": "2026-09-26T09:14:59.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.9139752900807381, "accuracy_std_dev": 0.006586842457168369, "run_count": 9, "total_questions": 1734, "avg_cost_per_task_usd": 0.14062165570934257, "avg_duration_per_task_ms": 294457.8246828143, "avg_output_tokens_per_task": 9226.012687427912, "last_run_timestamp": "2026-09-26T09:18:47.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9127381666666666, "accuracy_std_dev": 0.0041711655212369175, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.27201272474747473, "avg_duration_per_task_ms": 357401.7491582492, "avg_output_tokens_per_task": 38560.36363636364, "last_run_timestamp": "2026-09-26T09:19:10.819Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.9124577500000001, "accuracy_std_dev": 0.006628136630117118, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0378646856060606, "avg_duration_per_task_ms": 130990.08964646465, "avg_output_tokens_per_task": 28457.30808080808, "last_run_timestamp": "2026-09-26T08:54:48.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.9107743333333334, "accuracy_std_dev": 0.004956316261992276, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03197664478114478, "avg_duration_per_task_ms": 194795.97138047137, "avg_output_tokens_per_task": 26470.969696969696, "last_run_timestamp": "2026-09-26T08:52:40.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.9104936666666668, "accuracy_std_dev": 0.05077924822427192, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.10505623232323233, "avg_duration_per_task_ms": 246993.5, "avg_output_tokens_per_task": 9288.036195286195, "last_run_timestamp": "2026-09-26T09:35:57.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9102129999999999, "accuracy_std_dev": 0.002861334420627323, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0912948888888889, "avg_duration_per_task_ms": 155548.53703703705, "avg_output_tokens_per_task": 27869.387205387204, "last_run_timestamp": "2026-09-01T13:31:50.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.9096159477085782, "accuracy_std_dev": 0.03256740651027582, "run_count": 18, "total_questions": 3404, "avg_cost_per_task_usd": 0.1496695376028202, "avg_duration_per_task_ms": 473661.7364864865, "avg_output_tokens_per_task": 8839.687720329024, "last_run_timestamp": "2026-09-26T11:50:25.798Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.9091866447534767, "accuracy_std_dev": 0.0045401682512876225, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.029416860935524654, "avg_duration_per_task_ms": 453238.0587863464, "avg_output_tokens_per_task": 30418.960809102402, "last_run_timestamp": "2026-09-26T16:21:44.172Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2221867676767677, "avg_duration_per_task_ms": 298883.5202020202, "avg_output_tokens_per_task": 29352.99494949495, "last_run_timestamp": "2026-09-27T00:06:19.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stealth/union-alpha", "model_slug": "stealth/union-alpha", "model_name": "Union Alpha", "provider_name": "auto-routing", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 146448.00505050505, "avg_output_tokens_per_task": 4567.823232323232, "last_run_timestamp": "2026-09-15T01:07:47.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.9090566691616767, "accuracy_std_dev": 0.011036068624917007, "run_count": 7, "total_questions": 1336, "avg_cost_per_task_usd": 0.012388138473053893, "avg_duration_per_task_ms": 304706.8982035928, "avg_output_tokens_per_task": 16929.82260479042, "last_run_timestamp": "2026-09-26T08:52:35.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.9087729427112877, "accuracy_std_dev": 0.01605553275354013, "run_count": 9, "total_questions": 1763, "avg_cost_per_task_usd": 0.188728044242768, "avg_duration_per_task_ms": 517068.7197958026, "avg_output_tokens_per_task": 12546.453204764606, "last_run_timestamp": "2026-09-26T08:17:01.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.9086699999999999, "accuracy_std_dev": 0.0024899934738870384, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.035269609848484845, "avg_duration_per_task_ms": 184713.31186868687, "avg_output_tokens_per_task": 29215.651515151516, "last_run_timestamp": "2026-09-26T09:27:25.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.9074072500000001, "accuracy_std_dev": 0.004610545920766882, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.030798316919191916, "avg_duration_per_task_ms": 142600.18308080808, "avg_output_tokens_per_task": 28269.325757575756, "last_run_timestamp": "2026-09-26T09:35:41.692Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9073833393548387, "accuracy_std_dev": 0.021106101149904508, "run_count": 12, "total_questions": 2325, "avg_cost_per_task_usd": 0.03617074838709677, "avg_duration_per_task_ms": 259960.5711827957, "avg_output_tokens_per_task": 30308.94064516129, "last_run_timestamp": "2026-09-26T16:06:12.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.9069262857142857, "accuracy_std_dev": 0.01966827796016641, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.13709101803751803, "avg_duration_per_task_ms": 571292.8715728716, "avg_output_tokens_per_task": 38969.107503607505, "last_run_timestamp": "2026-09-26T08:52:35.549Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.9066039219088938, "accuracy_std_dev": 0.019658930346988745, "run_count": 14, "total_questions": 2766, "avg_cost_per_task_usd": 0.030758929501084598, "avg_duration_per_task_ms": 288550.2180043384, "avg_output_tokens_per_task": 25664.549168474332, "last_run_timestamp": "2026-09-26T16:08:07.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.905724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.054357919191919186, "avg_duration_per_task_ms": 31472.4797979798, "avg_output_tokens_per_task": 2509.6414141414143, "last_run_timestamp": "2026-09-26T08:54:32.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.9057234999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03042029292929293, "avg_duration_per_task_ms": 203300.4116161616, "avg_output_tokens_per_task": 29610.87373737374, "last_run_timestamp": "2026-09-26T10:10:49.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.9054903294117648, "accuracy_std_dev": 0.002950830881105991, "run_count": 3, "total_questions": 425, "avg_cost_per_task_usd": 0.0986494188235294, "avg_duration_per_task_ms": 590399.5082352941, "avg_output_tokens_per_task": 26203.308235294116, "last_run_timestamp": "2026-09-05T08:31:24.281Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9052352246520876, "accuracy_std_dev": 0.017017287952386474, "run_count": 13, "total_questions": 2515, "avg_cost_per_task_usd": 0.03245243419483101, "avg_duration_per_task_ms": 296394.9419483101, "avg_output_tokens_per_task": 27033.30656063618, "last_run_timestamp": "2026-09-26T16:32:47.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.9051626666666666, "accuracy_std_dev": 0.00649574344389378, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.028482462962962966, "avg_duration_per_task_ms": 244979.07912457912, "avg_output_tokens_per_task": 29233.41919191919, "last_run_timestamp": "2026-09-26T08:52:19.672Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.9041170761421321, "accuracy_std_dev": 0.007233580779177685, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.013811796954314722, "avg_duration_per_task_ms": 592278.3874788494, "avg_output_tokens_per_task": 28886.412859560067, "last_run_timestamp": "2026-09-26T09:15:49.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.9040400000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02929719696969697, "avg_duration_per_task_ms": 146549.648989899, "avg_output_tokens_per_task": 27342.593434343435, "last_run_timestamp": "2026-09-01T13:36:23.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto", "model_slug": "openrouter/auto", "model_name": "Auto Router", "provider_name": "auto-routing", "accuracy": 0.9040399999999998, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02179906313131313, "avg_duration_per_task_ms": 200555.77272727274, "avg_output_tokens_per_task": 25393.28787878788, "last_run_timestamp": "2026-09-15T14:00:47.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9037085815423515, "accuracy_std_dev": 0.00877916433950291, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.033949044247787614, "avg_duration_per_task_ms": 139050.61061946902, "avg_output_tokens_per_task": 28840.025284450065, "last_run_timestamp": "2026-09-26T08:51:27.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/switchyard", "model_slug": "nvidia/switchyard", "model_name": "NVIDIA: Switchyard", "provider_name": "auto-routing", "accuracy": 0.9032489624548736, "accuracy_std_dev": 0.03432136607100001, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.017134858483754508, "avg_duration_per_task_ms": 71272.78700361011, "avg_output_tokens_per_task": 4031.8837545126353, "last_run_timestamp": "2026-09-23T21:40:42.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.9031985, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14676221212121213, "avg_duration_per_task_ms": 106397.32070707071, "avg_output_tokens_per_task": 9655.194444444445, "last_run_timestamp": "2026-09-01T17:54:59.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.9031985, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.043649184343434344, "avg_duration_per_task_ms": 141329.08080808082, "avg_output_tokens_per_task": 28078.323232323233, "last_run_timestamp": "2026-09-01T13:21:17.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.902848790174002, "accuracy_std_dev": 0.012183292931305548, "run_count": 10, "total_questions": 1954, "avg_cost_per_task_usd": 0.030339729273285567, "avg_duration_per_task_ms": 232833.84339815762, "avg_output_tokens_per_task": 25157.37871033777, "last_run_timestamp": "2026-09-26T16:18:02.009Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.90256725, "accuracy_std_dev": 0.031891435832171314, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.042016968434343424, "avg_duration_per_task_ms": 260202.41792929292, "avg_output_tokens_per_task": 31470.176136363636, "last_run_timestamp": "2026-09-26T16:00:24.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9024125477178423, "accuracy_std_dev": 0.026053497227534046, "run_count": 11, "total_questions": 2169, "avg_cost_per_task_usd": 0.04220562056247118, "avg_duration_per_task_ms": 254266.19502074688, "avg_output_tokens_per_task": 31038.156754264637, "last_run_timestamp": "2026-09-26T16:06:21.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.9023565, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.053309800505050506, "avg_duration_per_task_ms": 26495.010101010103, "avg_output_tokens_per_task": 2451.373737373737, "last_run_timestamp": "2026-09-26T08:59:34.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.9013467199999999, "accuracy_std_dev": 0.005345045073860445, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.21570517737373737, "avg_duration_per_task_ms": 94985.5094949495, "avg_output_tokens_per_task": 7862.312121212121, "last_run_timestamp": "2026-09-26T08:55:06.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.901199375, "accuracy_std_dev": 0.005700202593713214, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.2040564772727273, "avg_duration_per_task_ms": 98886.75126262626, "avg_output_tokens_per_task": 7886.399621212121, "last_run_timestamp": "2026-09-26T09:36:04.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.9006730000000001, "accuracy_std_dev": 0.007008969420202443, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1542171708754209, "avg_duration_per_task_ms": 127676.21296296296, "avg_output_tokens_per_task": 9507.647306397306, "last_run_timestamp": "2026-09-01T17:57:22.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.9006104712571067, "accuracy_std_dev": 0.058973902356121696, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.14550911939355657, "avg_duration_per_task_ms": 301854.2046746684, "avg_output_tokens_per_task": 10187.655716993051, "last_run_timestamp": "2026-09-26T10:02:09.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.9004862222222222, "accuracy_std_dev": 0.044346110676956096, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.14017589562289562, "avg_duration_per_task_ms": 217013.79966329967, "avg_output_tokens_per_task": 9267.047699214367, "last_run_timestamp": "2026-09-26T09:14:25.270Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.9003924999999999, "accuracy_std_dev": 0.01706955011855908, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02751066750841751, "avg_duration_per_task_ms": 211685.4351851852, "avg_output_tokens_per_task": 25746.074915824916, "last_run_timestamp": "2026-09-26T08:50:52.330Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.900252375, "accuracy_std_dev": 0.024011889023656058, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.15187610795454545, "avg_duration_per_task_ms": 607070.2234848485, "avg_output_tokens_per_task": 38443.12626262626, "last_run_timestamp": "2026-09-26T09:23:06.229Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.8999515714285714, "accuracy_std_dev": 0.004917020144563225, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.23012572438672438, "avg_duration_per_task_ms": 111331.89033189033, "avg_output_tokens_per_task": 9001.132756132756, "last_run_timestamp": "2026-09-26T08:54:49.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.8998315, "accuracy_std_dev": 0.007575499999999957, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13947029797979799, "avg_duration_per_task_ms": 103780.42929292929, "avg_output_tokens_per_task": 9170.424242424242, "last_run_timestamp": "2026-09-01T17:56:45.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.8996633999999999, "accuracy_std_dev": 0.005792805834826492, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007343192929292929, "avg_duration_per_task_ms": 219337.0404040404, "avg_output_tokens_per_task": 25809.762626262625, "last_run_timestamp": "2026-08-08T09:30:52.456Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8995508333333332, "accuracy_std_dev": 0.004524373673657892, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.031289763468013466, "avg_duration_per_task_ms": 215945.1616161616, "avg_output_tokens_per_task": 26542.147306397306, "last_run_timestamp": "2026-09-26T09:19:06.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.8993638888888887, "accuracy_std_dev": 0.005599097039194869, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.009368053310886646, "avg_duration_per_task_ms": 186510.51515151514, "avg_output_tokens_per_task": 26068.07014590348, "last_run_timestamp": "2026-09-26T08:51:16.648Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.8993387243502051, "accuracy_std_dev": 0.026386926357751014, "run_count": 8, "total_questions": 1462, "avg_cost_per_task_usd": 0.027840691518467853, "avg_duration_per_task_ms": 740625.9062927497, "avg_output_tokens_per_task": 24883.40560875513, "last_run_timestamp": "2026-09-12T16:17:57.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.8989897499999999, "accuracy_std_dev": 0.007241239789393777, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.12451013636363638, "avg_duration_per_task_ms": 29637.002525252527, "avg_output_tokens_per_task": 2284.373737373737, "last_run_timestamp": "2026-09-26T08:53:11.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8987091666666667, "accuracy_std_dev": 0.006255284576970791, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05856475252525253, "avg_duration_per_task_ms": 31561.38973063973, "avg_output_tokens_per_task": 2534.496632996633, "last_run_timestamp": "2026-09-26T09:18:55.607Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.8986660000000002, "accuracy_std_dev": 0.006701794386580333, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.22345247746697744, "avg_duration_per_task_ms": 95310.8921911422, "avg_output_tokens_per_task": 8135.330031080031, "last_run_timestamp": "2026-09-26T09:16:27.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.8978795217391303, "accuracy_std_dev": 0.02199333682676437, "run_count": 7, "total_questions": 1242, "avg_cost_per_task_usd": 0.15500312399355878, "avg_duration_per_task_ms": 574496.192431562, "avg_output_tokens_per_task": 37507.40821256039, "last_run_timestamp": "2026-09-26T09:25:58.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.8978675, "accuracy_std_dev": 0.004085377899697062, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.057460788720538725, "avg_duration_per_task_ms": 25029.281986531987, "avg_output_tokens_per_task": 2489.7180134680134, "last_run_timestamp": "2026-09-26T09:20:45.216Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.897348959390863, "accuracy_std_dev": 0.009933136206097672, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.021320504230118442, "avg_duration_per_task_ms": 601175.5499153977, "avg_output_tokens_per_task": 17639.832487309646, "last_run_timestamp": "2026-09-26T09:15:29.044Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8973064285714285, "accuracy_std_dev": 0.009132616193107635, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.19710293722943723, "avg_duration_per_task_ms": 95577.20057720058, "avg_output_tokens_per_task": 7683.466089466089, "last_run_timestamp": "2026-09-26T09:05:17.040Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.8973061999999999, "accuracy_std_dev": 0.012997001706547546, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03622809090909091, "avg_duration_per_task_ms": 162801.12121212122, "avg_output_tokens_per_task": 27180.768686868687, "last_run_timestamp": "2026-09-26T08:53:11.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.8970957500000001, "accuracy_std_dev": 0.013079356862533413, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.20200678535353536, "avg_duration_per_task_ms": 97703.5946969697, "avg_output_tokens_per_task": 7866.200757575758, "last_run_timestamp": "2026-09-26T08:52:41.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.8967507774936062, "accuracy_std_dev": 0.0073637026710600875, "run_count": 18, "total_questions": 3519, "avg_cost_per_task_usd": 0.12523853253765274, "avg_duration_per_task_ms": 58154.57402671214, "avg_output_tokens_per_task": 4750.331628303496, "last_run_timestamp": "2026-09-26T08:53:17.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.8964645000000001, "accuracy_std_dev": 0.007575499999999957, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03375818181818182, "avg_duration_per_task_ms": 56405.70202020202, "avg_output_tokens_per_task": 3599.8333333333335, "last_run_timestamp": "2026-09-01T13:22:46.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.8961193926247291, "accuracy_std_dev": 0.016375418175062377, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.11571196312364425, "avg_duration_per_task_ms": 621953.1062906724, "avg_output_tokens_per_task": 36359.131597975414, "last_run_timestamp": "2026-09-26T09:20:04.760Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.8960958972972972, "accuracy_std_dev": 0.005877557334660528, "run_count": 3, "total_questions": 555, "avg_cost_per_task_usd": 0.1555473855855856, "avg_duration_per_task_ms": 83534.15135135135, "avg_output_tokens_per_task": 9299.446846846848, "last_run_timestamp": "2026-09-01T17:56:10.791Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.8959085241877256, "accuracy_std_dev": 0.035618009581258374, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.11392027942238267, "avg_duration_per_task_ms": 563860.525631769, "avg_output_tokens_per_task": 38531.1083032491, "last_run_timestamp": "2026-09-26T09:10:17.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.895623, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.016944118686868685, "avg_duration_per_task_ms": 304656.29545454547, "avg_output_tokens_per_task": 21473.98232323232, "last_run_timestamp": "2026-09-26T08:26:07.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.89562275, "accuracy_std_dev": 0.0051888143816771525, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.057555659090909095, "avg_duration_per_task_ms": 24846.121212121212, "avg_output_tokens_per_task": 2521.655303030303, "last_run_timestamp": "2026-09-26T09:36:25.776Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.8954697272727271, "accuracy_std_dev": 0.006025463303060488, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.20493744031221303, "avg_duration_per_task_ms": 100289.02800734619, "avg_output_tokens_per_task": 7987.1299357208445, "last_run_timestamp": "2026-09-26T08:51:26.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.8953789065420559, "accuracy_std_dev": 0.010355017014034617, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.012003740654205609, "avg_duration_per_task_ms": 284653.5062305296, "avg_output_tokens_per_task": 27098.440809968848, "last_run_timestamp": "2026-09-26T08:56:10.606Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8953486936299293, "accuracy_std_dev": 0.019178869929169434, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.03527696764408493, "avg_duration_per_task_ms": 158313.13549039434, "avg_output_tokens_per_task": 26466.745197168857, "last_run_timestamp": "2026-09-26T09:36:01.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.8950614444444446, "accuracy_std_dev": 0.007975755054623836, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.12023811111111112, "avg_duration_per_task_ms": 57861.70202020202, "avg_output_tokens_per_task": 4595.748597081931, "last_run_timestamp": "2026-09-26T08:56:21.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.8949365645569619, "accuracy_std_dev": 0.017816727566741866, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.021203940084388185, "avg_duration_per_task_ms": 226768.21476793248, "avg_output_tokens_per_task": 26685.461603375526, "last_run_timestamp": "2026-09-26T08:50:36.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.8946420862998922, "accuracy_std_dev": 0.007735019467331668, "run_count": 5, "total_questions": 927, "avg_cost_per_task_usd": 0.006440504854368933, "avg_duration_per_task_ms": 428399.7303128371, "avg_output_tokens_per_task": 13363.50377562028, "last_run_timestamp": "2026-09-26T08:57:06.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8939478279932548, "accuracy_std_dev": 0.009294310315828362, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.22570986565486229, "avg_duration_per_task_ms": 108477.89263631254, "avg_output_tokens_per_task": 8804.630691399663, "last_run_timestamp": "2026-09-26T08:50:43.725Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AkashML", "accuracy": 0.893939, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03405869696969697, "avg_duration_per_task_ms": 153813.0883838384, "avg_output_tokens_per_task": 9711.138888888889, "last_run_timestamp": "2026-06-20T09:09:33.534Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.8934859242520689, "accuracy_std_dev": 0.016678015536459194, "run_count": 8, "total_questions": 1571, "avg_cost_per_task_usd": 0.11211757733927435, "avg_duration_per_task_ms": 393101.5945257798, "avg_output_tokens_per_task": 7662.472310630172, "last_run_timestamp": "2026-09-26T09:51:40.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.8932607767114523, "accuracy_std_dev": 0.024235313237801887, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.043238984005118365, "avg_duration_per_task_ms": 591335.9296225208, "avg_output_tokens_per_task": 23011.071657069737, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.8931287407407407, "accuracy_std_dev": 0.007059243819500409, "run_count": 27, "total_questions": 5346, "avg_cost_per_task_usd": 0.2046095759446315, "avg_duration_per_task_ms": 71087.34867190423, "avg_output_tokens_per_task": 7454.418069584736, "last_run_timestamp": "2026-09-26T09:18:54.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8930976666666668, "accuracy_std_dev": 0.010182556173290759, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05509197895622895, "avg_duration_per_task_ms": 49604.97474747475, "avg_output_tokens_per_task": 3808.635521885522, "last_run_timestamp": "2026-09-01T13:23:50.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8930975, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.054297787878787876, "avg_duration_per_task_ms": 31463.27525252525, "avg_output_tokens_per_task": 2503.689393939394, "last_run_timestamp": "2026-09-26T08:53:18.330Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.892567903763697, "accuracy_std_dev": 0.02022831814050264, "run_count": 11, "total_questions": 2099, "avg_cost_per_task_usd": 0.15797696188661267, "avg_duration_per_task_ms": 338559.5883754169, "avg_output_tokens_per_task": 39638.616484040016, "last_run_timestamp": "2026-09-19T08:51:08.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.8920512903225806, "accuracy_std_dev": 0.019911726970242915, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.13115581593927894, "avg_duration_per_task_ms": 517762.9652118912, "avg_output_tokens_per_task": 37601.20493358634, "last_run_timestamp": "2026-09-26T09:00:46.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8918440425531915, "accuracy_std_dev": 0.0082731599766957, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.050205347517730495, "avg_duration_per_task_ms": 44870.67021276596, "avg_output_tokens_per_task": 3654.1524822695037, "last_run_timestamp": "2026-09-01T13:21:31.062Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.8914139999999999, "accuracy_std_dev": 0.023312104420236266, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.086096711489899, "avg_duration_per_task_ms": 172501.05744949495, "avg_output_tokens_per_task": 29063.86047979798, "last_run_timestamp": "2026-09-26T16:08:44.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Parasail", "accuracy": 0.8914139999999999, "accuracy_std_dev": 0.017677, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04263949494949495, "avg_duration_per_task_ms": 361913.06313131313, "avg_output_tokens_per_task": 15178.89898989899, "last_run_timestamp": "2026-05-23T08:52:13.739Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Decart", "accuracy": 0.8914139999999999, "accuracy_std_dev": 0.007575999999999972, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03735867676767677, "avg_duration_per_task_ms": 78882.84090909091, "avg_output_tokens_per_task": 10904.39393939394, "last_run_timestamp": "2026-06-20T08:34:12.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.8913139844097995, "accuracy_std_dev": 0.016321403571860956, "run_count": 12, "total_questions": 2245, "avg_cost_per_task_usd": 0.09373735100222716, "avg_duration_per_task_ms": 534503.8146993319, "avg_output_tokens_per_task": 28512.390200445436, "last_run_timestamp": "2026-09-26T08:56:57.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8905724999999999, "accuracy_std_dev": 0.011784499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.027024376262626264, "avg_duration_per_task_ms": 278293.2196969697, "avg_output_tokens_per_task": 26273.641414141413, "last_run_timestamp": "2026-09-19T09:00:31.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.890572, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.21863568181818183, "avg_duration_per_task_ms": 106364.56818181818, "avg_output_tokens_per_task": 8520.704545454546, "last_run_timestamp": "2026-09-01T13:30:25.730Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.890572, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1261595202020202, "avg_duration_per_task_ms": 738082.0303030303, "avg_output_tokens_per_task": 50200.232323232325, "last_run_timestamp": "2026-09-23T06:12:17.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.8905389900990098, "accuracy_std_dev": 0.007335499999999995, "run_count": 2, "total_questions": 303, "avg_cost_per_task_usd": 0.01631713201320132, "avg_duration_per_task_ms": 272632.9339933993, "avg_output_tokens_per_task": 22974.214521452144, "last_run_timestamp": "2026-09-26T09:25:09.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.8897304999999999, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0318945303030303, "avg_duration_per_task_ms": 52439.469696969696, "avg_output_tokens_per_task": 3386.712121212121, "last_run_timestamp": "2026-09-01T13:38:25.421Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.8897192682926828, "accuracy_std_dev": 0.012292154908071387, "run_count": 13, "total_questions": 2542, "avg_cost_per_task_usd": 0.0036178985051140834, "avg_duration_per_task_ms": 277074.0971675846, "avg_output_tokens_per_task": 19808.066876475215, "last_run_timestamp": "2026-09-26T09:01:36.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.888889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00832349494949495, "avg_duration_per_task_ms": 86259.9494949495, "avg_output_tokens_per_task": 13087.217171717171, "last_run_timestamp": "2026-09-24T09:11:45.985Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.8888888430379747, "accuracy_std_dev": 0.009622693519820156, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.21254848649789027, "avg_duration_per_task_ms": 102777.66371308017, "avg_output_tokens_per_task": 8273.845147679325, "last_run_timestamp": "2026-09-26T09:15:06.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.8887592307692308, "accuracy_std_dev": 0.013361790161926698, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.012804753690753689, "avg_duration_per_task_ms": 213849.3986013986, "avg_output_tokens_per_task": 24454.132867132867, "last_run_timestamp": "2026-09-26T08:57:03.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.888552, "accuracy_std_dev": 0.03117712682079604, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1316316707070707, "avg_duration_per_task_ms": 39126.25959595959, "avg_output_tokens_per_task": 2423.617171717172, "last_run_timestamp": "2026-09-26T08:53:20.595Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8883868300395255, "accuracy_std_dev": 0.008563760488459709, "run_count": 27, "total_questions": 5313, "avg_cost_per_task_usd": 0.1336053077357425, "avg_duration_per_task_ms": 59533.9862601167, "avg_output_tokens_per_task": 4780.512516469038, "last_run_timestamp": "2026-09-26T09:19:13.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.8876862857142855, "accuracy_std_dev": 0.018697250017157118, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.13258516161616163, "avg_duration_per_task_ms": 538569.6168831169, "avg_output_tokens_per_task": 33498.70923520924, "last_run_timestamp": "2026-09-26T09:29:09.038Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.8875662777777777, "accuracy_std_dev": 0.03225232181080149, "run_count": 4, "total_questions": 756, "avg_cost_per_task_usd": 0.0045816917989418, "avg_duration_per_task_ms": 604811.5727513228, "avg_output_tokens_per_task": 27586.727513227514, "last_run_timestamp": "2026-09-26T08:58:10.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.8872987222222223, "accuracy_std_dev": 0.006285923036662124, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.24028202581369246, "avg_duration_per_task_ms": 84935.12850729517, "avg_output_tokens_per_task": 8938.721380471381, "last_run_timestamp": "2026-09-26T08:53:09.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.8872055000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20363688131313135, "avg_duration_per_task_ms": 99426.9191919192, "avg_output_tokens_per_task": 7920.752525252526, "last_run_timestamp": "2026-09-01T13:21:53.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.8868242601351352, "accuracy_std_dev": 0.009599924258960477, "run_count": 9, "total_questions": 1776, "avg_cost_per_task_usd": 0.20454106644144146, "avg_duration_per_task_ms": 100059.57263513513, "avg_output_tokens_per_task": 7955.086148648648, "last_run_timestamp": "2026-09-26T08:50:54.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.8866439999999999, "accuracy_std_dev": 0.012472582491208459, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008770978114478115, "avg_duration_per_task_ms": 61645.40740740741, "avg_output_tokens_per_task": 12897.501683501683, "last_run_timestamp": "2026-09-24T09:11:57.282Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8864601803278688, "accuracy_std_dev": 0.006211138077858382, "run_count": 3, "total_questions": 549, "avg_cost_per_task_usd": 0.009764420765027322, "avg_duration_per_task_ms": 50630.12750455373, "avg_output_tokens_per_task": 7246.790528233151, "last_run_timestamp": "2026-09-01T17:51:11.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.8864530691399664, "accuracy_std_dev": 0.01029973730112676, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.023206354974704888, "avg_duration_per_task_ms": 81414.45531197301, "avg_output_tokens_per_task": 7880.962900505902, "last_run_timestamp": "2026-09-01T17:55:01.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.8863635000000001, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0069983762626262625, "avg_duration_per_task_ms": 635651.6565656565, "avg_output_tokens_per_task": 21974.18181818182, "last_run_timestamp": "2026-09-26T10:42:27.487Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8862431428571429, "accuracy_std_dev": 0.008624548475610979, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.016605583694083693, "avg_duration_per_task_ms": 131101.75468975469, "avg_output_tokens_per_task": 25668.465367965367, "last_run_timestamp": "2026-09-26T08:53:15.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.8860826666666668, "accuracy_std_dev": 0.004199386807089298, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0278726632996633, "avg_duration_per_task_ms": 157091.45622895623, "avg_output_tokens_per_task": 23053.37037037037, "last_run_timestamp": "2026-09-26T09:47:15.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-max", "model_slug": "sakana/fugu-max", "model_name": "Fugu Max", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07555469191919191, "avg_duration_per_task_ms": 225091.94949494948, "avg_output_tokens_per_task": 12279.550505050505, "last_run_timestamp": "2026-09-15T01:07:47.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.8849736038095237, "accuracy_std_dev": 0.03742100947662959, "run_count": 8, "total_questions": 1575, "avg_cost_per_task_usd": 0.1364461853968254, "avg_duration_per_task_ms": 794770.7498412698, "avg_output_tokens_per_task": 34155.479365079365, "last_run_timestamp": "2026-09-26T10:09:19.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8849202417582418, "accuracy_std_dev": 0.01938538920596302, "run_count": 3, "total_questions": 546, "avg_cost_per_task_usd": 0.04324962454212454, "avg_duration_per_task_ms": 205056.15750915752, "avg_output_tokens_per_task": 33761.00366300366, "last_run_timestamp": "2026-09-26T10:02:16.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.88468, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009158123737373737, "avg_duration_per_task_ms": 142387.94696969696, "avg_output_tokens_per_task": 32237.108585858587, "last_run_timestamp": "2026-09-26T09:33:13.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.8845707161572053, "accuracy_std_dev": 0.02143221713073307, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.16085208820960697, "avg_duration_per_task_ms": 756655.7283842795, "avg_output_tokens_per_task": 40813.194759825325, "last_run_timestamp": "2026-09-16T00:37:42.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.8841129772439951, "accuracy_std_dev": 0.0033388888402431233, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.011443802781289506, "avg_duration_per_task_ms": 364670.14664981037, "avg_output_tokens_per_task": 22004.87104930468, "last_run_timestamp": "2026-09-26T12:21:31.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.8839537178510998, "accuracy_std_dev": 0.013099282603342133, "run_count": 12, "total_questions": 2364, "avg_cost_per_task_usd": 0.00725531429780034, "avg_duration_per_task_ms": 489249.17766497465, "avg_output_tokens_per_task": 25997.4179357022, "last_run_timestamp": "2026-09-26T08:56:41.992Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11161540404040404, "avg_duration_per_task_ms": 930549.2727272727, "avg_output_tokens_per_task": 44407.63636363636, "last_run_timestamp": "2026-09-23T15:38:01.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8835014000000001, "accuracy_std_dev": 0.025562678362018323, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.14118799797979797, "avg_duration_per_task_ms": 41635.47171717172, "avg_output_tokens_per_task": 2614.5828282828284, "last_run_timestamp": "2026-09-26T08:54:00.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.8834305684210525, "accuracy_std_dev": 0.01461251838899233, "run_count": 13, "total_questions": 2565, "avg_cost_per_task_usd": 0.10276526354775828, "avg_duration_per_task_ms": 556976.750877193, "avg_output_tokens_per_task": 26780.168031189085, "last_run_timestamp": "2026-09-26T09:11:00.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8832771111111111, "accuracy_std_dev": 0.009589207202132645, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02110603142536476, "avg_duration_per_task_ms": 350549.90852974186, "avg_output_tokens_per_task": 24400.115039281707, "last_run_timestamp": "2026-09-26T08:58:18.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.8829965, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13198183333333333, "avg_duration_per_task_ms": 254155.68181818182, "avg_output_tokens_per_task": 43876.38636363636, "last_run_timestamp": "2026-09-01T17:57:36.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.882716, "accuracy_std_dev": 0.006496204173720757, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.027370449494949496, "avg_duration_per_task_ms": 130101.09932659933, "avg_output_tokens_per_task": 22617.942760942762, "last_run_timestamp": "2026-09-26T10:08:37.923Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.8825515716096324, "accuracy_std_dev": 0.020268991676360943, "run_count": 12, "total_questions": 2367, "avg_cost_per_task_usd": 0.1375574414871145, "avg_duration_per_task_ms": 860088.601182932, "avg_output_tokens_per_task": 38561.9945078158, "last_run_timestamp": "2026-08-29T18:18:56.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8824913999999999, "accuracy_std_dev": 0.008205959897537906, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.015855806060606062, "avg_duration_per_task_ms": 286632.0282828283, "avg_output_tokens_per_task": 24219.753535353535, "last_run_timestamp": "2026-09-26T09:30:12.511Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.8822544579439251, "accuracy_std_dev": 0.049358067293064335, "run_count": 15, "total_questions": 2889, "avg_cost_per_task_usd": 0.07833941329179646, "avg_duration_per_task_ms": 744926.2416060921, "avg_output_tokens_per_task": 34819.21737625476, "last_run_timestamp": "2026-08-29T20:15:38.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8822322634862387, "accuracy_std_dev": 0.012422583804613795, "run_count": 28, "total_questions": 5450, "avg_cost_per_task_usd": 0.1723975299082569, "avg_duration_per_task_ms": 339570.9011009174, "avg_output_tokens_per_task": 27806.31211009174, "last_run_timestamp": "2026-09-26T08:58:33.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.8821546666666666, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02880540235690236, "avg_duration_per_task_ms": 99924.20707070707, "avg_output_tokens_per_task": 23871.781144781144, "last_run_timestamp": "2026-09-26T09:49:42.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8821545714285715, "accuracy_std_dev": 0.014256633562332045, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.18132904473304476, "avg_duration_per_task_ms": 297976.3398268398, "avg_output_tokens_per_task": 45509.72438672439, "last_run_timestamp": "2026-09-26T09:19:29.410Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8818613645976111, "accuracy_std_dev": 0.010002367123265743, "run_count": 27, "total_questions": 5107, "avg_cost_per_task_usd": 0.1356296686900333, "avg_duration_per_task_ms": 62377.717642451535, "avg_output_tokens_per_task": 4931.54278441355, "last_run_timestamp": "2026-09-26T08:58:07.474Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.8806582222222221, "accuracy_std_dev": 0.005109133929167741, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.005640715488215489, "avg_duration_per_task_ms": 362315.8209876543, "avg_output_tokens_per_task": 28732.147025813694, "last_run_timestamp": "2026-09-26T09:34:56.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.880471, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0032475000000000004, "avg_duration_per_task_ms": 58949.292929292926, "avg_output_tokens_per_task": 5746.025252525253, "last_run_timestamp": "2026-09-23T06:09:34.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.8802532590673576, "accuracy_std_dev": 0.005991714074916775, "run_count": 3, "total_questions": 579, "avg_cost_per_task_usd": 0.14682200690846287, "avg_duration_per_task_ms": 338397.36614853196, "avg_output_tokens_per_task": 33348.9585492228, "last_run_timestamp": "2026-07-18T08:26:40.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8799936573033706, "accuracy_std_dev": 0.07692276680482653, "run_count": 12, "total_questions": 2136, "avg_cost_per_task_usd": 0.008731384363295881, "avg_duration_per_task_ms": 144796.46582397004, "avg_output_tokens_per_task": 23515.395599250936, "last_run_timestamp": "2026-09-26T08:54:04.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8798644568527919, "accuracy_std_dev": 0.004527248931623837, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.0029938477157360403, "avg_duration_per_task_ms": 316818.90016920475, "avg_output_tokens_per_task": 18321.88663282572, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.8792515918367347, "accuracy_std_dev": 0.00992611788026112, "run_count": 9, "total_questions": 1764, "avg_cost_per_task_usd": 0.004467375283446712, "avg_duration_per_task_ms": 383208.2947845805, "avg_output_tokens_per_task": 18476.153628117914, "last_run_timestamp": "2026-09-26T10:33:25.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.879189, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.020723571428571428, "avg_duration_per_task_ms": 141095.54497354498, "avg_output_tokens_per_task": 17734.21164021164, "last_run_timestamp": "2026-09-15T00:33:26.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "WandB", "accuracy": 0.8791716692015209, "accuracy_std_dev": 0.009087349844013949, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.05804701140684411, "avg_duration_per_task_ms": 798447.8948035488, "avg_output_tokens_per_task": 16682.354879594423, "last_run_timestamp": "2026-07-11T08:42:46.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8790546552363299, "accuracy_std_dev": 0.030844451113161732, "run_count": 6, "total_questions": 1079, "avg_cost_per_task_usd": 0.1346450278035218, "avg_duration_per_task_ms": 284974.0537534754, "avg_output_tokens_per_task": 31126.302131603337, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8788689803571428, "accuracy_std_dev": 0.011915461634880886, "run_count": 17, "total_questions": 3360, "avg_cost_per_task_usd": 0.0034551410714285715, "avg_duration_per_task_ms": 175260.2726190476, "avg_output_tokens_per_task": 17584.270833333332, "last_run_timestamp": "2026-08-29T18:22:03.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13332395959595958, "avg_duration_per_task_ms": 138478.24242424243, "avg_output_tokens_per_task": 9426.934343434343, "last_run_timestamp": "2026-08-01T08:50:28.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8787875, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0075271136363636365, "avg_duration_per_task_ms": 80896.84595959596, "avg_output_tokens_per_task": 8213.79797979798, "last_run_timestamp": "2026-09-01T17:51:13.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8787801424227066, "accuracy_std_dev": 0.016401176693323544, "run_count": 11, "total_questions": 1973, "avg_cost_per_task_usd": 0.011341015205271159, "avg_duration_per_task_ms": 261740.6791687785, "avg_output_tokens_per_task": 24571.48302078054, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8785529767441861, "accuracy_std_dev": 0.101852, "run_count": 2, "total_questions": 258, "avg_cost_per_task_usd": 0.1083256472868217, "avg_duration_per_task_ms": 620596.2751937985, "avg_output_tokens_per_task": 16165.639534883721, "last_run_timestamp": "2026-09-01T16:19:34.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8784510000000001, "accuracy_std_dev": 0.011536704780828876, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.022268736363636365, "avg_duration_per_task_ms": 229673.4696969697, "avg_output_tokens_per_task": 24972.11919191919, "last_run_timestamp": "2026-09-26T09:19:28.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.8784135555555556, "accuracy_std_dev": 0.017145903852336865, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.023216065656565654, "avg_duration_per_task_ms": 214871.89786756455, "avg_output_tokens_per_task": 24869.92312008979, "last_run_timestamp": "2026-09-26T09:39:38.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8783643322981367, "accuracy_std_dev": 0.020055520558938385, "run_count": 10, "total_questions": 1932, "avg_cost_per_task_usd": 0.1293156180124224, "avg_duration_per_task_ms": 63254.61180124224, "avg_output_tokens_per_task": 4948.346273291925, "last_run_timestamp": "2026-09-26T08:52:39.749Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.8781917204968944, "accuracy_std_dev": 0.004843231342812369, "run_count": 5, "total_questions": 966, "avg_cost_per_task_usd": 0.005745680124223602, "avg_duration_per_task_ms": 442762.4254658385, "avg_output_tokens_per_task": 20256.71635610766, "last_run_timestamp": "2026-07-18T08:31:23.912Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8779459999999999, "accuracy_std_dev": 0.010801196631237997, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01636711111111111, "avg_duration_per_task_ms": 223245.67845117845, "avg_output_tokens_per_task": 24540.49494949495, "last_run_timestamp": "2026-09-26T10:45:50.086Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.8779459999999999, "accuracy_std_dev": 0.014310000000000045, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13208357575757576, "avg_duration_per_task_ms": 257170.34595959596, "avg_output_tokens_per_task": 43907.46464646464, "last_run_timestamp": "2026-09-01T17:57:59.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.8779238157894738, "accuracy_std_dev": 0.08964719889246571, "run_count": 14, "total_questions": 2736, "avg_cost_per_task_usd": 0.02495466630116959, "avg_duration_per_task_ms": 199904.41374269006, "avg_output_tokens_per_task": 21043.42909356725, "last_run_timestamp": "2026-09-26T16:07:03.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.8776173653429603, "accuracy_std_dev": 0.01555889568113131, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.08483577111913358, "avg_duration_per_task_ms": 220187.85559566788, "avg_output_tokens_per_task": 21593.077978339352, "last_run_timestamp": "2026-09-26T10:06:10.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8771045000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14050905303030303, "avg_duration_per_task_ms": 185398.66919191918, "avg_output_tokens_per_task": 46713.32070707071, "last_run_timestamp": "2026-09-01T17:54:30.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Nebius", "accuracy": 0.8771041666666666, "accuracy_std_dev": 0.010779529510707913, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0291337297979798, "avg_duration_per_task_ms": 93389.48653198653, "avg_output_tokens_per_task": 8046.875420875421, "last_run_timestamp": "2026-05-23T16:38:19.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8771039999999999, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007407005050505051, "avg_duration_per_task_ms": 78924.21717171717, "avg_output_tokens_per_task": 8106.714646464647, "last_run_timestamp": "2026-09-01T17:50:43.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.8770211787234045, "accuracy_std_dev": 0.03195742141593335, "run_count": 6, "total_questions": 1175, "avg_cost_per_task_usd": 0.0032351353191489365, "avg_duration_per_task_ms": 354647.9063829787, "avg_output_tokens_per_task": 26251.109787234043, "last_run_timestamp": "2026-09-19T08:54:45.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.8768792503888027, "accuracy_std_dev": 0.0068076105343578604, "run_count": 4, "total_questions": 643, "avg_cost_per_task_usd": 0.007664079315707621, "avg_duration_per_task_ms": 987610.4805598756, "avg_output_tokens_per_task": 30419.066874027994, "last_run_timestamp": "2026-09-26T09:19:44.251Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8765966933797908, "accuracy_std_dev": 0.020189124617118887, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.004125730836236933, "avg_duration_per_task_ms": 453615.28397212544, "avg_output_tokens_per_task": 16054.564459930314, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.8759223861480078, "accuracy_std_dev": 0.007557144477403089, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.005635244149272613, "avg_duration_per_task_ms": 187978.53478810878, "avg_output_tokens_per_task": 19921.54459203036, "last_run_timestamp": "2026-08-29T19:12:25.584Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.8754206666666666, "accuracy_std_dev": 0.00901385812081723, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.061844909090909096, "avg_duration_per_task_ms": 262173.4595959596, "avg_output_tokens_per_task": 22965.14478114478, "last_run_timestamp": "2026-09-26T09:19:06.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.8748596666666667, "accuracy_std_dev": 0.006198499675100621, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.003131114478114478, "avg_duration_per_task_ms": 32869.14646464647, "avg_output_tokens_per_task": 5711.222222222223, "last_run_timestamp": "2026-09-23T06:08:44.627Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.8746655940919037, "accuracy_std_dev": 0.006656857328717377, "run_count": 14, "total_questions": 2742, "avg_cost_per_task_usd": 0.092771409190372, "avg_duration_per_task_ms": 766984.4496717724, "avg_output_tokens_per_task": 32707.43873085339, "last_run_timestamp": "2026-08-29T19:07:02.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.8745582826855123, "accuracy_std_dev": 0.047501460387912024, "run_count": 8, "total_questions": 1415, "avg_cost_per_task_usd": 0.07384584240282685, "avg_duration_per_task_ms": 519959.2473498233, "avg_output_tokens_per_task": 21893.836042402825, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.8745317312013828, "accuracy_std_dev": 0.011186512454241005, "run_count": 6, "total_questions": 1157, "avg_cost_per_task_usd": 0.0043274416594641305, "avg_duration_per_task_ms": 185028.52031114954, "avg_output_tokens_per_task": 21870.51080380294, "last_run_timestamp": "2026-09-26T08:53:59.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.8744798294036062, "accuracy_std_dev": 0.034886602236436054, "run_count": 11, "total_questions": 2163, "avg_cost_per_task_usd": 0.13725122098936662, "avg_duration_per_task_ms": 368551.85991678224, "avg_output_tokens_per_task": 32192.02172907998, "last_run_timestamp": "2026-09-26T08:53:37.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.8744172980132449, "accuracy_std_dev": 0.016480704550770754, "run_count": 7, "total_questions": 1359, "avg_cost_per_task_usd": 0.005027246504782929, "avg_duration_per_task_ms": 379290.6011773363, "avg_output_tokens_per_task": 17886.896983075792, "last_run_timestamp": "2026-07-18T08:30:06.336Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8740740000000001, "accuracy_std_dev": 0.03472083287595504, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1182012676767677, "avg_duration_per_task_ms": 28405.98686868687, "avg_output_tokens_per_task": 2155.238383838384, "last_run_timestamp": "2026-09-26T08:52:17.920Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.874074, "accuracy_std_dev": 0.004178433821421624, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07029311111111111, "avg_duration_per_task_ms": 171365.84242424244, "avg_output_tokens_per_task": 11443.449494949495, "last_run_timestamp": "2026-09-26T09:10:25.930Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8737373, "accuracy_std_dev": 0.01570253542616605, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.11062107323232323, "avg_duration_per_task_ms": 229265.31616161615, "avg_output_tokens_per_task": 31458.501515151514, "last_run_timestamp": "2026-09-26T08:52:34.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.8737372500000001, "accuracy_std_dev": 0.019341996831700202, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12990520643939396, "avg_duration_per_task_ms": 63171.816287878784, "avg_output_tokens_per_task": 4978.773358585859, "last_run_timestamp": "2026-09-26T08:54:39.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.8728743269428278, "accuracy_std_dev": 0.022779113769318488, "run_count": 15, "total_questions": 2921, "avg_cost_per_task_usd": 0.060696329681615875, "avg_duration_per_task_ms": 329375.96234166384, "avg_output_tokens_per_task": 22546.172543649434, "last_run_timestamp": "2026-08-22T17:03:07.750Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.8724287134608445, "accuracy_std_dev": 0.032129376762123, "run_count": 14, "total_questions": 2771, "avg_cost_per_task_usd": 0.1523031241429087, "avg_duration_per_task_ms": 425564.390111873, "avg_output_tokens_per_task": 11498.582100324793, "last_run_timestamp": "2026-09-26T09:12:17.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.872292240506329, "accuracy_std_dev": 0.013194491052666223, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.00790528016877637, "avg_duration_per_task_ms": 421318.1687763713, "avg_output_tokens_per_task": 21316.46075949367, "last_run_timestamp": "2026-09-26T09:05:47.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.8722870891089111, "accuracy_std_dev": 0.011050513265882567, "run_count": 17, "total_questions": 3333, "avg_cost_per_task_usd": 0.005860441644164416, "avg_duration_per_task_ms": 188504.78847884788, "avg_output_tokens_per_task": 20725.64086408641, "last_run_timestamp": "2026-08-29T18:42:44.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.8722740210280374, "accuracy_std_dev": 0.016757790257300998, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.004785041277258566, "avg_duration_per_task_ms": 167818.7609034268, "avg_output_tokens_per_task": 16930.278426791276, "last_run_timestamp": "2026-08-08T10:06:09.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8720540000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.22419905555555555, "avg_duration_per_task_ms": 276263.6666666667, "avg_output_tokens_per_task": 22325.227272727272, "last_run_timestamp": "2026-09-01T13:30:31.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AkashML", "accuracy": 0.8720536666666666, "accuracy_std_dev": 0.008584133982075453, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.07818634427609428, "avg_duration_per_task_ms": 229996.4814814815, "avg_output_tokens_per_task": 19586.67255892256, "last_run_timestamp": "2026-06-13T09:38:25.896Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.871542434599156, "accuracy_std_dev": 0.007338207797953819, "run_count": 11, "total_questions": 2133, "avg_cost_per_task_usd": 0.0043919292076887015, "avg_duration_per_task_ms": 387373.37552742613, "avg_output_tokens_per_task": 23530.04781997187, "last_run_timestamp": "2026-08-29T19:32:50.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8714926666666667, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03188070202020202, "avg_duration_per_task_ms": 240511.38888888888, "avg_output_tokens_per_task": 26418.183501683503, "last_run_timestamp": "2026-09-26T08:53:40.686Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.871212, "accuracy_std_dev": 0.010943000000000036, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.25165717424242423, "avg_duration_per_task_ms": 248967.9292929293, "avg_output_tokens_per_task": 23863.48232323232, "last_run_timestamp": "2026-09-01T13:53:50.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8711222842639593, "accuracy_std_dev": 0.0010748023074035804, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.04708872588832487, "avg_duration_per_task_ms": 168365.97292724196, "avg_output_tokens_per_task": 39058.98984771573, "last_run_timestamp": "2026-09-26T10:21:09.898Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.8710750330630069, "accuracy_std_dev": 0.028110456226890915, "run_count": 9, "total_questions": 1603, "avg_cost_per_task_usd": 0.09870775982532752, "avg_duration_per_task_ms": 221158.11790393014, "avg_output_tokens_per_task": 26829.437305053027, "last_run_timestamp": "2026-09-26T09:19:13.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8710435000000001, "accuracy_std_dev": 0.015469940324707153, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.007902119696969698, "avg_duration_per_task_ms": 203714.64696969697, "avg_output_tokens_per_task": 28019.556565656567, "last_run_timestamp": "2026-09-26T09:36:46.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.8710124489402697, "accuracy_std_dev": 0.014206989600055803, "run_count": 16, "total_questions": 3114, "avg_cost_per_task_usd": 0.005702225112395633, "avg_duration_per_task_ms": 187380.58863198457, "avg_output_tokens_per_task": 21072.890494540785, "last_run_timestamp": "2026-08-29T19:20:21.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.8710116425661915, "accuracy_std_dev": 0.01025225770257459, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.025059682281059068, "avg_duration_per_task_ms": 328330.6405295316, "avg_output_tokens_per_task": 20642.19348268839, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8709859808450705, "accuracy_std_dev": 0.00995822887470397, "run_count": 9, "total_questions": 1775, "avg_cost_per_task_usd": 0.007974392676056338, "avg_duration_per_task_ms": 311914.2202816901, "avg_output_tokens_per_task": 28209.84957746479, "last_run_timestamp": "2026-09-26T09:58:11.632Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.8709169574970483, "accuracy_std_dev": 0.025335242123251705, "run_count": 13, "total_questions": 2541, "avg_cost_per_task_usd": 0.09988292601338057, "avg_duration_per_task_ms": 696367.0350255804, "avg_output_tokens_per_task": 26502.97284533648, "last_run_timestamp": "2026-09-26T09:22:13.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.8705803019038983, "accuracy_std_dev": 0.09978272834395396, "run_count": 13, "total_questions": 2206, "avg_cost_per_task_usd": 0.01978426609247507, "avg_duration_per_task_ms": 262537.4936536718, "avg_output_tokens_per_task": 16415.930190389845, "last_run_timestamp": "2026-09-26T17:07:03.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.8700592503816793, "accuracy_std_dev": 0.011904817070833141, "run_count": 10, "total_questions": 1965, "avg_cost_per_task_usd": 0.03175990890585242, "avg_duration_per_task_ms": 495176.59541984735, "avg_output_tokens_per_task": 27384.936386768448, "last_run_timestamp": "2026-08-29T19:08:19.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.8700495714285714, "accuracy_std_dev": 0.014023880162648517, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.0307443746993747, "avg_duration_per_task_ms": 207892.97162097163, "avg_output_tokens_per_task": 13057.428811928812, "last_run_timestamp": "2026-08-29T18:36:08.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8699951448047163, "accuracy_std_dev": 0.03969212027542979, "run_count": 14, "total_questions": 2714, "avg_cost_per_task_usd": 0.00531859358879882, "avg_duration_per_task_ms": 114563.89867354458, "avg_output_tokens_per_task": 20095.400515843772, "last_run_timestamp": "2026-09-26T08:53:37.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.8699495, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00502325, "avg_duration_per_task_ms": 389434.8611111111, "avg_output_tokens_per_task": 17944.10101010101, "last_run_timestamp": "2026-08-08T10:05:07.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8695795853658537, "accuracy_std_dev": 0.014776870412912198, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.010841884146341461, "avg_duration_per_task_ms": 398387.18699186994, "avg_output_tokens_per_task": 22967.41768292683, "last_run_timestamp": "2026-09-26T09:27:32.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10538721464646464, "avg_duration_per_task_ms": 581981.0101010101, "avg_output_tokens_per_task": 34695.7398989899, "last_run_timestamp": "2026-08-08T09:03:54.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.869318125, "accuracy_std_dev": 0.03295688160474795, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10640666414141413, "avg_duration_per_task_ms": 513815.1142676768, "avg_output_tokens_per_task": 34567.331439393936, "last_run_timestamp": "2026-09-26T08:56:11.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.8692809647058825, "accuracy_std_dev": 0.009104176033557385, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.0047775359477124175, "avg_duration_per_task_ms": 482494.045751634, "avg_output_tokens_per_task": 26353.58431372549, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.8691380874524715, "accuracy_std_dev": 0.016741224006923745, "run_count": 16, "total_questions": 3156, "avg_cost_per_task_usd": 0.005154015209125474, "avg_duration_per_task_ms": 318150.1036121673, "avg_output_tokens_per_task": 18230.222116603294, "last_run_timestamp": "2026-08-29T18:13:33.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.8689024817073171, "accuracy_std_dev": 0.04756926554684933, "run_count": 3, "total_questions": 328, "avg_cost_per_task_usd": 0.014946170731707318, "avg_duration_per_task_ms": 479610.80792682926, "avg_output_tokens_per_task": 32324.884146341465, "last_run_timestamp": "2026-09-26T08:55:13.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.868687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13888562626262627, "avg_duration_per_task_ms": 1028019.2020202021, "avg_output_tokens_per_task": 46184.969696969696, "last_run_timestamp": "2026-07-18T16:16:27.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.868687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0028392929292929293, "avg_duration_per_task_ms": 49008.020202020205, "avg_output_tokens_per_task": 5520.065656565656, "last_run_timestamp": "2026-09-23T06:08:55.567Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.8683500000000001, "accuracy_std_dev": 0.014020687172888494, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08152287070707072, "avg_duration_per_task_ms": 190241.53434343435, "avg_output_tokens_per_task": 13202.136363636364, "last_run_timestamp": "2026-09-26T08:55:09.429Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.86835, "accuracy_std_dev": 0.01103428752570821, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07775004545454545, "avg_duration_per_task_ms": 107497.15858585859, "avg_output_tokens_per_task": 12580.208080808081, "last_run_timestamp": "2026-09-26T08:57:01.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.8682967644787642, "accuracy_std_dev": 0.012405441826741467, "run_count": 12, "total_questions": 2331, "avg_cost_per_task_usd": 0.15647866452166453, "avg_duration_per_task_ms": 840962.8275418276, "avg_output_tokens_per_task": 39244.66795366795, "last_run_timestamp": "2026-08-29T18:24:25.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.8681256666666667, "accuracy_std_dev": 0.02130941144710999, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02939427441077441, "avg_duration_per_task_ms": 110502.21548821549, "avg_output_tokens_per_task": 19394.752525252527, "last_run_timestamp": "2026-09-26T09:38:56.771Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.8679501874999999, "accuracy_std_dev": 0.017033291102348476, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.03806877335858586, "avg_duration_per_task_ms": 189574.66571969696, "avg_output_tokens_per_task": 10526.75946969697, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.8677674782608696, "accuracy_std_dev": 0.016613732149306157, "run_count": 15, "total_questions": 2967, "avg_cost_per_task_usd": 0.0769334031007752, "avg_duration_per_task_ms": 376119.02392989554, "avg_output_tokens_per_task": 22811.041118975398, "last_run_timestamp": "2026-08-29T18:39:19.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.8676579999999999, "accuracy_std_dev": 0.007874153076151538, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.006459232323232324, "avg_duration_per_task_ms": 253155.9539842873, "avg_output_tokens_per_task": 18088.584736251403, "last_run_timestamp": "2026-09-26T08:54:07.885Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8670031666666665, "accuracy_std_dev": 0.004661340237051534, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.07032303872053872, "avg_duration_per_task_ms": 224493.27861952863, "avg_output_tokens_per_task": 11452.973905723906, "last_run_timestamp": "2026-09-26T09:37:35.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.8669794312026003, "accuracy_std_dev": 0.015914615486612767, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.06791193391115927, "avg_duration_per_task_ms": 196363.11339833876, "avg_output_tokens_per_task": 18755.92163235825, "last_run_timestamp": "2026-08-29T16:55:12.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.8667228333333332, "accuracy_std_dev": 0.006024375774486715, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0256143728956229, "avg_duration_per_task_ms": 228795.63636363635, "avg_output_tokens_per_task": 21105.421717171717, "last_run_timestamp": "2026-09-19T09:03:56.948Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.8664904348591549, "accuracy_std_dev": 0.01831953501771554, "run_count": 7, "total_questions": 1136, "avg_cost_per_task_usd": 0.0762857491197183, "avg_duration_per_task_ms": 481292.88996478874, "avg_output_tokens_per_task": 19103.125880281692, "last_run_timestamp": "2026-08-15T08:47:41.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Wafer", "accuracy": 0.8661615, "accuracy_std_dev": 0.012626499999999985, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0186262601010101, "avg_duration_per_task_ms": 28912.50505050505, "avg_output_tokens_per_task": 7135.919191919192, "last_run_timestamp": "2026-06-13T16:14:13.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8659932, "accuracy_std_dev": 0.008010237759267841, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07810327272727272, "avg_duration_per_task_ms": 87737.49797979798, "avg_output_tokens_per_task": 12770.258585858586, "last_run_timestamp": "2026-09-26T08:55:49.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.8658808333333333, "accuracy_std_dev": 0.0104532872886422, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.07000543097643097, "avg_duration_per_task_ms": 223270.64393939395, "avg_output_tokens_per_task": 11398.105218855218, "last_run_timestamp": "2026-09-26T08:58:06.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.8655813274418604, "accuracy_std_dev": 0.019123465253362573, "run_count": 31, "total_questions": 5375, "avg_cost_per_task_usd": 0.15140991627906977, "avg_duration_per_task_ms": 236648.42009302325, "avg_output_tokens_per_task": 27703.360372093022, "last_run_timestamp": "2026-09-26T08:54:22.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.8655346706281833, "accuracy_std_dev": 0.02323406358048937, "run_count": 15, "total_questions": 2945, "avg_cost_per_task_usd": 0.09417275280135823, "avg_duration_per_task_ms": 559483.7314091681, "avg_output_tokens_per_task": 31466.001697792868, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.8655206622851365, "accuracy_std_dev": 0.014576764442083846, "run_count": 15, "total_questions": 2967, "avg_cost_per_task_usd": 0.04512668891135827, "avg_duration_per_task_ms": 460619.7182339063, "avg_output_tokens_per_task": 23222.32389619144, "last_run_timestamp": "2026-08-29T19:37:04.387Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.86532, "accuracy_std_dev": 0.008780061799326916, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02537659898989899, "avg_duration_per_task_ms": 218871.01414141414, "avg_output_tokens_per_task": 20892.29595959596, "last_run_timestamp": "2026-09-19T08:55:39.417Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.865319761904762, "accuracy_std_dev": 0.012651559952394205, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.04589295334295333, "avg_duration_per_task_ms": 204017.00841750842, "avg_output_tokens_per_task": 13051.606782106783, "last_run_timestamp": "2026-08-29T18:39:14.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.8651804917197452, "accuracy_std_dev": 0.013354329136557154, "run_count": 4, "total_questions": 785, "avg_cost_per_task_usd": 0.060597563057324846, "avg_duration_per_task_ms": 205391.9694267516, "avg_output_tokens_per_task": 23879.336305732482, "last_run_timestamp": "2026-09-26T09:20:06.107Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.8649570000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.020998656410256413, "avg_duration_per_task_ms": 619238.4615384615, "avg_output_tokens_per_task": 23774.34358974359, "last_run_timestamp": "2026-09-26T01:32:20.726Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.8648248729281768, "accuracy_std_dev": 0.02631682070007696, "run_count": 5, "total_questions": 905, "avg_cost_per_task_usd": 0.010246941436464088, "avg_duration_per_task_ms": 355027.6861878453, "avg_output_tokens_per_task": 21157.914917127073, "last_run_timestamp": "2026-09-26T09:56:32.081Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.8646244400527008, "accuracy_std_dev": 0.0070903829233687965, "run_count": 8, "total_questions": 1518, "avg_cost_per_task_usd": 0.0050142358366271405, "avg_duration_per_task_ms": 140856.23188405798, "avg_output_tokens_per_task": 17854.836627140976, "last_run_timestamp": "2026-08-08T09:25:45.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.8638886961538461, "accuracy_std_dev": 0.011642989955736238, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.10131371474358974, "avg_duration_per_task_ms": 295529.557051282, "avg_output_tokens_per_task": 25493.882692307692, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8638342436974789, "accuracy_std_dev": 0.020982236666300873, "run_count": 11, "total_questions": 2142, "avg_cost_per_task_usd": 0.12425809803921568, "avg_duration_per_task_ms": 840368.0368814192, "avg_output_tokens_per_task": 43681.42903828198, "last_run_timestamp": "2026-08-29T19:01:27.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8636842273684211, "accuracy_std_dev": 0.01763815223202249, "run_count": 5, "total_questions": 950, "avg_cost_per_task_usd": 0.08584259578947369, "avg_duration_per_task_ms": 395986.4378947368, "avg_output_tokens_per_task": 24656.30947368421, "last_run_timestamp": "2026-09-26T08:57:34.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.8636519239465571, "accuracy_std_dev": 0.028707041857903168, "run_count": 11, "total_questions": 1946, "avg_cost_per_task_usd": 0.010248814491264132, "avg_duration_per_task_ms": 324581.0220966084, "avg_output_tokens_per_task": 18963.51336073998, "last_run_timestamp": "2026-09-26T08:55:27.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0938164393939394, "avg_duration_per_task_ms": 667365.3939393939, "avg_output_tokens_per_task": 23446.50505050505, "last_run_timestamp": "2026-09-10T01:44:54.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.8631489999999998, "accuracy_std_dev": 0.03249701661124569, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.03565506804891016, "avg_duration_per_task_ms": 175446.75730994152, "avg_output_tokens_per_task": 12155.395534290272, "last_run_timestamp": "2026-08-15T15:55:02.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.8631252833052278, "accuracy_std_dev": 0.03388611321929843, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.09767507644744237, "avg_duration_per_task_ms": 708453.107925801, "avg_output_tokens_per_task": 37470.564362001125, "last_run_timestamp": "2026-09-21T15:33:37.909Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8631102531752752, "accuracy_std_dev": 0.01428710704815903, "run_count": 6, "total_questions": 1181, "avg_cost_per_task_usd": 0.006545317527519051, "avg_duration_per_task_ms": 246133.70618120237, "avg_output_tokens_per_task": 12855.507197290432, "last_run_timestamp": "2026-09-26T09:18:13.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.8630882576271186, "accuracy_std_dev": 0.015697174214519886, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.16561930621468926, "avg_duration_per_task_ms": 967589.7152542373, "avg_output_tokens_per_task": 49108.66779661017, "last_run_timestamp": "2026-08-29T20:16:17.828Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.8628256666666666, "accuracy_std_dev": 0.019902461007624172, "run_count": 5, "total_questions": 972, "avg_cost_per_task_usd": 0.08342864197530865, "avg_duration_per_task_ms": 454814.01543209876, "avg_output_tokens_per_task": 23652.057613168723, "last_run_timestamp": "2026-09-26T08:53:09.429Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.8627943749999999, "accuracy_std_dev": 0.00929733078546604, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.855192389520202, "avg_duration_per_task_ms": 483915.25, "avg_output_tokens_per_task": 34017.0119949495, "last_run_timestamp": "2026-09-26T08:53:16.951Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.8626742857142858, "accuracy_std_dev": 0.017858490299448887, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.08792987914862915, "avg_duration_per_task_ms": 406224.28066378064, "avg_output_tokens_per_task": 39434.920634920636, "last_run_timestamp": "2026-09-26T08:56:57.800Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.862584125, "accuracy_std_dev": 0.009408632345318578, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.8442858333333334, "avg_duration_per_task_ms": 415897.4154040404, "avg_output_tokens_per_task": 33580.344696969696, "last_run_timestamp": "2026-09-26T08:53:36.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.8622473714739068, "accuracy_std_dev": 0.021847945469408447, "run_count": 29, "total_questions": 5672, "avg_cost_per_task_usd": 0.2541298106488012, "avg_duration_per_task_ms": 337436.6962270804, "avg_output_tokens_per_task": 46060.357369534555, "last_run_timestamp": "2026-09-26T11:00:26.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Morph", "accuracy": 0.8620522502732241, "accuracy_std_dev": 0.014311015984733796, "run_count": 14, "total_questions": 2745, "avg_cost_per_task_usd": 0.031429440072859735, "avg_duration_per_task_ms": 137144.43606557377, "avg_output_tokens_per_task": 8964.896903460838, "last_run_timestamp": "2026-07-04T18:14:56.425Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.8619530000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008309641414141413, "avg_duration_per_task_ms": 730613.3737373737, "avg_output_tokens_per_task": 29358.666666666668, "last_run_timestamp": "2026-07-31T04:09:58.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.8619525, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.223144601010101, "avg_duration_per_task_ms": 273332.86363636365, "avg_output_tokens_per_task": 22220.545454545456, "last_run_timestamp": "2026-09-01T13:23:04.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8618233076923076, "accuracy_std_dev": 0.012536811386930849, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.0038161161616161623, "avg_duration_per_task_ms": 321804.18686868687, "avg_output_tokens_per_task": 18513.827505827507, "last_run_timestamp": "2026-08-22T17:16:03.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.04603966955336814, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004525512626262627, "avg_duration_per_task_ms": 565963.2584175084, "avg_output_tokens_per_task": 18978.082491582492, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8613799792531119, "accuracy_std_dev": 0.01086146499258382, "run_count": 11, "total_questions": 2169, "avg_cost_per_task_usd": 0.0035420834485938215, "avg_duration_per_task_ms": 187394.91378515444, "avg_output_tokens_per_task": 20972.25633932688, "last_run_timestamp": "2026-08-29T19:40:39.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.8612792, "accuracy_std_dev": 0.03136197207702347, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005120404040404041, "avg_duration_per_task_ms": 530889.794949495, "avg_output_tokens_per_task": 17542.636363636364, "last_run_timestamp": "2026-09-05T09:22:54.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.8608830257913248, "accuracy_std_dev": 0.018271073393392782, "run_count": 13, "total_questions": 2559, "avg_cost_per_task_usd": 0.06846565533411489, "avg_duration_per_task_ms": 377785.2809691286, "avg_output_tokens_per_task": 21384.166471277844, "last_run_timestamp": "2026-08-29T19:18:11.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.860810656081081, "accuracy_std_dev": 0.017596670139546283, "run_count": 8, "total_questions": 1480, "avg_cost_per_task_usd": 0.0684703912162162, "avg_duration_per_task_ms": 339840.025, "avg_output_tokens_per_task": 20257.95, "last_run_timestamp": "2026-07-18T08:30:34.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.860524491362764, "accuracy_std_dev": 0.01647408876083893, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.05773977287268074, "avg_duration_per_task_ms": 575351.7728726808, "avg_output_tokens_per_task": 23045.906589891234, "last_run_timestamp": "2026-08-01T17:46:48.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.8603764494556766, "accuracy_std_dev": 0.016565195248170166, "run_count": 10, "total_questions": 1929, "avg_cost_per_task_usd": 0.10031787610160703, "avg_duration_per_task_ms": 637584.9020217729, "avg_output_tokens_per_task": 34282.6588906169, "last_run_timestamp": "2026-09-19T08:55:10.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8603745, "accuracy_std_dev": 0.008427361219563313, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.8821099195075758, "avg_duration_per_task_ms": 416796.10890151514, "avg_output_tokens_per_task": 33415.74179292929, "last_run_timestamp": "2026-09-26T09:19:04.990Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.860269, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.20759804545454547, "avg_duration_per_task_ms": 271144.0252525252, "avg_output_tokens_per_task": 20665.12626262626, "last_run_timestamp": "2026-09-01T13:31:11.391Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8601822310030396, "accuracy_std_dev": 0.013547575585321513, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.007783161094224923, "avg_duration_per_task_ms": 526386.7112462006, "avg_output_tokens_per_task": 27793.246200607904, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.86000405028463, "accuracy_std_dev": 0.023944982277086784, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.04536207748260594, "avg_duration_per_task_ms": 361772.752055661, "avg_output_tokens_per_task": 22068.88678051866, "last_run_timestamp": "2026-08-29T19:11:30.664Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.8599289999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 188, "avg_cost_per_task_usd": 0.023036143617021276, "avg_duration_per_task_ms": 1599986.2287234042, "avg_output_tokens_per_task": 29232.622340425532, "last_run_timestamp": "2026-07-18T08:57:54.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.859638, "accuracy_std_dev": 0.008071103007024502, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.815575909090909, "avg_duration_per_task_ms": 465568.93876262626, "avg_output_tokens_per_task": 32451.6672979798, "last_run_timestamp": "2026-09-26T09:01:02.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8594275, "accuracy_std_dev": 0.013424092492604471, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0069255740740740744, "avg_duration_per_task_ms": 366969.99410774413, "avg_output_tokens_per_task": 23129.368686868685, "last_run_timestamp": "2026-09-26T09:34:13.855Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.85925909122807, "accuracy_std_dev": 0.016791103126745952, "run_count": 13, "total_questions": 2565, "avg_cost_per_task_usd": 0.016009586354775826, "avg_duration_per_task_ms": 298851.8736842105, "avg_output_tokens_per_task": 18611.48382066277, "last_run_timestamp": "2026-08-08T09:24:32.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.8587961250000001, "accuracy_std_dev": 0.026570942984948328, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0075656887626262635, "avg_duration_per_task_ms": 497586.5126262626, "avg_output_tokens_per_task": 24566.76767676768, "last_run_timestamp": "2026-09-26T08:53:09.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.8584595735950045, "accuracy_std_dev": 0.02051306669344079, "run_count": 17, "total_questions": 3363, "avg_cost_per_task_usd": 0.0565897927445733, "avg_duration_per_task_ms": 340557.95628902764, "avg_output_tokens_per_task": 21041.84418673803, "last_run_timestamp": "2026-08-29T19:21:48.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.8580247037037037, "accuracy_std_dev": 0.03117987540449769, "run_count": 15, "total_questions": 2916, "avg_cost_per_task_usd": 0.06713021399176955, "avg_duration_per_task_ms": 850575.8528806585, "avg_output_tokens_per_task": 29951.768861454046, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8580246666666665, "accuracy_std_dev": 0.007694244595078101, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06644984511784513, "avg_duration_per_task_ms": 213157.5361952862, "avg_output_tokens_per_task": 10806.75925925926, "last_run_timestamp": "2026-09-26T09:18:24.826Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.857954375, "accuracy_std_dev": 0.013038726633930733, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.7458450347222222, "avg_duration_per_task_ms": 493231.4678030303, "avg_output_tokens_per_task": 29674.352904040403, "last_run_timestamp": "2026-09-26T09:34:52.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8577439999999998, "accuracy_std_dev": 0.02077245929590428, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.12316677878787877, "avg_duration_per_task_ms": 891518.1712121213, "avg_output_tokens_per_task": 39909.007575757576, "last_run_timestamp": "2026-08-29T18:43:04.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.8577028343558282, "accuracy_std_dev": 0.011904490003355866, "run_count": 5, "total_questions": 978, "avg_cost_per_task_usd": 0.14577651738241312, "avg_duration_per_task_ms": 258434.56646216768, "avg_output_tokens_per_task": 28881.36707566462, "last_run_timestamp": "2026-09-26T09:21:12.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.8575786315028903, "accuracy_std_dev": 0.022850648176995846, "run_count": 21, "total_questions": 4152, "avg_cost_per_task_usd": 0.03863133718689789, "avg_duration_per_task_ms": 314388.73578998074, "avg_output_tokens_per_task": 13377.785886319845, "last_run_timestamp": "2026-08-29T19:19:13.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.8575267943548386, "accuracy_std_dev": 0.0136809305307607, "run_count": 4, "total_questions": 744, "avg_cost_per_task_usd": 0.18561700403225806, "avg_duration_per_task_ms": 513052.8830645161, "avg_output_tokens_per_task": 55552.403225806454, "last_run_timestamp": "2026-07-18T08:26:48.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.857418, "accuracy_std_dev": null, "run_count": 1, "total_questions": 173, "avg_cost_per_task_usd": 0.08778269942196533, "avg_duration_per_task_ms": 317630.5202312139, "avg_output_tokens_per_task": 36575.57225433526, "last_run_timestamp": "2026-08-04T03:12:38.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.857191527607362, "accuracy_std_dev": 0.013500948849617942, "run_count": 5, "total_questions": 978, "avg_cost_per_task_usd": 0.1052768057259714, "avg_duration_per_task_ms": 186949.86605316974, "avg_output_tokens_per_task": 27345.398773006134, "last_run_timestamp": "2026-09-26T08:51:46.229Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.8571427393675027, "accuracy_std_dev": 0.024697655769633415, "run_count": 14, "total_questions": 2751, "avg_cost_per_task_usd": 0.08261968629589239, "avg_duration_per_task_ms": 564800.7659033079, "avg_output_tokens_per_task": 35564.60014540167, "last_run_timestamp": "2026-09-26T08:59:56.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.8569615784966699, "accuracy_std_dev": 0.053359451282668766, "run_count": 16, "total_questions": 3153, "avg_cost_per_task_usd": 0.004682514747859181, "avg_duration_per_task_ms": 225937.11703139867, "avg_output_tokens_per_task": 16787.288931176656, "last_run_timestamp": "2026-08-29T18:30:11.054Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.8569022, "accuracy_std_dev": 0.010212534228094395, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.8410627575757575, "avg_duration_per_task_ms": 450697.24494949495, "avg_output_tokens_per_task": 33467.82424242424, "last_run_timestamp": "2026-09-26T09:29:09.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.856844813993174, "accuracy_std_dev": 0.04300853558120186, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.08689430546075085, "avg_duration_per_task_ms": 500988.61945392494, "avg_output_tokens_per_task": 22774.62059158134, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8567280117827869, "accuracy_std_dev": 0.019208364564689006, "run_count": 10, "total_questions": 1952, "avg_cost_per_task_usd": 0.009705317622950821, "avg_duration_per_task_ms": 263176.2423155738, "avg_output_tokens_per_task": 17464.506147540982, "last_run_timestamp": "2026-08-29T18:07:13.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8565655999999999, "accuracy_std_dev": 0.01503514097838793, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.09365999898989899, "avg_duration_per_task_ms": 671877.2212121212, "avg_output_tokens_per_task": 28686.372727272726, "last_run_timestamp": "2026-09-26T08:52:43.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8564577918781726, "accuracy_std_dev": 0.028181512947320644, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.10150419204737732, "avg_duration_per_task_ms": 330925.20050761424, "avg_output_tokens_per_task": 22836.06937394247, "last_run_timestamp": "2026-09-26T09:19:52.896Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.8563503216174183, "accuracy_std_dev": 0.016757180106276432, "run_count": 17, "total_questions": 3215, "avg_cost_per_task_usd": 0.1523061856920684, "avg_duration_per_task_ms": 200594.1632970451, "avg_output_tokens_per_task": 27265.382270606533, "last_run_timestamp": "2026-09-26T09:17:01.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.8563168223350255, "accuracy_std_dev": 0.012593349140849445, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.01789011252115059, "avg_duration_per_task_ms": 343652.3730964467, "avg_output_tokens_per_task": 35569.74111675127, "last_run_timestamp": "2026-09-26T08:50:11.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.8562288, "accuracy_std_dev": 0.011497073373689477, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1913848090909091, "avg_duration_per_task_ms": 350329.27575757576, "avg_output_tokens_per_task": 55974.5, "last_run_timestamp": "2026-08-29T18:20:49.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.85590693376174, "accuracy_std_dev": 0.1796436210829807, "run_count": 12, "total_questions": 2023, "avg_cost_per_task_usd": 0.14563704152249132, "avg_duration_per_task_ms": 271299.9115175482, "avg_output_tokens_per_task": 11255.300543746911, "last_run_timestamp": "2026-09-26T09:35:00.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8555556, "accuracy_std_dev": 0.0176375124050984, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.011364495959595959, "avg_duration_per_task_ms": 652022.0676767677, "avg_output_tokens_per_task": 33080.43131313131, "last_run_timestamp": "2026-09-26T08:57:03.952Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8553997803738318, "accuracy_std_dev": 0.03094177075749652, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.06754141316199376, "avg_duration_per_task_ms": 582176.5778816199, "avg_output_tokens_per_task": 42569.91900311527, "last_run_timestamp": "2026-09-26T09:04:08.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.8553822048846674, "accuracy_std_dev": 0.018862412436841176, "run_count": 8, "total_questions": 1474, "avg_cost_per_task_usd": 0.05061357462686567, "avg_duration_per_task_ms": 845573.0067842605, "avg_output_tokens_per_task": 25929.323609226594, "last_run_timestamp": "2026-07-18T16:22:39.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.8552118345784417, "accuracy_std_dev": 0.013541336870486602, "run_count": 5, "total_questions": 937, "avg_cost_per_task_usd": 0.009773050160085377, "avg_duration_per_task_ms": 36346.31910352188, "avg_output_tokens_per_task": 8563.84204909285, "last_run_timestamp": "2026-08-01T18:26:19.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8547375714285715, "accuracy_std_dev": 0.04301245273458486, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1766843759018759, "avg_duration_per_task_ms": 45692.21067821068, "avg_output_tokens_per_task": 3336.1709956709956, "last_run_timestamp": "2026-09-26T09:24:09.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8546835012658227, "accuracy_std_dev": 0.02399739526567811, "run_count": 11, "total_questions": 1975, "avg_cost_per_task_usd": 0.12927437518987342, "avg_duration_per_task_ms": 439949.8698734177, "avg_output_tokens_per_task": 41561.01063291139, "last_run_timestamp": "2026-08-29T19:17:01.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8546573333333333, "accuracy_std_dev": 0.015141104458467416, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007946281144781145, "avg_duration_per_task_ms": 637289.4747474748, "avg_output_tokens_per_task": 29143.16835016835, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.8543179200968524, "accuracy_std_dev": 0.02075318154457823, "run_count": 3, "total_questions": 413, "avg_cost_per_task_usd": 0.004252978208232445, "avg_duration_per_task_ms": 444698.63196125906, "avg_output_tokens_per_task": 8315.878934624698, "last_run_timestamp": "2026-09-26T09:19:00.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8541664999999999, "accuracy_std_dev": 0.020077776601257404, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.8391522569444444, "avg_duration_per_task_ms": 453772.9583333333, "avg_output_tokens_per_task": 33398.183712121216, "last_run_timestamp": "2026-09-26T09:34:32.494Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.8541419724300905, "accuracy_std_dev": 0.03420096150574074, "run_count": 13, "total_questions": 2539, "avg_cost_per_task_usd": 0.17093107877116975, "avg_duration_per_task_ms": 297710.6187475384, "avg_output_tokens_per_task": 38736.540764080346, "last_run_timestamp": "2026-09-26T08:54:40.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.8539230009478672, "accuracy_std_dev": 0.01405882861282702, "run_count": 16, "total_questions": 3165, "avg_cost_per_task_usd": 0.01950426066350711, "avg_duration_per_task_ms": 171550.25529225907, "avg_output_tokens_per_task": 12814.184834123223, "last_run_timestamp": "2026-08-01T17:57:47.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.8537537303044951, "accuracy_std_dev": 0.0606172032579932, "run_count": 21, "total_questions": 4138, "avg_cost_per_task_usd": 0.045678762928951185, "avg_duration_per_task_ms": 198289.18946350896, "avg_output_tokens_per_task": 12610.571532141132, "last_run_timestamp": "2026-08-29T18:30:43.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.8536342352941175, "accuracy_std_dev": 0.01673036914795934, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.013844663398692813, "avg_duration_per_task_ms": 136198.07783719548, "avg_output_tokens_per_task": 11446.42899584076, "last_run_timestamp": "2026-08-01T17:57:21.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8535522308340354, "accuracy_std_dev": 0.010038567733441294, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.01690193850042123, "avg_duration_per_task_ms": 472592.7733782645, "avg_output_tokens_per_task": 33567.315080033695, "last_run_timestamp": "2026-09-26T08:54:58.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8529878011634057, "accuracy_std_dev": 0.011201391026239146, "run_count": 29, "total_questions": 5673, "avg_cost_per_task_usd": 0.04108059227921735, "avg_duration_per_task_ms": 443819.24202362064, "avg_output_tokens_per_task": 23556.23320994183, "last_run_timestamp": "2026-08-29T19:16:17.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8526935, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20698726515151514, "avg_duration_per_task_ms": 257611.93181818182, "avg_output_tokens_per_task": 19625.295454545456, "last_run_timestamp": "2026-09-01T13:50:27.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.8524682087067861, "accuracy_std_dev": 0.02973171903175686, "run_count": 12, "total_questions": 2343, "avg_cost_per_task_usd": 0.02875936875800256, "avg_duration_per_task_ms": 363978.1912078532, "avg_output_tokens_per_task": 24367.118224498507, "last_run_timestamp": "2026-08-29T19:03:08.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.8522881577868852, "accuracy_std_dev": 0.009664339658766143, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.09954490881147542, "avg_duration_per_task_ms": 487265.8411885246, "avg_output_tokens_per_task": 30959.861680327867, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.8520922857142859, "accuracy_std_dev": 0.06769645437022032, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.04623776310726311, "avg_duration_per_task_ms": 201066.01322751323, "avg_output_tokens_per_task": 13148.871332371333, "last_run_timestamp": "2026-08-29T20:00:02.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.16585097979797978, "avg_duration_per_task_ms": 389107.60101010103, "avg_output_tokens_per_task": 37536.631313131315, "last_run_timestamp": "2026-08-04T03:12:10.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.851851625, "accuracy_std_dev": 0.018151435989595288, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.7516495170454545, "avg_duration_per_task_ms": 500493.72222222225, "avg_output_tokens_per_task": 29958.29797979798, "last_run_timestamp": "2026-09-26T08:55:10.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.851679139925373, "accuracy_std_dev": 0.03687109443675745, "run_count": 28, "total_questions": 4824, "avg_cost_per_task_usd": 0.011843935737976785, "avg_duration_per_task_ms": 411338.9653814262, "avg_output_tokens_per_task": 16150.16169154229, "last_run_timestamp": "2026-09-26T08:52:37.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.8516331806375443, "accuracy_std_dev": 0.05745577028591585, "run_count": 13, "total_questions": 2541, "avg_cost_per_task_usd": 0.06650903935458481, "avg_duration_per_task_ms": 606257.2499016136, "avg_output_tokens_per_task": 37357.11216056671, "last_run_timestamp": "2026-09-26T09:41:13.780Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8516111428571428, "accuracy_std_dev": 0.032073947329028685, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.831446111111111, "avg_duration_per_task_ms": 427691.9718614719, "avg_output_tokens_per_task": 33200.131313131315, "last_run_timestamp": "2026-09-26T08:56:21.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.8511034444444444, "accuracy_std_dev": 0.03586305000387234, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.17666739506172838, "avg_duration_per_task_ms": 44619.677328843994, "avg_output_tokens_per_task": 3328.8120089786757, "last_run_timestamp": "2026-09-26T08:53:27.502Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8510597058823527, "accuracy_std_dev": 0.02313439249450838, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.1586523764111705, "avg_duration_per_task_ms": 153997.80986333927, "avg_output_tokens_per_task": 15572.684789067142, "last_run_timestamp": "2026-09-26T09:21:26.479Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.8510099000000001, "accuracy_std_dev": 0.015100002764569275, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.7538109368686869, "avg_duration_per_task_ms": 501864.8787878788, "avg_output_tokens_per_task": 30068.36717171717, "last_run_timestamp": "2026-09-26T09:17:18.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.8508604502868068, "accuracy_std_dev": 0.02478573563844512, "run_count": 6, "total_questions": 1046, "avg_cost_per_task_usd": 0.010149271510516252, "avg_duration_per_task_ms": 344062.4971319312, "avg_output_tokens_per_task": 20091.341300191205, "last_run_timestamp": "2026-09-26T09:16:37.664Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8507282936078008, "accuracy_std_dev": 0.03297843398312891, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.05085055074033947, "avg_duration_per_task_ms": 213234.5695196822, "avg_output_tokens_per_task": 21095.91910436981, "last_run_timestamp": "2026-09-26T09:01:34.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8506943125000002, "accuracy_std_dev": 0.0170931360994653, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.8004648680555555, "avg_duration_per_task_ms": 479005.81407828286, "avg_output_tokens_per_task": 30341.69191919192, "last_run_timestamp": "2026-09-26T09:19:41.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8505651288135594, "accuracy_std_dev": 0.014603362238425309, "run_count": 6, "total_questions": 1180, "avg_cost_per_task_usd": 0.008425746610169493, "avg_duration_per_task_ms": 368412.6118644068, "avg_output_tokens_per_task": 24250.365254237287, "last_run_timestamp": "2026-09-26T09:22:08.527Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8505522105263156, "accuracy_std_dev": 0.04204091224118485, "run_count": 13, "total_questions": 2565, "avg_cost_per_task_usd": 0.11401860623781677, "avg_duration_per_task_ms": 498314.546588694, "avg_output_tokens_per_task": 33037.62729044834, "last_run_timestamp": "2026-09-26T08:50:56.054Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.8503937480314961, "accuracy_std_dev": 0.00023499999999998522, "run_count": 2, "total_questions": 381, "avg_cost_per_task_usd": 0.023863905511811022, "avg_duration_per_task_ms": 137111.656167979, "avg_output_tokens_per_task": 36528.412073490814, "last_run_timestamp": "2026-09-26T09:52:38.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.8501715195876289, "accuracy_std_dev": 0.027779548602524146, "run_count": 5, "total_questions": 970, "avg_cost_per_task_usd": 0.13975330515463918, "avg_duration_per_task_ms": 383469.5556701031, "avg_output_tokens_per_task": 31875.5793814433, "last_run_timestamp": "2026-09-26T08:53:10.326Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8501683333333333, "accuracy_std_dev": 0.041790840030375614, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.17284996857463525, "avg_duration_per_task_ms": 43624.98316498317, "avg_output_tokens_per_task": 3253.4483726150393, "last_run_timestamp": "2026-09-26T08:54:57.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.8499715520304567, "accuracy_std_dev": 0.014031811236486494, "run_count": 12, "total_questions": 2364, "avg_cost_per_task_usd": 0.027818640862944163, "avg_duration_per_task_ms": 497777.6065989848, "avg_output_tokens_per_task": 25887.815143824027, "last_run_timestamp": "2026-08-29T19:21:17.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.8498074999999999, "accuracy_std_dev": 0.008564809236054237, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.06874417352092352, "avg_duration_per_task_ms": 217155.5533910534, "avg_output_tokens_per_task": 21389.247474747473, "last_run_timestamp": "2026-09-26T08:53:37.648Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.8498050435706695, "accuracy_std_dev": 0.03351477910773096, "run_count": 5, "total_questions": 941, "avg_cost_per_task_usd": 0.11714972582359193, "avg_duration_per_task_ms": 505045.9829968119, "avg_output_tokens_per_task": 33189.51115834219, "last_run_timestamp": "2026-09-26T08:54:17.442Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.8497266914754098, "accuracy_std_dev": 0.0316272928565439, "run_count": 16, "total_questions": 3050, "avg_cost_per_task_usd": 0.10351839540983607, "avg_duration_per_task_ms": 840685.2724590164, "avg_output_tokens_per_task": 32158.24557377049, "last_run_timestamp": "2026-08-29T18:13:11.491Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8495983231939164, "accuracy_std_dev": 0.010674015891406563, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.09633560709759188, "avg_duration_per_task_ms": 314235.76045627374, "avg_output_tokens_per_task": 30867.523447401774, "last_run_timestamp": "2026-09-26T09:23:32.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.8492335655172413, "accuracy_std_dev": 0.026170202058877375, "run_count": 9, "total_questions": 1740, "avg_cost_per_task_usd": 0.029375192528735626, "avg_duration_per_task_ms": 542113.9201149425, "avg_output_tokens_per_task": 28046.42528735632, "last_run_timestamp": "2026-09-26T08:57:03.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.8490153107221006, "accuracy_std_dev": 0.02104665580299624, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.04150975711159738, "avg_duration_per_task_ms": 669606.0547045952, "avg_output_tokens_per_task": 34749.842450765864, "last_run_timestamp": "2026-08-29T19:20:22.044Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.8489460772833725, "accuracy_std_dev": 0.06927676739771367, "run_count": 13, "total_questions": 2562, "avg_cost_per_task_usd": 0.15593850858704136, "avg_duration_per_task_ms": 473082.737704918, "avg_output_tokens_per_task": 35500.65378610461, "last_run_timestamp": "2026-09-26T08:52:39.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.848817300658376, "accuracy_std_dev": 0.015530404525060451, "run_count": 21, "total_questions": 4101, "avg_cost_per_task_usd": 0.05035830358449157, "avg_duration_per_task_ms": 546041.7607900512, "avg_output_tokens_per_task": 25470.43672275055, "last_run_timestamp": "2026-08-30T00:19:02.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.8486918037542662, "accuracy_std_dev": 0.014893804491204496, "run_count": 6, "total_questions": 1172, "avg_cost_per_task_usd": 0.017789553754266208, "avg_duration_per_task_ms": 299607.09129692835, "avg_output_tokens_per_task": 35353.55375426621, "last_run_timestamp": "2026-09-26T08:54:42.351Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8486816067172265, "accuracy_std_dev": 0.00921244198054446, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.0456744940411701, "avg_duration_per_task_ms": 261759.05597688697, "avg_output_tokens_per_task": 41683.36944745395, "last_run_timestamp": "2026-09-26T16:05:38.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.20520298484848487, "avg_duration_per_task_ms": 278139.5202020202, "avg_output_tokens_per_task": 20422.565656565657, "last_run_timestamp": "2026-09-01T13:34:32.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.8484846666666668, "accuracy_std_dev": 0.022121414381142606, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1076985303030303, "avg_duration_per_task_ms": 498790.32828282827, "avg_output_tokens_per_task": 31614.96127946128, "last_run_timestamp": "2026-08-29T19:12:34.610Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8482611842239186, "accuracy_std_dev": 0.011728653128556577, "run_count": 10, "total_questions": 1965, "avg_cost_per_task_usd": 0.12833305750636134, "avg_duration_per_task_ms": 389662.5424936387, "avg_output_tokens_per_task": 33100.7262086514, "last_run_timestamp": "2026-09-26T08:55:48.508Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.848238375, "accuracy_std_dev": 0.029464977306795932, "run_count": 20, "total_questions": 3936, "avg_cost_per_task_usd": 0.22566900152439032, "avg_duration_per_task_ms": 509900.2350101626, "avg_output_tokens_per_task": 42987.483485772354, "last_run_timestamp": "2026-09-26T08:55:16.971Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.8481524347826088, "accuracy_std_dev": 0.010044369703747568, "run_count": 13, "total_questions": 2553, "avg_cost_per_task_usd": 0.026485490795142973, "avg_duration_per_task_ms": 580834.6169212691, "avg_output_tokens_per_task": 27399.61222091657, "last_run_timestamp": "2026-09-26T09:15:06.166Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.848103465060241, "accuracy_std_dev": 0.01673605276986749, "run_count": 19, "total_questions": 3735, "avg_cost_per_task_usd": 0.017738112449799195, "avg_duration_per_task_ms": 321964.4128514056, "avg_output_tokens_per_task": 14661.384471218205, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.8480290637966805, "accuracy_std_dev": 0.06331280368811666, "run_count": 10, "total_questions": 1928, "avg_cost_per_task_usd": 0.05295942946058091, "avg_duration_per_task_ms": 591223.8205394191, "avg_output_tokens_per_task": 18578.95072614108, "last_run_timestamp": "2026-09-12T09:19:57.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8479360327102803, "accuracy_std_dev": 0.008095688383477137, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.03592315147975078, "avg_duration_per_task_ms": 355113.16238317755, "avg_output_tokens_per_task": 36306.2511682243, "last_run_timestamp": "2026-09-26T16:12:42.605Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.8476976540731995, "accuracy_std_dev": 0.03320482882435078, "run_count": 13, "total_questions": 2541, "avg_cost_per_task_usd": 0.08207621723730815, "avg_duration_per_task_ms": 875595.4120425029, "avg_output_tokens_per_task": 34778.203856749315, "last_run_timestamp": "2026-08-29T18:42:20.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.847643, "accuracy_std_dev": 0.01430999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02130802777777778, "avg_duration_per_task_ms": 487650.29545454547, "avg_output_tokens_per_task": 42398.98737373737, "last_run_timestamp": "2026-09-26T09:15:51.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8475419999999999, "accuracy_std_dev": 0.02235490172109912, "run_count": 25, "total_questions": 4950, "avg_cost_per_task_usd": 0.16931346101010097, "avg_duration_per_task_ms": 157193.26565656567, "avg_output_tokens_per_task": 15870.87696969697, "last_run_timestamp": "2026-09-26T09:21:17.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8473202765017668, "accuracy_std_dev": 0.011288322955111134, "run_count": 6, "total_questions": 1132, "avg_cost_per_task_usd": 0.016132280918727912, "avg_duration_per_task_ms": 230171.71908127208, "avg_output_tokens_per_task": 32052.750883392226, "last_run_timestamp": "2026-09-26T08:49:42.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Together", "accuracy": 0.8472220833333332, "accuracy_std_dev": 0.01989710415051367, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.02657060563973064, "avg_duration_per_task_ms": 47738.242424242424, "avg_output_tokens_per_task": 7342.873316498317, "last_run_timestamp": "2026-06-20T16:15:59.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.8470345574324325, "accuracy_std_dev": 0.011874871521592171, "run_count": 18, "total_questions": 3552, "avg_cost_per_task_usd": 0.011837505067567568, "avg_duration_per_task_ms": 328739.2460585586, "avg_output_tokens_per_task": 13305.292511261261, "last_run_timestamp": "2026-08-15T14:57:08.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.8469764115044247, "accuracy_std_dev": 0.012587309196211092, "run_count": 14, "total_questions": 2712, "avg_cost_per_task_usd": 0.02411464749262537, "avg_duration_per_task_ms": 588302.2522123894, "avg_output_tokens_per_task": 27531.24004424779, "last_run_timestamp": "2026-09-26T09:22:28.862Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.8469509168474333, "accuracy_std_dev": 0.01813136080720607, "run_count": 21, "total_questions": 4149, "avg_cost_per_task_usd": 0.01589632562063148, "avg_duration_per_task_ms": 154447.76644974694, "avg_output_tokens_per_task": 13181.300795372379, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8468012500000001, "accuracy_std_dev": 0.031201721542047957, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.11716699116161616, "avg_duration_per_task_ms": 561187.7866161616, "avg_output_tokens_per_task": 31215.503787878788, "last_run_timestamp": "2026-08-29T18:26:17.965Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8462764299380979, "accuracy_std_dev": 0.015770782183096275, "run_count": 9, "total_questions": 1777, "avg_cost_per_task_usd": 0.08913706978052896, "avg_duration_per_task_ms": 416673.65053460887, "avg_output_tokens_per_task": 22175.449071468767, "last_run_timestamp": "2026-09-26T09:15:30.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.8460472371705964, "accuracy_std_dev": 0.020750268207603947, "run_count": 4, "total_questions": 721, "avg_cost_per_task_usd": 0.1417408987517337, "avg_duration_per_task_ms": 223845.73370319002, "avg_output_tokens_per_task": 32811.28294036061, "last_run_timestamp": "2026-09-26T09:01:35.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DekaLLM", "accuracy": 0.8459595, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0029617095959595958, "avg_duration_per_task_ms": 113309.4494949495, "avg_output_tokens_per_task": 2921.4873737373737, "last_run_timestamp": "2026-05-16T08:55:42.302Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8458658888888889, "accuracy_std_dev": 0.03972899765618852, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.17558623344556676, "avg_duration_per_task_ms": 45322.60044893378, "avg_output_tokens_per_task": 3305.652076318743, "last_run_timestamp": "2026-09-26T08:54:44.118Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.8458657777777778, "accuracy_std_dev": 0.025409284277723633, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.15791502805836138, "avg_duration_per_task_ms": 153529.50112233445, "avg_output_tokens_per_task": 15577.850168350169, "last_run_timestamp": "2026-09-26T09:19:06.330Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "WandB", "accuracy": 0.8458599302325581, "accuracy_std_dev": 0.025858269701844596, "run_count": 15, "total_questions": 2967, "avg_cost_per_task_usd": 0.012189807549713516, "avg_duration_per_task_ms": 182654.23222109876, "avg_output_tokens_per_task": 10138.908324907314, "last_run_timestamp": "2026-07-18T16:18:27.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.8451892474226804, "accuracy_std_dev": 0.006483228158872687, "run_count": 5, "total_questions": 970, "avg_cost_per_task_usd": 0.007323164948453608, "avg_duration_per_task_ms": 502972.8824742268, "avg_output_tokens_per_task": 18045.067010309278, "last_run_timestamp": "2026-08-29T18:17:30.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8451178333333332, "accuracy_std_dev": 0.062380510090403164, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.10489853282828282, "avg_duration_per_task_ms": 490349.9595959596, "avg_output_tokens_per_task": 26821.65404040404, "last_run_timestamp": "2026-09-26T08:57:17.745Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.8451176666666667, "accuracy_std_dev": 0.014740528220597193, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1084811388888889, "avg_duration_per_task_ms": 330556.0925925926, "avg_output_tokens_per_task": 28808.55387205387, "last_run_timestamp": "2026-09-26T09:15:10.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.8448117272727271, "accuracy_std_dev": 0.03913291354785384, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0030450344352617085, "avg_duration_per_task_ms": 533396.9063360882, "avg_output_tokens_per_task": 12009.300275482094, "last_run_timestamp": "2026-07-18T08:42:46.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.8445602826784282, "accuracy_std_dev": 0.023734129927349024, "run_count": 13, "total_questions": 2494, "avg_cost_per_task_usd": 0.14965243223736974, "avg_duration_per_task_ms": 310432.4819566961, "avg_output_tokens_per_task": 38965.461106655974, "last_run_timestamp": "2026-09-26T08:55:26.766Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.8443695555555555, "accuracy_std_dev": 0.00866138720107314, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.14565384680134677, "avg_duration_per_task_ms": 171228.07014590348, "avg_output_tokens_per_task": 32219.744668911335, "last_run_timestamp": "2026-09-26T09:17:52.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.844276, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.059444419191919194, "avg_duration_per_task_ms": 215120.31818181818, "avg_output_tokens_per_task": 34746.81060606061, "last_run_timestamp": "2026-09-26T10:57:32.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8440735980528513, "accuracy_std_dev": 0.022142710827570043, "run_count": 11, "total_questions": 2157, "avg_cost_per_task_usd": 0.1289552053778396, "avg_duration_per_task_ms": 442026.2132591562, "avg_output_tokens_per_task": 33600.40519239685, "last_run_timestamp": "2026-09-26T10:36:35.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8435148007968127, "accuracy_std_dev": 0.009824077042964072, "run_count": 16, "total_questions": 3012, "avg_cost_per_task_usd": 0.0045806557104913674, "avg_duration_per_task_ms": 254608.46049136785, "avg_output_tokens_per_task": 11284.547476759628, "last_run_timestamp": "2026-08-29T18:39:38.845Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Baidu", "accuracy": 0.8434342499999999, "accuracy_std_dev": 0.01988383838541994, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00799521717171717, "avg_duration_per_task_ms": 74649.10037878787, "avg_output_tokens_per_task": 7345.006313131313, "last_run_timestamp": "2026-06-06T16:35:15.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8431678722157092, "accuracy_std_dev": 0.03311619265359203, "run_count": 13, "total_questions": 2559, "avg_cost_per_task_usd": 0.182964666275889, "avg_duration_per_task_ms": 327141.2329034779, "avg_output_tokens_per_task": 41509.383743649865, "last_run_timestamp": "2026-09-26T08:56:02.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.8431370045248868, "accuracy_std_dev": 0.004260933614280586, "run_count": 7, "total_questions": 1326, "avg_cost_per_task_usd": 0.039398003770739064, "avg_duration_per_task_ms": 50754.730015082954, "avg_output_tokens_per_task": 14781.932126696833, "last_run_timestamp": "2026-09-01T17:50:42.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.8430508404840484, "accuracy_std_dev": 0.050661991069463994, "run_count": 14, "total_questions": 2727, "avg_cost_per_task_usd": 0.13254691932526588, "avg_duration_per_task_ms": 173628.933626696, "avg_output_tokens_per_task": 33041.062706270626, "last_run_timestamp": "2026-07-18T08:26:40.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.842761, "accuracy_std_dev": 0.013366603308245514, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.11932980505050504, "avg_duration_per_task_ms": 317023.23434343433, "avg_output_tokens_per_task": 26929.522222222222, "last_run_timestamp": "2026-09-26T08:52:53.172Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8427608000000001, "accuracy_std_dev": 0.011595230077924271, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.08432281212121212, "avg_duration_per_task_ms": 322252.27373737376, "avg_output_tokens_per_task": 20908.12626262626, "last_run_timestamp": "2026-09-26T08:55:48.606Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.8427522467005077, "accuracy_std_dev": 0.018612571921389303, "run_count": 15, "total_questions": 2955, "avg_cost_per_task_usd": 0.030284539763113367, "avg_duration_per_task_ms": 443994.4812182741, "avg_output_tokens_per_task": 24288.26192893401, "last_run_timestamp": "2026-08-29T19:06:57.766Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.8425925, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06301305808080808, "avg_duration_per_task_ms": 90351.56313131313, "avg_output_tokens_per_task": 13876.954545454546, "last_run_timestamp": "2026-09-01T17:51:50.923Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.8425925, "accuracy_std_dev": 0.008788056170166437, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.14231570202020202, "avg_duration_per_task_ms": 163901.45454545456, "avg_output_tokens_per_task": 32169.752525252527, "last_run_timestamp": "2026-09-26T09:14:46.960Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.8425191583372257, "accuracy_std_dev": 0.019868589783045544, "run_count": 11, "total_questions": 2141, "avg_cost_per_task_usd": 0.19488549135917793, "avg_duration_per_task_ms": 339625.2237272303, "avg_output_tokens_per_task": 44550.5791686128, "last_run_timestamp": "2026-09-26T09:23:36.793Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.842400931034483, "accuracy_std_dev": 0.020596427323446166, "run_count": 20, "total_questions": 3915, "avg_cost_per_task_usd": 0.055737818135376774, "avg_duration_per_task_ms": 573338.7954022988, "avg_output_tokens_per_task": 25592.180587484036, "last_run_timestamp": "2026-08-30T00:28:09.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.8423992077798862, "accuracy_std_dev": 0.022061600294686224, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.1114357827324478, "avg_duration_per_task_ms": 504012.30803289055, "avg_output_tokens_per_task": 34670.05819101835, "last_run_timestamp": "2026-08-29T20:19:40.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8419116213235294, "accuracy_std_dev": 0.013588325247266868, "run_count": 14, "total_questions": 2720, "avg_cost_per_task_usd": 0.1566976856617647, "avg_duration_per_task_ms": 305686.83897058823, "avg_output_tokens_per_task": 35495.729044117645, "last_run_timestamp": "2026-08-29T19:00:02.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.8417577430497052, "accuracy_std_dev": 0.014641061103015125, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.13158069755686608, "avg_duration_per_task_ms": 164554.15669755687, "avg_output_tokens_per_task": 29682.097725358046, "last_run_timestamp": "2026-09-26T08:53:13.623Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.8415826000000001, "accuracy_std_dev": 0.01677086684223569, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.013069406060606059, "avg_duration_per_task_ms": 474851.301010101, "avg_output_tokens_per_task": 38641.41818181818, "last_run_timestamp": "2026-09-26T08:53:10.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.8414969770992364, "accuracy_std_dev": 0.03970141746131365, "run_count": 16, "total_questions": 3144, "avg_cost_per_task_usd": 0.0710119901399491, "avg_duration_per_task_ms": 630148.5734732824, "avg_output_tokens_per_task": 28421.371501272264, "last_run_timestamp": "2026-08-29T18:40:35.044Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.8414075126536513, "accuracy_std_dev": 0.022108573391159597, "run_count": 21, "total_questions": 4149, "avg_cost_per_task_usd": 0.0148124528802121, "avg_duration_per_task_ms": 317232.769824054, "avg_output_tokens_per_task": 14106.06435285611, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8410292857142857, "accuracy_std_dev": 0.03771800444544936, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.6859342640692642, "avg_duration_per_task_ms": 470200.2972582973, "avg_output_tokens_per_task": 27535.334054834057, "last_run_timestamp": "2026-09-26T08:58:47.240Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DekaLLM", "accuracy": 0.8410117804878049, "accuracy_std_dev": 0.0033504999999999785, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.07247923035230353, "avg_duration_per_task_ms": 402624.47967479675, "avg_output_tokens_per_task": 25758.56639566396, "last_run_timestamp": "2026-07-04T09:58:40.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8408546584615385, "accuracy_std_dev": 0.004112945970955614, "run_count": 5, "total_questions": 975, "avg_cost_per_task_usd": 0.09125773538461539, "avg_duration_per_task_ms": 289113.2892307692, "avg_output_tokens_per_task": 22386.701538461537, "last_run_timestamp": "2026-09-19T08:59:39.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.840404, "accuracy_std_dev": 0.008869764145680555, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.07076540101010101, "avg_duration_per_task_ms": 297317.25757575757, "avg_output_tokens_per_task": 20594.954545454544, "last_run_timestamp": "2026-09-26T08:53:05.194Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.8402299275862066, "accuracy_std_dev": 0.03863386507191665, "run_count": 9, "total_questions": 1740, "avg_cost_per_task_usd": 0.16238871264367818, "avg_duration_per_task_ms": 375858.65287356323, "avg_output_tokens_per_task": 36786.99310344827, "last_run_timestamp": "2026-08-15T08:57:45.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.8401273660017347, "accuracy_std_dev": 0.00647490120044743, "run_count": 6, "total_questions": 1153, "avg_cost_per_task_usd": 0.022382765828274063, "avg_duration_per_task_ms": 71469.82740676496, "avg_output_tokens_per_task": 18471.962705984388, "last_run_timestamp": "2026-09-26T10:30:17.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "SiliconFlow", "accuracy": 0.8398268571428571, "accuracy_std_dev": 0.01764351869205047, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03009014502164502, "avg_duration_per_task_ms": 211303.82539682538, "avg_output_tokens_per_task": 13627.257575757576, "last_run_timestamp": "2026-06-06T17:25:31.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.8395753932584269, "accuracy_std_dev": 0.02635514356497756, "run_count": 11, "total_questions": 2136, "avg_cost_per_task_usd": 0.054479627808988774, "avg_duration_per_task_ms": 414791.2270599251, "avg_output_tokens_per_task": 21872.586142322096, "last_run_timestamp": "2026-08-01T17:53:10.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.8392968267419962, "accuracy_std_dev": 0.0037396706405778625, "run_count": 4, "total_questions": 531, "avg_cost_per_task_usd": 0.12392348210922788, "avg_duration_per_task_ms": 100276.54613935969, "avg_output_tokens_per_task": 27948.913370998118, "last_run_timestamp": "2026-09-26T09:05:37.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.839052991845056, "accuracy_std_dev": 0.019945151183070924, "run_count": 15, "total_questions": 2943, "avg_cost_per_task_usd": 0.053893335372069334, "avg_duration_per_task_ms": 628913.658851512, "avg_output_tokens_per_task": 30616.803601766904, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "ModelRun", "accuracy": 0.8390480151770657, "accuracy_std_dev": 0.04955882783964433, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.04416806408094435, "avg_duration_per_task_ms": 290822.7987633502, "avg_output_tokens_per_task": 23210.459808881395, "last_run_timestamp": "2026-07-11T08:27:26.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.8387204, "accuracy_std_dev": 0.02117180060930105, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.020740932323232323, "avg_duration_per_task_ms": 437294.63434343436, "avg_output_tokens_per_task": 37324.31616161616, "last_run_timestamp": "2026-09-26T09:10:17.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.8386324061538462, "accuracy_std_dev": 0.032593817304240734, "run_count": 15, "total_questions": 2925, "avg_cost_per_task_usd": 0.05106006564102564, "avg_duration_per_task_ms": 250208.00615384616, "avg_output_tokens_per_task": 22368.053333333333, "last_run_timestamp": "2026-09-26T10:00:05.608Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.838579522693997, "accuracy_std_dev": 0.017426082982771814, "run_count": 7, "total_questions": 1366, "avg_cost_per_task_usd": 0.0794436691068814, "avg_duration_per_task_ms": 602046.7005856516, "avg_output_tokens_per_task": 29605.27232796486, "last_run_timestamp": "2026-09-26T09:19:10.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.8383037142857144, "accuracy_std_dev": 0.03628775495967758, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.05614424146224147, "avg_duration_per_task_ms": 269081.58946608944, "avg_output_tokens_per_task": 12677.165464165464, "last_run_timestamp": "2026-08-29T18:50:46.982Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.838284705882353, "accuracy_std_dev": 0.05170697825901454, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.18276167468805704, "avg_duration_per_task_ms": 40754.59982174688, "avg_output_tokens_per_task": 3290.764111705288, "last_run_timestamp": "2026-09-26T08:53:17.875Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.838277288056206, "accuracy_std_dev": 0.03168625568690031, "run_count": 13, "total_questions": 2562, "avg_cost_per_task_usd": 0.10126575644028102, "avg_duration_per_task_ms": 880551.862607338, "avg_output_tokens_per_task": 31631.52537080406, "last_run_timestamp": "2026-09-26T10:24:21.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.8382395753791257, "accuracy_std_dev": 0.015894466289989074, "run_count": 17, "total_questions": 3363, "avg_cost_per_task_usd": 0.016586219744275946, "avg_duration_per_task_ms": 262022.0906928338, "avg_output_tokens_per_task": 15243.496580434135, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8381625397574123, "accuracy_std_dev": 0.007901272276048144, "run_count": 9, "total_questions": 1484, "avg_cost_per_task_usd": 0.017624041105121297, "avg_duration_per_task_ms": 189126.76886792452, "avg_output_tokens_per_task": 35057.77156334232, "last_run_timestamp": "2026-09-26T09:29:08.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.8380671171597633, "accuracy_std_dev": 0.02773812208207325, "run_count": 5, "total_questions": 845, "avg_cost_per_task_usd": 0.07336154792899409, "avg_duration_per_task_ms": 417593.0852071006, "avg_output_tokens_per_task": 24105.62721893491, "last_run_timestamp": "2026-09-26T08:54:14.754Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.8377145373291274, "accuracy_std_dev": 0.08364258942907019, "run_count": 5, "total_questions": 951, "avg_cost_per_task_usd": 0.09799871188222924, "avg_duration_per_task_ms": 610509.8033648791, "avg_output_tokens_per_task": 25161.19242902208, "last_run_timestamp": "2026-09-26T08:53:09.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8376065934065935, "accuracy_std_dev": 0.00571412028808169, "run_count": 3, "total_questions": 546, "avg_cost_per_task_usd": 0.036677783882783886, "avg_duration_per_task_ms": 47951.65201465201, "avg_output_tokens_per_task": 14583.362637362638, "last_run_timestamp": "2026-09-01T17:50:52.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8375589327830189, "accuracy_std_dev": 0.020056898198209345, "run_count": 9, "total_questions": 1696, "avg_cost_per_task_usd": 0.054286748820754724, "avg_duration_per_task_ms": 282159.9534198113, "avg_output_tokens_per_task": 22660.685141509435, "last_run_timestamp": "2026-09-26T09:50:39.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.8373233845303869, "accuracy_std_dev": 0.00973152140401156, "run_count": 14, "total_questions": 2715, "avg_cost_per_task_usd": 0.06885601362799264, "avg_duration_per_task_ms": 716196.5233885819, "avg_output_tokens_per_task": 41564.71123388582, "last_run_timestamp": "2026-09-26T16:00:20.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8371357823193916, "accuracy_std_dev": 0.009460368146759528, "run_count": 6, "total_questions": 1052, "avg_cost_per_task_usd": 0.08486342680608365, "avg_duration_per_task_ms": 599922.150190114, "avg_output_tokens_per_task": 21002.96958174905, "last_run_timestamp": "2026-09-19T08:48:08.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.8368604243386243, "accuracy_std_dev": 0.053504834431884106, "run_count": 6, "total_questions": 945, "avg_cost_per_task_usd": 0.05630135555555556, "avg_duration_per_task_ms": 268944.617989418, "avg_output_tokens_per_task": 18648.48253968254, "last_run_timestamp": "2026-08-15T15:24:07.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.8368515680218921, "accuracy_std_dev": 0.03421836730617917, "run_count": 13, "total_questions": 2558, "avg_cost_per_task_usd": 0.12014493940578577, "avg_duration_per_task_ms": 750575.0445660673, "avg_output_tokens_per_task": 36877.55043002345, "last_run_timestamp": "2026-08-29T18:43:32.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.8367000000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06203523232323233, "avg_duration_per_task_ms": 92697.9696969697, "avg_output_tokens_per_task": 13658.787878787878, "last_run_timestamp": "2026-09-01T17:49:43.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.8366999999999999, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04958680555555556, "avg_duration_per_task_ms": 536787.6388888889, "avg_output_tokens_per_task": 49485.26515151515, "last_run_timestamp": "2026-09-26T16:26:15.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.8366909151079136, "accuracy_std_dev": 0.01869795941406443, "run_count": 4, "total_questions": 695, "avg_cost_per_task_usd": 0.020156113669064748, "avg_duration_per_task_ms": 242800.49064748202, "avg_output_tokens_per_task": 40096.50647482014, "last_run_timestamp": "2026-09-26T09:48:09.064Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.8366301736263737, "accuracy_std_dev": 0.012674357340709624, "run_count": 3, "total_questions": 455, "avg_cost_per_task_usd": 0.04995332967032967, "avg_duration_per_task_ms": 107459.2945054945, "avg_output_tokens_per_task": 15090.074725274726, "last_run_timestamp": "2026-09-26T08:53:14.372Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.8365793063670413, "accuracy_std_dev": 0.023337994672784947, "run_count": 7, "total_questions": 1335, "avg_cost_per_task_usd": 0.01897062172284644, "avg_duration_per_task_ms": 793025.5857677902, "avg_output_tokens_per_task": 41115.47715355805, "last_run_timestamp": "2026-09-26T12:41:27.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8362919349112425, "accuracy_std_dev": 0.01693241709290274, "run_count": 18, "total_questions": 3549, "avg_cost_per_task_usd": 0.04999668385460693, "avg_duration_per_task_ms": 467378.05945336714, "avg_output_tokens_per_task": 26951.275852352774, "last_run_timestamp": "2026-08-22T17:01:35.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.83627925, "accuracy_std_dev": 0.037120416716242575, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015161977272727275, "avg_duration_per_task_ms": 425327.9482323232, "avg_output_tokens_per_task": 27500.13005050505, "last_run_timestamp": "2026-08-29T19:58:27.585Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.836234166760247, "accuracy_std_dev": 0.02982784551020275, "run_count": 27, "total_questions": 5343, "avg_cost_per_task_usd": 0.1657105358412877, "avg_duration_per_task_ms": 150009.68556990454, "avg_output_tokens_per_task": 15316.003930376193, "last_run_timestamp": "2026-09-26T08:59:40.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.836139, "accuracy_std_dev": 0.05678065343641852, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.005011018518518518, "avg_duration_per_task_ms": 214818.47643097644, "avg_output_tokens_per_task": 17529.722222222223, "last_run_timestamp": "2026-09-26T11:06:41.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.8358585000000001, "accuracy_std_dev": 0.02753853053904659, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1297035244107744, "avg_duration_per_task_ms": 386511.07744107745, "avg_output_tokens_per_task": 35313.07323232323, "last_run_timestamp": "2026-09-26T08:54:10.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.8355473623188407, "accuracy_std_dev": 0.012606082718439233, "run_count": 11, "total_questions": 1656, "avg_cost_per_task_usd": 0.03472509842995169, "avg_duration_per_task_ms": 236602.51086956522, "avg_output_tokens_per_task": 27679.197463768116, "last_run_timestamp": "2026-09-26T16:00:25.681Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.8354201847715735, "accuracy_std_dev": 0.021899629119538376, "run_count": 15, "total_questions": 2955, "avg_cost_per_task_usd": 0.1346993539763113, "avg_duration_per_task_ms": 635579.6727580372, "avg_output_tokens_per_task": 34556.54077834179, "last_run_timestamp": "2026-08-29T18:39:39.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Parasail", "accuracy": 0.8353531999999999, "accuracy_std_dev": 0.00989691835674118, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03142896161616161, "avg_duration_per_task_ms": 541746.1080808081, "avg_output_tokens_per_task": 15052.834343434344, "last_run_timestamp": "2026-06-06T16:02:18.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto-beta", "model_slug": "openrouter/auto-beta", "model_name": "Auto Router (Beta)", "provider_name": "auto-routing", "accuracy": 0.835017, "accuracy_std_dev": 0.062145249638783075, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025753547138047138, "avg_duration_per_task_ms": 248297.88383838383, "avg_output_tokens_per_task": 30752.395622895623, "last_run_timestamp": "2026-09-15T14:04:40.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.835017, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.036737896464646466, "avg_duration_per_task_ms": 49536.31060606061, "avg_output_tokens_per_task": 14606.348484848484, "last_run_timestamp": "2026-09-01T17:49:40.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.8349114289772727, "accuracy_std_dev": 0.012684612209645646, "run_count": 11, "total_questions": 2112, "avg_cost_per_task_usd": 0.09840011221590908, "avg_duration_per_task_ms": 269461.8380681818, "avg_output_tokens_per_task": 24744.351799242424, "last_run_timestamp": "2026-07-18T08:29:37.313Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.8348945269320842, "accuracy_std_dev": 0.07887946023533471, "run_count": 13, "total_questions": 2562, "avg_cost_per_task_usd": 0.07108125370804061, "avg_duration_per_task_ms": 478944.9949258392, "avg_output_tokens_per_task": 31685.776346604216, "last_run_timestamp": "2026-09-26T09:14:27.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "AtlasCloud", "accuracy": 0.8348081017699116, "accuracy_std_dev": 0.015122908520600433, "run_count": 7, "total_questions": 1356, "avg_cost_per_task_usd": 0.034566790560471974, "avg_duration_per_task_ms": 347006.30088495574, "avg_output_tokens_per_task": 13778.697640117995, "last_run_timestamp": "2026-06-30T22:55:56.214Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8348017092511013, "accuracy_std_dev": 0.013128455115293883, "run_count": 7, "total_questions": 1362, "avg_cost_per_task_usd": 0.03501912334801763, "avg_duration_per_task_ms": 566392.0389133627, "avg_output_tokens_per_task": 32664.345814977973, "last_run_timestamp": "2026-08-29T18:47:45.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SambaNova", "accuracy": 0.8345957500000001, "accuracy_std_dev": 0.014451436422982317, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0074238952020202024, "avg_duration_per_task_ms": 38293.18308080808, "avg_output_tokens_per_task": 7280.891414141414, "last_run_timestamp": "2026-05-16T16:15:42.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.8344516577181208, "accuracy_std_dev": 0.048501499999999975, "run_count": 2, "total_questions": 298, "avg_cost_per_task_usd": 0.0903846644295302, "avg_duration_per_task_ms": 304392.4194630872, "avg_output_tokens_per_task": 22866.939597315435, "last_run_timestamp": "2026-07-18T08:27:05.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8341984152249134, "accuracy_std_dev": 0.029002188381032377, "run_count": 3, "total_questions": 578, "avg_cost_per_task_usd": 0.005492470588235294, "avg_duration_per_task_ms": 956260.6712802768, "avg_output_tokens_per_task": 15404.778546712803, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.834175, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.028012333333333334, "avg_duration_per_task_ms": 352518.654040404, "avg_output_tokens_per_task": 14875.09595959596, "last_run_timestamp": "2026-09-26T09:19:39.685Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.8339535412719892, "accuracy_std_dev": 0.08422964195615594, "run_count": 15, "total_questions": 2956, "avg_cost_per_task_usd": 0.09844996684709066, "avg_duration_per_task_ms": 493001.6951962111, "avg_output_tokens_per_task": 30313.425236806495, "last_run_timestamp": "2026-08-29T18:23:09.250Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8339505777777776, "accuracy_std_dev": 0.01071049161258874, "run_count": 6, "total_questions": 1080, "avg_cost_per_task_usd": 0.025328656481481478, "avg_duration_per_task_ms": 650579.8324074075, "avg_output_tokens_per_task": 40296.40185185185, "last_run_timestamp": "2026-09-26T09:00:45.770Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.8338056683673469, "accuracy_std_dev": 0.013414099005395167, "run_count": 18, "total_questions": 3528, "avg_cost_per_task_usd": 0.004950865079365079, "avg_duration_per_task_ms": 179413.4075963719, "avg_output_tokens_per_task": 16477.43820861678, "last_run_timestamp": "2026-08-29T18:49:32.713Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8337061610738257, "accuracy_std_dev": 0.0693376467642218, "run_count": 5, "total_questions": 894, "avg_cost_per_task_usd": 0.009961087248322148, "avg_duration_per_task_ms": 513095.80536912754, "avg_output_tokens_per_task": 19701.401565995526, "last_run_timestamp": "2026-09-26T08:54:24.468Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.8332027746478873, "accuracy_std_dev": 0.026066320644252704, "run_count": 13, "total_questions": 2556, "avg_cost_per_task_usd": 0.06387245735524257, "avg_duration_per_task_ms": 894067.6103286385, "avg_output_tokens_per_task": 29485.598591549297, "last_run_timestamp": "2026-08-29T19:12:56.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.833159912591051, "accuracy_std_dev": 0.01602299589215448, "run_count": 5, "total_questions": 961, "avg_cost_per_task_usd": 0.14537621123829345, "avg_duration_per_task_ms": 277832.8116545265, "avg_output_tokens_per_task": 32850.81373569199, "last_run_timestamp": "2026-09-26T09:19:06.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.8331351764705882, "accuracy_std_dev": 0.011719105633588751, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.25533701039809864, "avg_duration_per_task_ms": 277560.3027332145, "avg_output_tokens_per_task": 55800.19964349376, "last_run_timestamp": "2026-09-26T09:16:57.240Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.8326118571428571, "accuracy_std_dev": 0.010792989132217443, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.039859138528138535, "avg_duration_per_task_ms": 221347.44805194804, "avg_output_tokens_per_task": 14514.689033189034, "last_run_timestamp": "2026-09-26T09:19:17.245Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.8324914999999999, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.023044303030303033, "avg_duration_per_task_ms": 640134.2070707071, "avg_output_tokens_per_task": 54011.681818181816, "last_run_timestamp": "2026-09-26T08:54:28.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.8323511666666668, "accuracy_std_dev": 0.013383536583139077, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1359044436026936, "avg_duration_per_task_ms": 151136.74915824915, "avg_output_tokens_per_task": 30646.316498316497, "last_run_timestamp": "2026-09-26T08:54:57.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.8319833976435935, "accuracy_std_dev": 0.0051619355292529005, "run_count": 7, "total_questions": 1358, "avg_cost_per_task_usd": 0.03598950073637703, "avg_duration_per_task_ms": 200380.98085419735, "avg_output_tokens_per_task": 13861.594256259204, "last_run_timestamp": "2026-09-26T09:19:46.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.8318619561643835, "accuracy_std_dev": 0.01930397520188926, "run_count": 17, "total_questions": 3285, "avg_cost_per_task_usd": 0.003945210350076104, "avg_duration_per_task_ms": 335822.299238965, "avg_output_tokens_per_task": 10098.203652968037, "last_run_timestamp": "2026-08-29T19:56:51.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.8316497999999998, "accuracy_std_dev": 0.03697562642282075, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.01486108888888889, "avg_duration_per_task_ms": 312116.7202020202, "avg_output_tokens_per_task": 12505.067171717172, "last_run_timestamp": "2026-07-18T09:15:42.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Together", "accuracy": 0.8316496666666666, "accuracy_std_dev": 0.020755514356966008, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.05561522937710438, "avg_duration_per_task_ms": 213796.11658249158, "avg_output_tokens_per_task": 17341.332912457914, "last_run_timestamp": "2026-06-20T17:09:19.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.8315097680525164, "accuracy_std_dev": 0.043329048047164274, "run_count": 14, "total_questions": 2742, "avg_cost_per_task_usd": 0.16222809883296865, "avg_duration_per_task_ms": 711251.6560904449, "avg_output_tokens_per_task": 37157.696571845365, "last_run_timestamp": "2026-08-29T18:21:26.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.8313691904761906, "accuracy_std_dev": 0.0137097002420047, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.06259450793650795, "avg_duration_per_task_ms": 231605.2477152477, "avg_output_tokens_per_task": 28760.637566137568, "last_run_timestamp": "2026-08-30T00:09:46.578Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8313591965192169, "accuracy_std_dev": 0.02385695725514725, "run_count": 21, "total_questions": 4137, "avg_cost_per_task_usd": 0.015896513173797436, "avg_duration_per_task_ms": 285786.35170413344, "avg_output_tokens_per_task": 13215.3529127387, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.8312629347826086, "accuracy_std_dev": 0.044536486139877485, "run_count": 4, "total_questions": 644, "avg_cost_per_task_usd": 0.005694647515527951, "avg_duration_per_task_ms": 270388.4906832298, "avg_output_tokens_per_task": 13521.194099378881, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.8309764, "accuracy_std_dev": 0.005061551248382267, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.24307877272727274, "avg_duration_per_task_ms": 341071.7717171717, "avg_output_tokens_per_task": 40681.778787878786, "last_run_timestamp": "2026-09-26T08:54:33.880Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.8309565294117646, "accuracy_std_dev": 0.021130248647287467, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.03042767617349971, "avg_duration_per_task_ms": 128034.97237076648, "avg_output_tokens_per_task": 16834.001782531195, "last_run_timestamp": "2026-08-29T18:13:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.8309340194750211, "accuracy_std_dev": 0.0227049955198312, "run_count": 6, "total_questions": 1181, "avg_cost_per_task_usd": 0.0975369652836579, "avg_duration_per_task_ms": 214278.25232853513, "avg_output_tokens_per_task": 28093.950889077052, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.8308604629080119, "accuracy_std_dev": 0.014193547884550253, "run_count": 7, "total_questions": 1348, "avg_cost_per_task_usd": 0.037891371661721074, "avg_duration_per_task_ms": 428497.83827893174, "avg_output_tokens_per_task": 44654.94213649852, "last_run_timestamp": "2026-09-26T08:56:01.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.8308259809976247, "accuracy_std_dev": 0.05464180945926158, "run_count": 13, "total_questions": 2526, "avg_cost_per_task_usd": 0.04734239192399049, "avg_duration_per_task_ms": 550280.8665874901, "avg_output_tokens_per_task": 22668.492082343626, "last_run_timestamp": "2026-09-26T08:01:18.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.830808, "accuracy_std_dev": 0.008551060402078817, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.09822325252525253, "avg_duration_per_task_ms": 297130.1747474747, "avg_output_tokens_per_task": 24088.215151515153, "last_run_timestamp": "2026-09-19T08:48:09.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Together", "accuracy": 0.8308079999999999, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04318659848484849, "avg_duration_per_task_ms": 391952.8409090909, "avg_output_tokens_per_task": 15921.767676767677, "last_run_timestamp": "2026-05-16T10:24:57.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "WandB", "accuracy": 0.8302964372990355, "accuracy_std_dev": 0.05435128994016608, "run_count": 5, "total_questions": 933, "avg_cost_per_task_usd": 0.004838944265809216, "avg_duration_per_task_ms": 341319.115755627, "avg_output_tokens_per_task": 13605.640943193997, "last_run_timestamp": "2026-07-18T08:04:02.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8302680551724138, "accuracy_std_dev": 0.010265872005588431, "run_count": 22, "total_questions": 4350, "avg_cost_per_task_usd": 0.004694437471264369, "avg_duration_per_task_ms": 306186.3236781609, "avg_output_tokens_per_task": 12163.184827586207, "last_run_timestamp": "2026-08-29T16:40:58.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.8301106049723759, "accuracy_std_dev": 0.03034424675983057, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.12463168692449354, "avg_duration_per_task_ms": 509594.1510128913, "avg_output_tokens_per_task": 35443.973756906074, "last_run_timestamp": "2026-09-26T08:54:25.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.8300985157116451, "accuracy_std_dev": 0.047147674805281085, "run_count": 33, "total_questions": 6492, "avg_cost_per_task_usd": 0.0361898150030807, "avg_duration_per_task_ms": 283344.9191312384, "avg_output_tokens_per_task": 18752.18977202711, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.8298668701517706, "accuracy_std_dev": 0.04231782629659911, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.16520087015177065, "avg_duration_per_task_ms": 161607.96964586846, "avg_output_tokens_per_task": 16300.88813940416, "last_run_timestamp": "2026-09-26T08:54:23.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8296198191945158, "accuracy_std_dev": 0.010647141525623993, "run_count": 6, "total_questions": 1167, "avg_cost_per_task_usd": 0.1271454147386461, "avg_duration_per_task_ms": 247054.93658954583, "avg_output_tokens_per_task": 29738.498714652957, "last_run_timestamp": "2026-09-26T08:53:15.658Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8293609150259068, "accuracy_std_dev": 0.029508056688451866, "run_count": 15, "total_questions": 2895, "avg_cost_per_task_usd": 0.13191214404145077, "avg_duration_per_task_ms": 620405.4127806562, "avg_output_tokens_per_task": 37376.21899827288, "last_run_timestamp": "2026-08-29T18:14:11.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.8293536141356256, "accuracy_std_dev": 0.03173512046656424, "run_count": 16, "total_questions": 3141, "avg_cost_per_task_usd": 0.14317697262018464, "avg_duration_per_task_ms": 539342.0986946832, "avg_output_tokens_per_task": 35578.65934415791, "last_run_timestamp": "2026-08-29T18:09:15.344Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.8293137510353228, "accuracy_std_dev": 0.13959916580818144, "run_count": 21, "total_questions": 4105, "avg_cost_per_task_usd": 0.04072066552984167, "avg_duration_per_task_ms": 187753.5790499391, "avg_output_tokens_per_task": 11251.794153471375, "last_run_timestamp": "2026-08-29T19:36:49.687Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8291486389610391, "accuracy_std_dev": 0.04390329909775197, "run_count": 6, "total_questions": 1155, "avg_cost_per_task_usd": 0.028787966233766234, "avg_duration_per_task_ms": 286215.91515151516, "avg_output_tokens_per_task": 8890.160173160173, "last_run_timestamp": "2026-09-26T09:45:13.813Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.8289048695652176, "accuracy_std_dev": 0.025607506073269167, "run_count": 21, "total_questions": 4140, "avg_cost_per_task_usd": 0.04958357536231883, "avg_duration_per_task_ms": 535038.0852657005, "avg_output_tokens_per_task": 25244.089371980677, "last_run_timestamp": "2026-08-30T00:30:19.495Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.8287637142857143, "accuracy_std_dev": 0.08258088982275724, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007342964646464646, "avg_duration_per_task_ms": 721777.8398268399, "avg_output_tokens_per_task": 30149.034632034633, "last_run_timestamp": "2026-09-26T11:30:22.248Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.8287408778346745, "accuracy_std_dev": 0.05300294804063007, "run_count": 21, "total_questions": 4101, "avg_cost_per_task_usd": 0.10757520897342113, "avg_duration_per_task_ms": 651824.1992197025, "avg_output_tokens_per_task": 34289.174347720065, "last_run_timestamp": "2026-08-29T18:11:51.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.8286193999999999, "accuracy_std_dev": 0.012838925852266618, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.01266686868686869, "avg_duration_per_task_ms": 861046.6656565657, "avg_output_tokens_per_task": 37375.524242424246, "last_run_timestamp": "2026-09-26T08:09:25.366Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.8285513974854932, "accuracy_std_dev": 0.024874965961581598, "run_count": 16, "total_questions": 3102, "avg_cost_per_task_usd": 0.05205874306898775, "avg_duration_per_task_ms": 908984.3558994198, "avg_output_tokens_per_task": 29744.421018697616, "last_run_timestamp": "2026-08-22T17:50:04.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.828497570984456, "accuracy_std_dev": 0.015230029187102692, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.11257541554404144, "avg_duration_per_task_ms": 438124.0186528497, "avg_output_tokens_per_task": 31261.672538860104, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8284124801026957, "accuracy_std_dev": 0.01497190145646912, "run_count": 12, "total_questions": 2337, "avg_cost_per_task_usd": 0.00971603380402225, "avg_duration_per_task_ms": 567277.3761232349, "avg_output_tokens_per_task": 34466.249037227215, "last_run_timestamp": "2026-09-26T08:54:38.079Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.8279570000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 186, "avg_cost_per_task_usd": 0.024709930107526885, "avg_duration_per_task_ms": 419969.8602150538, "avg_output_tokens_per_task": 20901.392473118278, "last_run_timestamp": "2026-08-01T18:03:43.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.8273554038986354, "accuracy_std_dev": 0.018032038830334687, "run_count": 13, "total_questions": 2565, "avg_cost_per_task_usd": 0.020172549707602336, "avg_duration_per_task_ms": 335505.9828460039, "avg_output_tokens_per_task": 20130.481091617934, "last_run_timestamp": "2026-08-29T16:15:12.678Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8269733333333333, "accuracy_std_dev": 0.015136600014534322, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.06164716273849608, "avg_duration_per_task_ms": 496054.37934904604, "avg_output_tokens_per_task": 48584.97418630752, "last_run_timestamp": "2026-09-26T16:19:15.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Together", "accuracy": 0.826964236582224, "accuracy_std_dev": 0.043702263548038585, "run_count": 12, "total_questions": 2329, "avg_cost_per_task_usd": 0.017868838127951914, "avg_duration_per_task_ms": 211852.5800772864, "avg_output_tokens_per_task": 15131.554744525547, "last_run_timestamp": "2026-07-18T08:29:12.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.8269358, "accuracy_std_dev": 0.006767606146932625, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.04578402121212121, "avg_duration_per_task_ms": 60291.652525252524, "avg_output_tokens_per_task": 11285.754545454545, "last_run_timestamp": "2026-09-26T09:19:03.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.8267805538461538, "accuracy_std_dev": 0.020665297105829934, "run_count": 12, "total_questions": 2340, "avg_cost_per_task_usd": 0.08551232649572652, "avg_duration_per_task_ms": 577054.2572649573, "avg_output_tokens_per_task": 31689.464957264958, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.8266137120954002, "accuracy_std_dev": 0.009032395856780211, "run_count": 9, "total_questions": 1761, "avg_cost_per_task_usd": 0.005160955139125498, "avg_duration_per_task_ms": 482576.3299261783, "avg_output_tokens_per_task": 12303.996592844975, "last_run_timestamp": "2026-08-01T18:25:37.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.826599, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04722597138047138, "avg_duration_per_task_ms": 258616.0824915825, "avg_output_tokens_per_task": 15105.597643097643, "last_run_timestamp": "2026-09-26T09:35:33.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8264543613323679, "accuracy_std_dev": 0.018188360323715465, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.20616153874004342, "avg_duration_per_task_ms": 315117.1817523534, "avg_output_tokens_per_task": 46875.9225199131, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8263186666666665, "accuracy_std_dev": 0.011640048205322105, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.04355452946127946, "avg_duration_per_task_ms": 227785.6026936027, "avg_output_tokens_per_task": 14401.111111111111, "last_run_timestamp": "2026-09-26T09:34:17.391Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.8262184258271078, "accuracy_std_dev": 0.01942525748194862, "run_count": 5, "total_questions": 937, "avg_cost_per_task_usd": 0.02446647598719317, "avg_duration_per_task_ms": 331857.0437566702, "avg_output_tokens_per_task": 48723.26467449306, "last_run_timestamp": "2026-09-26T08:54:40.393Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8261094014447884, "accuracy_std_dev": 0.026551028658038853, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.2076026150670795, "avg_duration_per_task_ms": 857894.439628483, "avg_output_tokens_per_task": 47560.28689370485, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8260196253164556, "accuracy_std_dev": 0.053781736077150054, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.024082872573839665, "avg_duration_per_task_ms": 553189.7814345992, "avg_output_tokens_per_task": 25590.895780590716, "last_run_timestamp": "2026-09-26T09:17:20.976Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.8259301444575472, "accuracy_std_dev": 0.05179941930880393, "run_count": 26, "total_questions": 5088, "avg_cost_per_task_usd": 0.03116529402515724, "avg_duration_per_task_ms": 245803.80483490566, "avg_output_tokens_per_task": 15695.919025157233, "last_run_timestamp": "2026-08-01T17:44:01.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8258873707224335, "accuracy_std_dev": 0.009676826643527784, "run_count": 6, "total_questions": 1052, "avg_cost_per_task_usd": 0.025350776615969583, "avg_duration_per_task_ms": 120122.08840304182, "avg_output_tokens_per_task": 20151.38783269962, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.8255112924187727, "accuracy_std_dev": 0.033062154820560075, "run_count": 21, "total_questions": 4155, "avg_cost_per_task_usd": 0.01651286329723225, "avg_duration_per_task_ms": 227671.96774969916, "avg_output_tokens_per_task": 13721.429602888087, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.8253399469387755, "accuracy_std_dev": 0.06202361079266511, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.048117852040816325, "avg_duration_per_task_ms": 1077180.0714285714, "avg_output_tokens_per_task": 26155.976530612246, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.8252867162300974, "accuracy_std_dev": 0.1181226298338754, "run_count": 21, "total_questions": 3894, "avg_cost_per_task_usd": 0.03139470313302517, "avg_duration_per_task_ms": 234154.43913713406, "avg_output_tokens_per_task": 9983.509758602979, "last_run_timestamp": "2026-08-29T18:43:58.355Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.8249211384507535, "accuracy_std_dev": 0.0352196214425993, "run_count": 15, "total_questions": 2853, "avg_cost_per_task_usd": 0.08481681317910972, "avg_duration_per_task_ms": 388842.21381002455, "avg_output_tokens_per_task": 22797.929547844375, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.8249157142857142, "accuracy_std_dev": 0.019837994431424387, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1225871738816739, "avg_duration_per_task_ms": 339042.2626262626, "avg_output_tokens_per_task": 50612.39971139971, "last_run_timestamp": "2026-09-26T08:56:23.149Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.8246351666666666, "accuracy_std_dev": 0.013862659393933843, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.034730324915824916, "avg_duration_per_task_ms": 240485.28282828283, "avg_output_tokens_per_task": 13780.222222222223, "last_run_timestamp": "2026-09-26T09:15:01.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.8246351666666666, "accuracy_std_dev": 0.0064044884369392945, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.13257150673400672, "avg_duration_per_task_ms": 144313.25757575757, "avg_output_tokens_per_task": 32437.94191919192, "last_run_timestamp": "2026-09-26T08:54:42.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.824495, "accuracy_std_dev": 0.00300556117888158, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.04055735353535353, "avg_duration_per_task_ms": 210547.6452020202, "avg_output_tokens_per_task": 15056.00883838384, "last_run_timestamp": "2026-09-26T09:19:00.848Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.82449475, "accuracy_std_dev": 0.010869783171135461, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06350338131313131, "avg_duration_per_task_ms": 123563.64393939394, "avg_output_tokens_per_task": 13357.755050505051, "last_run_timestamp": "2026-09-01T17:56:39.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.8244158226324237, "accuracy_std_dev": 0.01458128650491802, "run_count": 19, "total_questions": 3738, "avg_cost_per_task_usd": 0.0051121361690743715, "avg_duration_per_task_ms": 427263.127340824, "avg_output_tokens_per_task": 11900.191011235955, "last_run_timestamp": "2026-08-29T18:40:24.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.8237132857142857, "accuracy_std_dev": 0.024208164205574968, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.037303974025974025, "avg_duration_per_task_ms": 279740.5851370851, "avg_output_tokens_per_task": 14323.054112554113, "last_run_timestamp": "2026-09-26T09:20:53.278Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.8234829675785206, "accuracy_std_dev": 0.014126665871322934, "run_count": 15, "total_questions": 2961, "avg_cost_per_task_usd": 0.013195947315096249, "avg_duration_per_task_ms": 394720.7257683215, "avg_output_tokens_per_task": 13095.860182370821, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8233658066759387, "accuracy_std_dev": 0.08442661821970915, "run_count": 11, "total_questions": 2157, "avg_cost_per_task_usd": 0.09823874269819194, "avg_duration_per_task_ms": 882801.9360222531, "avg_output_tokens_per_task": 41455.85813630042, "last_run_timestamp": "2026-09-26T09:26:01.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.8232579938524589, "accuracy_std_dev": 0.05951885425846167, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.08256164446721312, "avg_duration_per_task_ms": 628781.7581967213, "avg_output_tokens_per_task": 23867.87807377049, "last_run_timestamp": "2026-09-26T09:35:44.547Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.8230056978021977, "accuracy_std_dev": 0.043112993157555894, "run_count": 17, "total_questions": 3276, "avg_cost_per_task_usd": 0.09837996916971918, "avg_duration_per_task_ms": 887456.5876068376, "avg_output_tokens_per_task": 30118.1742979243, "last_run_timestamp": "2026-08-29T18:08:11.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.8228419167330677, "accuracy_std_dev": 0.11242608196694337, "run_count": 13, "total_questions": 2510, "avg_cost_per_task_usd": 0.007463425099601593, "avg_duration_per_task_ms": 612656.6079681275, "avg_output_tokens_per_task": 26820.300398406376, "last_run_timestamp": "2026-09-26T09:19:42.439Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.8227271999999999, "accuracy_std_dev": 0.039294904353618175, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0058356378787878796, "avg_duration_per_task_ms": 243021.81515151515, "avg_output_tokens_per_task": 13918.35101010101, "last_run_timestamp": "2026-08-01T20:03:34.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8226246495327102, "accuracy_std_dev": 0.027849632903150384, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.03971392640186916, "avg_duration_per_task_ms": 354960.40303738316, "avg_output_tokens_per_task": 41628.609034267916, "last_run_timestamp": "2026-09-26T16:24:56.918Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.8225956482593038, "accuracy_std_dev": 0.159966755691973, "run_count": 13, "total_questions": 2499, "avg_cost_per_task_usd": 0.08340546258503403, "avg_duration_per_task_ms": 451358.10764305724, "avg_output_tokens_per_task": 22627.635454181673, "last_run_timestamp": "2026-09-26T08:54:41.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.822297, "accuracy_std_dev": 0.014221076658725011, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.31472648260381597, "avg_duration_per_task_ms": 289101.1150392817, "avg_output_tokens_per_task": 20792.76374859708, "last_run_timestamp": "2026-09-26T08:55:59.561Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8221650670103093, "accuracy_std_dev": 0.01835186937268852, "run_count": 6, "total_questions": 1164, "avg_cost_per_task_usd": 0.13827411597938147, "avg_duration_per_task_ms": 261027.06529209623, "avg_output_tokens_per_task": 31195.84621993127, "last_run_timestamp": "2026-09-26T09:19:47.728Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.8221166532066507, "accuracy_std_dev": 0.015661128267637187, "run_count": 13, "total_questions": 2526, "avg_cost_per_task_usd": 0.03727458986539984, "avg_duration_per_task_ms": 329571.73871733964, "avg_output_tokens_per_task": 20674.40657165479, "last_run_timestamp": "2026-08-29T16:44:34.185Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.8221100000000001, "accuracy_std_dev": 0.019325780306224248, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.011398650392817058, "avg_duration_per_task_ms": 623049.0549943884, "avg_output_tokens_per_task": 33683.20538720539, "last_run_timestamp": "2026-09-26T09:34:52.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.8219105872274142, "accuracy_std_dev": 0.01870885637365363, "run_count": 10, "total_questions": 1926, "avg_cost_per_task_usd": 0.054632894080996884, "avg_duration_per_task_ms": 274976.66355140187, "avg_output_tokens_per_task": 22789.01453790239, "last_run_timestamp": "2026-08-01T17:52:55.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.8217910573969281, "accuracy_std_dev": 0.01861988887693676, "run_count": 19, "total_questions": 3711, "avg_cost_per_task_usd": 0.10542012718943684, "avg_duration_per_task_ms": 593247.7383454595, "avg_output_tokens_per_task": 33480.1341956346, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.821549, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.001158979797979798, "avg_duration_per_task_ms": 153563.0101010101, "avg_output_tokens_per_task": 18533.79292929293, "last_run_timestamp": "2026-09-23T18:11:58.526Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8215488, "accuracy_std_dev": 0.011711955283384605, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.039391856565656566, "avg_duration_per_task_ms": 244469.52525252526, "avg_output_tokens_per_task": 15370.382828282829, "last_run_timestamp": "2026-09-26T09:19:26.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "auto-routing", "accuracy": 0.8215312538860104, "accuracy_std_dev": 0.022946010202695863, "run_count": 3, "total_questions": 579, "avg_cost_per_task_usd": 0.08296633678756477, "avg_duration_per_task_ms": 239744.6183074266, "avg_output_tokens_per_task": 23823.246977547496, "last_run_timestamp": "2026-06-30T21:35:33.459Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.8214834459833794, "accuracy_std_dev": 0.011465056212986574, "run_count": 11, "total_questions": 2166, "avg_cost_per_task_usd": 0.005549661126500463, "avg_duration_per_task_ms": 487617.81809787627, "avg_output_tokens_per_task": 16572.74192059095, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "auto-routing", "accuracy": 0.82133825, "accuracy_std_dev": 0.012610410379424602, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.003432248737373737, "avg_duration_per_task_ms": 86387.43244949495, "avg_output_tokens_per_task": 11387.400883838383, "last_run_timestamp": "2026-06-13T08:37:07.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "Xiaomi", "accuracy": 0.8210677142857142, "accuracy_std_dev": 0.013459362028123072, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0033843903318903318, "avg_duration_per_task_ms": 85055.07142857143, "avg_output_tokens_per_task": 11231.03823953824, "last_run_timestamp": "2026-06-13T08:05:57.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.8207635888324873, "accuracy_std_dev": 0.020270393054191847, "run_count": 7, "total_questions": 1379, "avg_cost_per_task_usd": 0.03637786294416244, "avg_duration_per_task_ms": 250330.74111675128, "avg_output_tokens_per_task": 14481.398839738942, "last_run_timestamp": "2026-09-26T08:56:59.861Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Ambient", "accuracy": 0.8207071666666668, "accuracy_std_dev": 0.015360537180457651, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008930407407407408, "avg_duration_per_task_ms": 167713.53535353535, "avg_output_tokens_per_task": 8891.933501683501, "last_run_timestamp": "2026-06-13T16:35:11.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.8206795856832972, "accuracy_std_dev": 0.013489140010307805, "run_count": 21, "total_questions": 4149, "avg_cost_per_task_usd": 0.011921808146541337, "avg_duration_per_task_ms": 368546.352374066, "avg_output_tokens_per_task": 11799.792239093758, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.8206135555555556, "accuracy_std_dev": 0.02409886708315059, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03784499887766555, "avg_duration_per_task_ms": 323218.42199775536, "avg_output_tokens_per_task": 39347.21212121212, "last_run_timestamp": "2026-09-26T16:29:49.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.8201054571428571, "accuracy_std_dev": 0.026122159766927572, "run_count": 3, "total_questions": 315, "avg_cost_per_task_usd": 0.055772555555555554, "avg_duration_per_task_ms": 137698.42222222223, "avg_output_tokens_per_task": 23066.479365079365, "last_run_timestamp": "2026-09-12T16:00:25.695Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.8197154974692696, "accuracy_std_dev": 0.01804488276618475, "run_count": 21, "total_questions": 4149, "avg_cost_per_task_usd": 0.004644195468787659, "avg_duration_per_task_ms": 274482.4740901422, "avg_output_tokens_per_task": 11932.80212099301, "last_run_timestamp": "2026-08-29T19:19:45.463Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.8194444939024391, "accuracy_std_dev": 0.0983013278247835, "run_count": 15, "total_questions": 2952, "avg_cost_per_task_usd": 0.04524065040650407, "avg_duration_per_task_ms": 186780.3099593496, "avg_output_tokens_per_task": 21656.31910569106, "last_run_timestamp": "2026-09-26T09:18:49.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.8193043333333333, "accuracy_std_dev": 0.0031744380430068155, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.14333356734006736, "avg_duration_per_task_ms": 321739.1531986532, "avg_output_tokens_per_task": 28661.96127946128, "last_run_timestamp": "2026-09-26T09:35:22.612Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.8188576490610326, "accuracy_std_dev": 0.0725401109287967, "run_count": 13, "total_questions": 1704, "avg_cost_per_task_usd": 0.020530700117370893, "avg_duration_per_task_ms": 66110.05575117371, "avg_output_tokens_per_task": 20662.859154929578, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.818474192504259, "accuracy_std_dev": 0.05364142082052881, "run_count": 18, "total_questions": 3522, "avg_cost_per_task_usd": 0.11224730863145937, "avg_duration_per_task_ms": 720587.8109028961, "avg_output_tokens_per_task": 35871.42277115276, "last_run_timestamp": "2026-08-29T18:49:03.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.8183890113981765, "accuracy_std_dev": 0.04470928783762385, "run_count": 20, "total_questions": 3948, "avg_cost_per_task_usd": 0.07126378191489362, "avg_duration_per_task_ms": 604114.2841945288, "avg_output_tokens_per_task": 32383.42781155015, "last_run_timestamp": "2026-08-29T18:40:30.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8181816666666667, "accuracy_std_dev": 0.013255807590972678, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.17135338047138046, "avg_duration_per_task_ms": 45814.95117845118, "avg_output_tokens_per_task": 3234.355218855219, "last_run_timestamp": "2026-08-15T09:15:35.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Together", "accuracy": 0.8181816249999999, "accuracy_std_dev": 0.014060086761267687, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.07144717613636363, "avg_duration_per_task_ms": 198887.53787878787, "avg_output_tokens_per_task": 16429.516414141413, "last_run_timestamp": "2026-06-20T09:16:50.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.818035862576065, "accuracy_std_dev": 0.01278056208505712, "run_count": 10, "total_questions": 1972, "avg_cost_per_task_usd": 0.0909173985801217, "avg_duration_per_task_ms": 165913.36257606492, "avg_output_tokens_per_task": 22308.299188640973, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.817971375, "accuracy_std_dev": 0.021218711111525495, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0422043106060606, "avg_duration_per_task_ms": 258582.00315656565, "avg_output_tokens_per_task": 14114.647727272728, "last_run_timestamp": "2026-09-26T09:36:02.338Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8177008571428571, "accuracy_std_dev": 0.044711167868022074, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.16188713708513708, "avg_duration_per_task_ms": 160453.3152958153, "avg_output_tokens_per_task": 16045.246753246753, "last_run_timestamp": "2026-09-26T08:55:42.287Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8173940034094783, "accuracy_std_dev": 0.05734385363328043, "run_count": 31, "total_questions": 5866, "avg_cost_per_task_usd": 0.005422403000340948, "avg_duration_per_task_ms": 373758.65342652577, "avg_output_tokens_per_task": 12923.683088987385, "last_run_timestamp": "2026-08-29T20:12:46.798Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.8172269616858236, "accuracy_std_dev": 0.059023830851696865, "run_count": 12, "total_questions": 2349, "avg_cost_per_task_usd": 0.1289344618986803, "avg_duration_per_task_ms": 699975.4461472967, "avg_output_tokens_per_task": 43383.50404427416, "last_run_timestamp": "2026-08-29T19:22:50.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.8171395238095238, "accuracy_std_dev": 0.02033844061452071, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.004526424482924483, "avg_duration_per_task_ms": 331078.8936988937, "avg_output_tokens_per_task": 11555.212602212603, "last_run_timestamp": "2026-08-29T19:12:36.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.8168373000310271, "accuracy_std_dev": 0.052236706041768964, "run_count": 19, "total_questions": 3223, "avg_cost_per_task_usd": 0.0374196242631089, "avg_duration_per_task_ms": 545799.8814768849, "avg_output_tokens_per_task": 31060.19795221843, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8165784761904761, "accuracy_std_dev": 0.01245241634466425, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.3341618246753246, "avg_duration_per_task_ms": 300449.1738816739, "avg_output_tokens_per_task": 21181.114718614717, "last_run_timestamp": "2026-09-26T08:51:53.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Ambient", "accuracy": 0.8164983333333333, "accuracy_std_dev": 0.010378007269649053, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002570441077441077, "avg_duration_per_task_ms": 557971.7020202021, "avg_output_tokens_per_task": 4954.074074074074, "last_run_timestamp": "2026-06-13T08:04:34.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.8161555499021526, "accuracy_std_dev": 0.033245258304521796, "run_count": 8, "total_questions": 1533, "avg_cost_per_task_usd": 0.01330266210045662, "avg_duration_per_task_ms": 93162.18590998043, "avg_output_tokens_per_task": 17543.737769080235, "last_run_timestamp": "2026-08-29T18:36:28.592Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.81596961928934, "accuracy_std_dev": 0.04643267204753484, "run_count": 21, "total_questions": 4137, "avg_cost_per_task_usd": 0.07539528402223834, "avg_duration_per_task_ms": 501375.46241237613, "avg_output_tokens_per_task": 29453.204737732656, "last_run_timestamp": "2026-08-29T19:11:37.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.8159271462140993, "accuracy_std_dev": 0.011945687451440362, "run_count": 4, "total_questions": 766, "avg_cost_per_task_usd": 0.10126304177545693, "avg_duration_per_task_ms": 221714.96997389034, "avg_output_tokens_per_task": 28424.819843342037, "last_run_timestamp": "2026-08-04T03:11:59.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.815885572815534, "accuracy_std_dev": 0.04096815993777976, "run_count": 12, "total_questions": 2369, "avg_cost_per_task_usd": 0.057723436893203896, "avg_duration_per_task_ms": 941535.38159561, "avg_output_tokens_per_task": 29544.111439425917, "last_run_timestamp": "2026-09-26T10:24:45.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.8157559809402797, "accuracy_std_dev": 0.02360825327613207, "run_count": 8, "total_questions": 1574, "avg_cost_per_task_usd": 0.021597616899618807, "avg_duration_per_task_ms": 295953.15184243966, "avg_output_tokens_per_task": 17957.35959339263, "last_run_timestamp": "2026-09-26T08:54:33.107Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.8154462499999999, "accuracy_std_dev": 0.00789342203594232, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.024953141414141414, "avg_duration_per_task_ms": 185879.93371212122, "avg_output_tokens_per_task": 16543.260101010103, "last_run_timestamp": "2026-09-26T08:53:11.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8153759444444445, "accuracy_std_dev": 0.010779567108769665, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.36484757070707075, "avg_duration_per_task_ms": 359825.24242424243, "avg_output_tokens_per_task": 22980.845679012345, "last_run_timestamp": "2026-09-26T08:54:41.498Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.8151450853503185, "accuracy_std_dev": 0.14825549375062988, "run_count": 12, "total_questions": 2355, "avg_cost_per_task_usd": 0.06763006751592358, "avg_duration_per_task_ms": 395557.31549893843, "avg_output_tokens_per_task": 18616.247983014862, "last_run_timestamp": "2026-09-26T08:55:14.254Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.8149456927157002, "accuracy_std_dev": 0.18265885331301351, "run_count": 16, "total_questions": 2828, "avg_cost_per_task_usd": 0.17484592397454032, "avg_duration_per_task_ms": 315586.27050919377, "avg_output_tokens_per_task": 31965.719589816126, "last_run_timestamp": "2026-09-26T08:56:04.781Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8143338571428571, "accuracy_std_dev": 0.014248444761532711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026212479076479073, "avg_duration_per_task_ms": 329590.34992784995, "avg_output_tokens_per_task": 17511.52380952381, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.814332751988431, "accuracy_std_dev": 0.01885441572018766, "run_count": 21, "total_questions": 4149, "avg_cost_per_task_usd": 0.06707576355748375, "avg_duration_per_task_ms": 462943.6165341046, "avg_output_tokens_per_task": 33393.012774162446, "last_run_timestamp": "2026-08-29T19:17:49.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.8142204705882352, "accuracy_std_dev": 0.03855851822744482, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.019141188354129528, "avg_duration_per_task_ms": 245806.30392156861, "avg_output_tokens_per_task": 17822.011883541294, "last_run_timestamp": "2026-08-29T19:18:38.408Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.8137254984162897, "accuracy_std_dev": 0.08405639786637797, "run_count": 24, "total_questions": 4420, "avg_cost_per_task_usd": 0.010863166515837101, "avg_duration_per_task_ms": 478036.79049773753, "avg_output_tokens_per_task": 11973.086425339367, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.8136272825651302, "accuracy_std_dev": 0.008773473238499465, "run_count": 3, "total_questions": 499, "avg_cost_per_task_usd": 0.0068851422845691386, "avg_duration_per_task_ms": 1090136, "avg_output_tokens_per_task": 17404.21643286573, "last_run_timestamp": "2026-08-15T10:05:32.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.813131, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.037260742424242425, "avg_duration_per_task_ms": 35156.99494949495, "avg_output_tokens_per_task": 2566.8030303030305, "last_run_timestamp": "2026-08-01T08:32:23.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.8127893802083332, "accuracy_std_dev": 0.02323940439151189, "run_count": 12, "total_questions": 2304, "avg_cost_per_task_usd": 0.014347588107638889, "avg_duration_per_task_ms": 360100.9900173611, "avg_output_tokens_per_task": 13752.019097222223, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.8123588325266213, "accuracy_std_dev": 0.08477891888464963, "run_count": 12, "total_questions": 2066, "avg_cost_per_task_usd": 0.08345451210067764, "avg_duration_per_task_ms": 526015.0217812198, "avg_output_tokens_per_task": 26015.62391093901, "last_run_timestamp": "2026-09-26T09:56:28.417Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8119979337349396, "accuracy_std_dev": 0.0940565298454681, "run_count": 14, "total_questions": 2656, "avg_cost_per_task_usd": 0.06128782756024097, "avg_duration_per_task_ms": 908022.2884036144, "avg_output_tokens_per_task": 28257.003765060243, "last_run_timestamp": "2026-08-29T19:12:33.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Fireworks", "accuracy": 0.8117484228596261, "accuracy_std_dev": 0.09706275314151007, "run_count": 12, "total_questions": 2301, "avg_cost_per_task_usd": 0.019746013472403304, "avg_duration_per_task_ms": 100475.48109517602, "avg_output_tokens_per_task": 16488.117340286833, "last_run_timestamp": "2026-07-18T08:31:24.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AkashML", "accuracy": 0.8109296153846153, "accuracy_std_dev": 0.14891345125817648, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.008492456099456099, "avg_duration_per_task_ms": 161496.31857031857, "avg_output_tokens_per_task": 7371.756021756022, "last_run_timestamp": "2026-07-01T03:19:17.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.8107742, "accuracy_std_dev": 0.00793923498581567, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.032759715151515156, "avg_duration_per_task_ms": 674236.5636363636, "avg_output_tokens_per_task": 15717.559595959596, "last_run_timestamp": "2026-09-26T09:29:37.459Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "WandB", "accuracy": 0.8106060000000002, "accuracy_std_dev": 0.016940078320362043, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.000941689393939394, "avg_duration_per_task_ms": 123178.65404040404, "avg_output_tokens_per_task": 9148.02398989899, "last_run_timestamp": "2026-06-20T16:49:31.612Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.8099312975495916, "accuracy_std_dev": 0.096225883038589, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.08600893154414625, "avg_duration_per_task_ms": 491078.3780630105, "avg_output_tokens_per_task": 31731.871645274212, "last_run_timestamp": "2026-09-26T09:22:12.564Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.8092030833333335, "accuracy_std_dev": 0.01675517045301904, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.33934875, "avg_duration_per_task_ms": 354134.20664983167, "avg_output_tokens_per_task": 22457.28787878788, "last_run_timestamp": "2026-09-26T08:56:43.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.8089819438325991, "accuracy_std_dev": 0.06057756732336092, "run_count": 14, "total_questions": 2724, "avg_cost_per_task_usd": 0.15691310939794417, "avg_duration_per_task_ms": 677800.3755506608, "avg_output_tokens_per_task": 35807.34140969163, "last_run_timestamp": "2026-08-29T18:07:06.751Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.808541063084112, "accuracy_std_dev": 0.015819760309078213, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.011975492601246106, "avg_duration_per_task_ms": 326659.34890965733, "avg_output_tokens_per_task": 12473.692757009347, "last_run_timestamp": "2026-08-30T00:01:09.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.808081, "accuracy_std_dev": 0.12407749055328285, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.10305112289562289, "avg_duration_per_task_ms": 1499684.5976430976, "avg_output_tokens_per_task": 25661.05218855219, "last_run_timestamp": "2026-09-26T09:01:35.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.8078395250344509, "accuracy_std_dev": 0.08454229752765441, "run_count": 13, "total_questions": 2177, "avg_cost_per_task_usd": 0.06321491272393201, "avg_duration_per_task_ms": 708932.5346807534, "avg_output_tokens_per_task": 20144.69820854387, "last_run_timestamp": "2026-08-01T18:16:39.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.8074874931077692, "accuracy_std_dev": 0.0754265571969271, "run_count": 17, "total_questions": 3192, "avg_cost_per_task_usd": 0.1512801278195489, "avg_duration_per_task_ms": 334510.98558897246, "avg_output_tokens_per_task": 31449.75845864662, "last_run_timestamp": "2026-09-26T08:52:36.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.8069288421052634, "accuracy_std_dev": 0.01849876504630044, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.020217205209994687, "avg_duration_per_task_ms": 157255.7658160553, "avg_output_tokens_per_task": 16045.147793726741, "last_run_timestamp": "2026-08-29T19:24:09.844Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.8067576807017544, "accuracy_std_dev": 0.04457842166074652, "run_count": 13, "total_questions": 2565, "avg_cost_per_task_usd": 0.07463826237816762, "avg_duration_per_task_ms": 680798.849122807, "avg_output_tokens_per_task": 28652.57037037037, "last_run_timestamp": "2026-09-26T10:35:48.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "AtlasCloud", "accuracy": 0.8064057959183675, "accuracy_std_dev": 0.010831854991438695, "run_count": 6, "total_questions": 1176, "avg_cost_per_task_usd": 0.026680215136054425, "avg_duration_per_task_ms": 253897.95323129251, "avg_output_tokens_per_task": 11557.03656462585, "last_run_timestamp": "2026-07-01T04:42:33.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.8063971999999998, "accuracy_std_dev": 0.01659741351415936, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00614619090909091, "avg_duration_per_task_ms": 535780.8212121212, "avg_output_tokens_per_task": 18359.183838383837, "last_run_timestamp": "2026-08-29T19:12:20.528Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.8059367902364605, "accuracy_std_dev": 0.1514017392727079, "run_count": 14, "total_questions": 2622, "avg_cost_per_task_usd": 0.0040933535469107555, "avg_duration_per_task_ms": 403372.7581998474, "avg_output_tokens_per_task": 10095.213577421815, "last_run_timestamp": "2026-08-29T18:15:38.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.8058781554959786, "accuracy_std_dev": 0.01700166220410292, "run_count": 17, "total_questions": 3357, "avg_cost_per_task_usd": 0.014998186773905272, "avg_duration_per_task_ms": 223628.16324098897, "avg_output_tokens_per_task": 13349.141197497765, "last_run_timestamp": "2026-08-29T18:59:02.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.805556, "accuracy_std_dev": null, "run_count": 1, "total_questions": 192, "avg_cost_per_task_usd": 0.16585933333333333, "avg_duration_per_task_ms": 688743.8177083334, "avg_output_tokens_per_task": 42426.447916666664, "last_run_timestamp": "2026-07-31T00:21:35.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.805555546875, "accuracy_std_dev": 0.01288499999999998, "run_count": 2, "total_questions": 384, "avg_cost_per_task_usd": 0.0362872109375, "avg_duration_per_task_ms": 729621.7864583334, "avg_output_tokens_per_task": 36271.864583333336, "last_run_timestamp": "2026-09-12T16:45:54.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.8051947142857142, "accuracy_std_dev": 0.014445950710090603, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.022680748196248195, "avg_duration_per_task_ms": 254100.05483405484, "avg_output_tokens_per_task": 17520.435786435788, "last_run_timestamp": "2026-09-26T08:51:26.240Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.8049008888888889, "accuracy_std_dev": 0.023075324904150284, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.3397021683501683, "avg_duration_per_task_ms": 355471.79349046014, "avg_output_tokens_per_task": 22495.737373737375, "last_run_timestamp": "2026-09-26T09:00:45.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.8048629695290858, "accuracy_std_dev": 0.015273014540588393, "run_count": 11, "total_questions": 2166, "avg_cost_per_task_usd": 0.011905135734072023, "avg_duration_per_task_ms": 306077.56463527237, "avg_output_tokens_per_task": 12391.826869806095, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.804714, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.024081237373737373, "avg_duration_per_task_ms": 24354.015151515152, "avg_output_tokens_per_task": 1625.5555555555557, "last_run_timestamp": "2026-08-01T08:38:11.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.8043396111111111, "accuracy_std_dev": 0.016346822909594834, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.02834438776655444, "avg_duration_per_task_ms": 298008.4017957351, "avg_output_tokens_per_task": 16109.744388327721, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.8039509370988447, "accuracy_std_dev": 0.04606482842170186, "run_count": 12, "total_questions": 2337, "avg_cost_per_task_usd": 0.004562442875481386, "avg_duration_per_task_ms": 529537.7351305092, "avg_output_tokens_per_task": 12022.224646983312, "last_run_timestamp": "2026-08-29T19:15:43.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.8028880787003609, "accuracy_std_dev": 0.016209074791917563, "run_count": 21, "total_questions": 4155, "avg_cost_per_task_usd": 0.017011109987966302, "avg_duration_per_task_ms": 166436.28519855597, "avg_output_tokens_per_task": 11224.947051744886, "last_run_timestamp": "2026-08-29T18:07:47.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8025491428571428, "accuracy_std_dev": 0.025111070964409142, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.33407534559884555, "avg_duration_per_task_ms": 352449.7582972583, "avg_output_tokens_per_task": 22169.862914862915, "last_run_timestamp": "2026-09-26T08:59:18.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.8022664747948416, "accuracy_std_dev": 0.03416052727843685, "run_count": 13, "total_questions": 2559, "avg_cost_per_task_usd": 0.10685159788980071, "avg_duration_per_task_ms": 697410.7983587339, "avg_output_tokens_per_task": 34433.65064478312, "last_run_timestamp": "2026-08-29T18:40:24.764Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8021885000000001, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04024864141414142, "avg_duration_per_task_ms": 73206.09595959596, "avg_output_tokens_per_task": 26709.396464646463, "last_run_timestamp": "2026-09-01T17:53:21.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DeepInfra", "accuracy": 0.8019233052236256, "accuracy_std_dev": 0.09230759113302772, "run_count": 15, "total_questions": 2929, "avg_cost_per_task_usd": 0.010292674291567086, "avg_duration_per_task_ms": 247238.68316831684, "avg_output_tokens_per_task": 8923.839877091157, "last_run_timestamp": "2026-07-11T15:51:15.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8016385562742562, "accuracy_std_dev": 0.007745278218856984, "run_count": 4, "total_questions": 773, "avg_cost_per_task_usd": 0.0594386532988357, "avg_duration_per_task_ms": 220979.64165588617, "avg_output_tokens_per_task": 17329.77360931436, "last_run_timestamp": "2026-09-26T08:53:08.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.8016302663043479, "accuracy_std_dev": 0.016866645256903415, "run_count": 6, "total_questions": 1104, "avg_cost_per_task_usd": 0.053515479166666664, "avg_duration_per_task_ms": 272256.60507246375, "avg_output_tokens_per_task": 12863.78713768116, "last_run_timestamp": "2026-07-18T08:30:38.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.8015572499999999, "accuracy_std_dev": 0.051298469831183385, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.15178521254208757, "avg_duration_per_task_ms": 725758.385942761, "avg_output_tokens_per_task": 49481.54461279461, "last_run_timestamp": "2026-09-26T09:47:43.340Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.8015315724299064, "accuracy_std_dev": 0.03958967650685164, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.03791428894080996, "avg_duration_per_task_ms": 408085.0218068536, "avg_output_tokens_per_task": 37928.07982866043, "last_run_timestamp": "2026-09-26T16:31:04.519Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8013465, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.041531338383838386, "avg_duration_per_task_ms": 80828.55555555556, "avg_output_tokens_per_task": 27572.47474747475, "last_run_timestamp": "2026-09-01T17:50:00.305Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.8012819999999999, "accuracy_std_dev": 0.08131801393957507, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.0891624254079254, "avg_duration_per_task_ms": 504605.8787878788, "avg_output_tokens_per_task": 28207.464257964257, "last_run_timestamp": "2026-09-26T10:00:58.971Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.8004351497695853, "accuracy_std_dev": 0.16221306402555435, "run_count": 16, "total_questions": 2604, "avg_cost_per_task_usd": 0.07896243087557604, "avg_duration_per_task_ms": 344311.57757296466, "avg_output_tokens_per_task": 18416.05376344086, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.8001267261055633, "accuracy_std_dev": 0.014586786566292738, "run_count": 11, "total_questions": 2103, "avg_cost_per_task_usd": 0.010285525915359011, "avg_duration_per_task_ms": 337066.20827389444, "avg_output_tokens_per_task": 10713.090347123158, "last_run_timestamp": "2026-08-15T14:56:55.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.799421448833034, "accuracy_std_dev": 0.026141795886530258, "run_count": 17, "total_questions": 3342, "avg_cost_per_task_usd": 0.010835476361460206, "avg_duration_per_task_ms": 320191.3922800718, "avg_output_tokens_per_task": 10995.220227408738, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.7994009233361414, "accuracy_std_dev": 0.05494984321908906, "run_count": 18, "total_questions": 3561, "avg_cost_per_task_usd": 0.004052114574557709, "avg_duration_per_task_ms": 221480.79837124402, "avg_output_tokens_per_task": 9875.90255546195, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.7988753739571999, "accuracy_std_dev": 0.14188825709410938, "run_count": 14, "total_questions": 2757, "avg_cost_per_task_usd": 0.004531228509249184, "avg_duration_per_task_ms": 359570.6064562931, "avg_output_tokens_per_task": 11210.340587595212, "last_run_timestamp": "2026-08-29T19:18:32.746Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.7988215, "accuracy_std_dev": 0.010942500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.25381196717171717, "avg_duration_per_task_ms": 225693.02525252526, "avg_output_tokens_per_task": 25403.512626262625, "last_run_timestamp": "2026-09-01T13:22:15.928Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.79798, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2411195303030303, "avg_duration_per_task_ms": 184685.6616161616, "avg_output_tokens_per_task": 24018.91919191919, "last_run_timestamp": "2026-09-01T13:34:01.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.79798, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03602157070707071, "avg_duration_per_task_ms": 32858.843434343435, "avg_output_tokens_per_task": 2479.818181818182, "last_run_timestamp": "2026-08-01T08:32:36.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.7979798, "accuracy_std_dev": 0.09258320044241286, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010301554545454545, "avg_duration_per_task_ms": 568986.8757575757, "avg_output_tokens_per_task": 24756.19292929293, "last_run_timestamp": "2026-08-29T19:51:27.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.7979313627639155, "accuracy_std_dev": 0.011188259129097978, "run_count": 16, "total_questions": 3126, "avg_cost_per_task_usd": 0.040375727767114525, "avg_duration_per_task_ms": 913771.9331413948, "avg_output_tokens_per_task": 18420.52111324376, "last_run_timestamp": "2026-08-29T20:02:23.660Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7979040119760479, "accuracy_std_dev": 0.02389207889003172, "run_count": 4, "total_questions": 668, "avg_cost_per_task_usd": 0.15626441916167666, "avg_duration_per_task_ms": 1788980.868263473, "avg_output_tokens_per_task": 46933.58532934132, "last_run_timestamp": "2026-08-22T16:44:47.658Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.7972682237569061, "accuracy_std_dev": 0.038434880339272746, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.035596821823204415, "avg_duration_per_task_ms": 774714.9424493554, "avg_output_tokens_per_task": 20523.184622467772, "last_run_timestamp": "2026-09-26T09:34:13.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.7964700671480144, "accuracy_std_dev": 0.03714125671277268, "run_count": 21, "total_questions": 4155, "avg_cost_per_task_usd": 0.01084919157641396, "avg_duration_per_task_ms": 353681.98291215405, "avg_output_tokens_per_task": 10705.062815884476, "last_run_timestamp": "2026-08-30T00:18:59.365Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7964375923664123, "accuracy_std_dev": 0.11378331267012533, "run_count": 4, "total_questions": 655, "avg_cost_per_task_usd": 0.011028116030534352, "avg_duration_per_task_ms": 270893.28854961833, "avg_output_tokens_per_task": 23218.567938931297, "last_run_timestamp": "2026-09-12T08:53:48.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7962965, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.044405648989898995, "avg_duration_per_task_ms": 249218.0202020202, "avg_output_tokens_per_task": 22108.909090909092, "last_run_timestamp": "2026-09-01T18:00:27.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.796296, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04404542424242425, "avg_duration_per_task_ms": 221111.49494949495, "avg_output_tokens_per_task": 21928.032828282827, "last_run_timestamp": "2026-09-01T18:07:51.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.79613792022792, "accuracy_std_dev": 0.03686958809587348, "run_count": 16, "total_questions": 3159, "avg_cost_per_task_usd": 0.03350250300728078, "avg_duration_per_task_ms": 409082.3953782843, "avg_output_tokens_per_task": 16575.017094017094, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.795641611111111, "accuracy_std_dev": 0.10336463207668649, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.02163882828282828, "avg_duration_per_task_ms": 103515.63888888889, "avg_output_tokens_per_task": 16579.40095398429, "last_run_timestamp": "2026-08-29T19:13:10.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7956189598572703, "accuracy_std_dev": 0.02768081371661319, "run_count": 17, "total_questions": 3363, "avg_cost_per_task_usd": 0.009608434136187926, "avg_duration_per_task_ms": 421216.80582812964, "avg_output_tokens_per_task": 24272.67320844484, "last_run_timestamp": "2026-08-29T18:49:24.022Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7953039029322547, "accuracy_std_dev": 0.0703931299351167, "run_count": 15, "total_questions": 2967, "avg_cost_per_task_usd": 0.07654278227165487, "avg_duration_per_task_ms": 510887.7367711493, "avg_output_tokens_per_task": 24439.035389282104, "last_run_timestamp": "2026-08-22T17:17:31.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7936028, "accuracy_std_dev": 0.016273367695716804, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.017573431313131313, "avg_duration_per_task_ms": 59390.138383838384, "avg_output_tokens_per_task": 5706.428282828283, "last_run_timestamp": "2026-08-29T18:05:48.084Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.7929293571428571, "accuracy_std_dev": 0.017530103445245845, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.011040959595959597, "avg_duration_per_task_ms": 175837.8427128427, "avg_output_tokens_per_task": 6562.907287157287, "last_run_timestamp": "2026-07-18T16:16:42.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7929289999999999, "accuracy_std_dev": 0.016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03961820454545454, "avg_duration_per_task_ms": 540887.0883838384, "avg_output_tokens_per_task": 13782.598484848484, "last_run_timestamp": "2026-08-08T09:30:55.143Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7924127262693157, "accuracy_std_dev": 0.05800564622951288, "run_count": 21, "total_questions": 4077, "avg_cost_per_task_usd": 0.07450861589403973, "avg_duration_per_task_ms": 572032.6973264655, "avg_output_tokens_per_task": 33361.80819229826, "last_run_timestamp": "2026-08-29T18:23:09.250Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.7920985116580311, "accuracy_std_dev": 0.11713420011607767, "run_count": 12, "total_questions": 2316, "avg_cost_per_task_usd": 0.10608431692573402, "avg_duration_per_task_ms": 736930.4978411053, "avg_output_tokens_per_task": 31587.977979274612, "last_run_timestamp": "2026-08-29T19:03:21.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7920687490720117, "accuracy_std_dev": 0.023696432575952595, "run_count": 15, "total_questions": 2694, "avg_cost_per_task_usd": 0.035895181143281364, "avg_duration_per_task_ms": 624123.5913140312, "avg_output_tokens_per_task": 19506.129547141798, "last_run_timestamp": "2026-08-29T18:31:11.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7919846781170483, "accuracy_std_dev": 0.05508513093858188, "run_count": 13, "total_questions": 2358, "avg_cost_per_task_usd": 0.0414582417302799, "avg_duration_per_task_ms": 452143.6522476675, "avg_output_tokens_per_task": 21406.899491094147, "last_run_timestamp": "2026-09-26T09:56:00.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.7910053138664271, "accuracy_std_dev": 0.15895006299418277, "run_count": 17, "total_questions": 3339, "avg_cost_per_task_usd": 0.011569411799940102, "avg_duration_per_task_ms": 139037.51362683438, "avg_output_tokens_per_task": 9589.577118897874, "last_run_timestamp": "2026-08-01T18:21:26.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7909026798917945, "accuracy_std_dev": 0.03754431133546038, "run_count": 17, "total_questions": 3327, "avg_cost_per_task_usd": 0.06559778960024044, "avg_duration_per_task_ms": 355359.5933273219, "avg_output_tokens_per_task": 19344.585813044785, "last_run_timestamp": "2026-08-29T18:54:50.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.790404, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0189197702020202, "avg_duration_per_task_ms": 459328.30303030304, "avg_output_tokens_per_task": 25138.35606060606, "last_run_timestamp": "2026-08-29T19:17:42.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.7900883420677869, "accuracy_std_dev": 0.09870007461496674, "run_count": 18, "total_questions": 3511, "avg_cost_per_task_usd": 0.015616578182853886, "avg_duration_per_task_ms": 212237.6132156081, "avg_output_tokens_per_task": 15795.737396753062, "last_run_timestamp": "2026-08-29T20:00:43.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.790014, "accuracy_std_dev": 0.07589864878664765, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.016462866666666666, "avg_duration_per_task_ms": 474335.2464135021, "avg_output_tokens_per_task": 17867.735021097047, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7888498053927315, "accuracy_std_dev": 0.014122812704500139, "run_count": 13, "total_questions": 2559, "avg_cost_per_task_usd": 0.13022648261039466, "avg_duration_per_task_ms": 677813.3364595545, "avg_output_tokens_per_task": 32555.26768268855, "last_run_timestamp": "2026-08-29T20:08:40.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.7888041145038167, "accuracy_std_dev": 0.010988374090480538, "run_count": 16, "total_questions": 3144, "avg_cost_per_task_usd": 0.039076502544529254, "avg_duration_per_task_ms": 708080.6669847328, "avg_output_tokens_per_task": 17823.317748091602, "last_run_timestamp": "2026-08-29T19:47:52.368Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7885914761215629, "accuracy_std_dev": 0.02234077768730239, "run_count": 7, "total_questions": 1382, "avg_cost_per_task_usd": 0.06106606946454414, "avg_duration_per_task_ms": 179070.87192474675, "avg_output_tokens_per_task": 19226.02532561505, "last_run_timestamp": "2026-09-26T09:36:55.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7884858931297708, "accuracy_std_dev": 0.009706627796897349, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.01825112468193384, "avg_duration_per_task_ms": 147479.76017811705, "avg_output_tokens_per_task": 15055.673664122138, "last_run_timestamp": "2026-09-26T09:15:51.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7884401666666667, "accuracy_std_dev": 0.023761946338748348, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.04315188552188553, "avg_duration_per_task_ms": 84212.74747474748, "avg_output_tokens_per_task": 26885.046296296296, "last_run_timestamp": "2026-09-01T17:55:54.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7882558207964601, "accuracy_std_dev": 0.04689700002782695, "run_count": 10, "total_questions": 1808, "avg_cost_per_task_usd": 0.00581368860619469, "avg_duration_per_task_ms": 499550.75, "avg_output_tokens_per_task": 13199.183628318584, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.7862293764980027, "accuracy_std_dev": 0.1666251748521213, "run_count": 16, "total_questions": 3004, "avg_cost_per_task_usd": 0.00942010652463382, "avg_duration_per_task_ms": 248834.9277629827, "avg_output_tokens_per_task": 7858.000332889481, "last_run_timestamp": "2026-07-18T16:53:38.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "WandB", "accuracy": 0.7861595487316422, "accuracy_std_dev": 0.02054339937029649, "run_count": 4, "total_questions": 749, "avg_cost_per_task_usd": 0.023644552736982648, "avg_duration_per_task_ms": 116214.79706275034, "avg_output_tokens_per_task": 18836.48865153538, "last_run_timestamp": "2026-07-18T08:04:57.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.785920719858156, "accuracy_std_dev": 0.021394714621875175, "run_count": 13, "total_questions": 2538, "avg_cost_per_task_usd": 0.04721227226162333, "avg_duration_per_task_ms": 569592.5594956658, "avg_output_tokens_per_task": 20532.268321513002, "last_run_timestamp": "2026-08-29T16:48:15.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.7853535, "accuracy_std_dev": 0.02272750000000001, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.002243229797979798, "avg_duration_per_task_ms": 474421.00757575757, "avg_output_tokens_per_task": 14645.75, "last_run_timestamp": "2026-06-20T12:44:34.391Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "AtlasCloud", "accuracy": 0.7850717877461707, "accuracy_std_dev": 0.024805753669301024, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.024575857038657913, "avg_duration_per_task_ms": 416592.56673960615, "avg_output_tokens_per_task": 11405.052516411379, "last_run_timestamp": "2026-07-01T04:44:07.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7847221249999999, "accuracy_std_dev": 0.15720335838129978, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.017239123106060607, "avg_duration_per_task_ms": 568625.6464646464, "avg_output_tokens_per_task": 27456.18876262626, "last_run_timestamp": "2026-09-26T08:08:15.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.7835429351415094, "accuracy_std_dev": 0.03263391726736184, "run_count": 13, "total_questions": 2544, "avg_cost_per_task_usd": 0.004777952437106918, "avg_duration_per_task_ms": 504503.3958333333, "avg_output_tokens_per_task": 11386.457547169812, "last_run_timestamp": "2026-08-01T18:07:34.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.7829818899297424, "accuracy_std_dev": 0.05788395901278767, "run_count": 13, "total_questions": 2562, "avg_cost_per_task_usd": 0.012435911787665888, "avg_duration_per_task_ms": 294718.2923497268, "avg_output_tokens_per_task": 12376.93169398907, "last_run_timestamp": "2026-08-29T16:52:23.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.7824072499999999, "accuracy_std_dev": 0.0034452088742919643, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.04613476767676768, "avg_duration_per_task_ms": 199616.12626262626, "avg_output_tokens_per_task": 21878.295454545456, "last_run_timestamp": "2026-09-01T19:10:24.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.7823070191082802, "accuracy_std_dev": 0.03047559886602813, "run_count": 12, "total_questions": 2355, "avg_cost_per_task_usd": 0.0051865269639065825, "avg_duration_per_task_ms": 669935.3341825902, "avg_output_tokens_per_task": 12408.887048832272, "last_run_timestamp": "2026-08-01T22:57:01.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.7819865, "accuracy_std_dev": 0.016169511173295674, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.252381888047138, "avg_duration_per_task_ms": 227878.92592592593, "avg_output_tokens_per_task": 25272.329124579126, "last_run_timestamp": "2026-09-01T13:35:35.064Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7816287308868499, "accuracy_std_dev": 0.04679161436791379, "run_count": 15, "total_questions": 2943, "avg_cost_per_task_usd": 0.03975552463472646, "avg_duration_per_task_ms": 656449.841658172, "avg_output_tokens_per_task": 20722.471967380225, "last_run_timestamp": "2026-08-29T18:19:18.341Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.7808641666666666, "accuracy_std_dev": 0.010351374971739527, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.021081101010101008, "avg_duration_per_task_ms": 286187.4873737374, "avg_output_tokens_per_task": 12540.748316498317, "last_run_timestamp": "2026-08-30T00:44:42.602Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Fireworks", "accuracy": 0.7803030000000001, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006998724747474747, "avg_duration_per_task_ms": 57307.29797979798, "avg_output_tokens_per_task": 4087.7727272727275, "last_run_timestamp": "2026-05-09T16:48:20.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Fireworks", "accuracy": 0.780303, "accuracy_std_dev": 0.08147600069959253, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07258149494949495, "avg_duration_per_task_ms": 318265.46338383836, "avg_output_tokens_per_task": 18090.291666666668, "last_run_timestamp": "2026-05-23T11:07:21.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.7802894910474485, "accuracy_std_dev": 0.1302593050858772, "run_count": 13, "total_questions": 2234, "avg_cost_per_task_usd": 0.020609962399283796, "avg_duration_per_task_ms": 473420.5805729633, "avg_output_tokens_per_task": 12826.314682184422, "last_run_timestamp": "2026-08-01T23:27:37.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.7800773017429193, "accuracy_std_dev": 0.021465098000437065, "run_count": 14, "total_questions": 2754, "avg_cost_per_task_usd": 0.04367889179375453, "avg_duration_per_task_ms": 454980.2806826434, "avg_output_tokens_per_task": 19988.257080610023, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.7794614999999999, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013140674242424242, "avg_duration_per_task_ms": 69975.75757575757, "avg_output_tokens_per_task": 10390.04797979798, "last_run_timestamp": "2026-09-01T17:51:03.252Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.7793682236135957, "accuracy_std_dev": 0.06789026487076204, "run_count": 3, "total_questions": 559, "avg_cost_per_task_usd": 0.029017345259391772, "avg_duration_per_task_ms": 2038522.3381037568, "avg_output_tokens_per_task": 57994.92486583184, "last_run_timestamp": "2026-09-26T09:27:27.697Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7783505180412369, "accuracy_std_dev": 0.01799990454372838, "run_count": 12, "total_questions": 2328, "avg_cost_per_task_usd": 0.012542361254295534, "avg_duration_per_task_ms": 614270.4780927835, "avg_output_tokens_per_task": 31292.053694158076, "last_run_timestamp": "2026-08-29T16:45:34.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7781144000000001, "accuracy_std_dev": 0.02296971973359712, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.20166651515151518, "avg_duration_per_task_ms": 657386.6868686868, "avg_output_tokens_per_task": 48003.93838383839, "last_run_timestamp": "2026-08-29T18:10:32.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7773231747713049, "accuracy_std_dev": 0.02619804956117758, "run_count": 11, "total_questions": 2077, "avg_cost_per_task_usd": 0.009061719306692344, "avg_duration_per_task_ms": 444770.355801637, "avg_output_tokens_per_task": 36121.13769860376, "last_run_timestamp": "2026-09-26T17:33:23.516Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Friendli", "accuracy": 0.7772597692307692, "accuracy_std_dev": 0.2020442328909321, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.056136898989899, "avg_duration_per_task_ms": 168843.6891996892, "avg_output_tokens_per_task": 17466.01320901321, "last_run_timestamp": "2026-07-01T03:02:14.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.774354836492891, "accuracy_std_dev": 0.025023485914745035, "run_count": 13, "total_questions": 2532, "avg_cost_per_task_usd": 0.005451107424960505, "avg_duration_per_task_ms": 662585.4802527647, "avg_output_tokens_per_task": 34288.59873617694, "last_run_timestamp": "2026-09-26T19:06:56.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.774186536121673, "accuracy_std_dev": 0.045556886694000515, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.0076262953105196455, "avg_duration_per_task_ms": 513514.7541191382, "avg_output_tokens_per_task": 15020.147021546261, "last_run_timestamp": "2026-09-26T08:55:07.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7733335691428571, "accuracy_std_dev": 0.10983726595387482, "run_count": 6, "total_questions": 875, "avg_cost_per_task_usd": 0.013626054857142857, "avg_duration_per_task_ms": 549081.1371428572, "avg_output_tokens_per_task": 22934.75657142857, "last_run_timestamp": "2026-09-26T10:43:59.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.7733209195128408, "accuracy_std_dev": 0.08870509019372362, "run_count": 21, "total_questions": 3777, "avg_cost_per_task_usd": 0.01586187847498014, "avg_duration_per_task_ms": 345375.779719354, "avg_output_tokens_per_task": 16140.08895949166, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7727855930680358, "accuracy_std_dev": 0.05706427864424032, "run_count": 12, "total_questions": 2337, "avg_cost_per_task_usd": 0.0018226653829696191, "avg_duration_per_task_ms": 478170.8801882756, "avg_output_tokens_per_task": 11197.934103551563, "last_run_timestamp": "2026-08-22T17:47:50.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7721407260997067, "accuracy_std_dev": 0.028224906684324803, "run_count": 11, "total_questions": 1705, "avg_cost_per_task_usd": 0.009176458064516129, "avg_duration_per_task_ms": 41666.574780058654, "avg_output_tokens_per_task": 12170.292082111437, "last_run_timestamp": "2026-08-29T19:07:43.372Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7715098846153846, "accuracy_std_dev": 0.09651313170593676, "run_count": 12, "total_questions": 2340, "avg_cost_per_task_usd": 0.09523880213675213, "avg_duration_per_task_ms": 1068926.4589743589, "avg_output_tokens_per_task": 33358.00341880342, "last_run_timestamp": "2026-08-29T18:41:58.319Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7714368913043479, "accuracy_std_dev": 0.014261246953428234, "run_count": 3, "total_questions": 552, "avg_cost_per_task_usd": 0.23206160869565215, "avg_duration_per_task_ms": 164099.60869565216, "avg_output_tokens_per_task": 15307.03079710145, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.770754223767383, "accuracy_std_dev": 0.04391892511863639, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.005076832279814581, "avg_duration_per_task_ms": 274364.3603034134, "avg_output_tokens_per_task": 33323.141171512856, "last_run_timestamp": "2026-09-26T16:14:01.905Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7698924683870967, "accuracy_std_dev": 0.033487219191307406, "run_count": 12, "total_questions": 2325, "avg_cost_per_task_usd": 0.011860219354838712, "avg_duration_per_task_ms": 404374.8103225806, "avg_output_tokens_per_task": 29903.957849462364, "last_run_timestamp": "2026-08-29T16:43:11.936Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Phala", "accuracy": 0.7688491071428571, "accuracy_std_dev": 0.036143499999999995, "run_count": 2, "total_questions": 168, "avg_cost_per_task_usd": 0.013457690476190477, "avg_duration_per_task_ms": 239500.45238095237, "avg_output_tokens_per_task": 24312.886904761905, "last_run_timestamp": "2026-07-18T09:01:08.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.7685185, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013512386363636364, "avg_duration_per_task_ms": 71792.87121212122, "avg_output_tokens_per_task": 10688.070707070707, "last_run_timestamp": "2026-09-01T17:55:24.769Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.7680775714285716, "accuracy_std_dev": 0.014666364043105512, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.011148957431457432, "avg_duration_per_task_ms": 51087.674122174125, "avg_output_tokens_per_task": 18441.45286195286, "last_run_timestamp": "2026-08-29T18:36:51.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Chutes", "accuracy": 0.7673517422506738, "accuracy_std_dev": 0.1696644004135443, "run_count": 16, "total_questions": 2968, "avg_cost_per_task_usd": 0.04616438106469003, "avg_duration_per_task_ms": 423953.0438005391, "avg_output_tokens_per_task": 18353.38847708895, "last_run_timestamp": "2026-07-18T16:43:10.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.7672560000000002, "accuracy_std_dev": 0.0089379450938121, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013739436868686868, "avg_duration_per_task_ms": 117371.45833333333, "avg_output_tokens_per_task": 10352.156565656565, "last_run_timestamp": "2026-09-01T17:58:15.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7672559999999999, "accuracy_std_dev": 0.008469894233892938, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.27037940151515155, "avg_duration_per_task_ms": 263755.6675084175, "avg_output_tokens_per_task": 17837.74579124579, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7671957142857142, "accuracy_std_dev": 0.009679973770836427, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.017645118326118325, "avg_duration_per_task_ms": 105010.88528138529, "avg_output_tokens_per_task": 6033.81746031746, "last_run_timestamp": "2026-09-26T09:17:54.791Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.01935999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2670539015151515, "avg_duration_per_task_ms": 177095.6616161616, "avg_output_tokens_per_task": 17616.689393939392, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7668169148936171, "accuracy_std_dev": 0.053587367069901595, "run_count": 8, "total_questions": 1551, "avg_cost_per_task_usd": 0.010489795615731785, "avg_duration_per_task_ms": 700478.4300451322, "avg_output_tokens_per_task": 31633.707930367505, "last_run_timestamp": "2026-09-12T09:07:23.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05139547999999999, "avg_duration_per_task_ms": 172458.62, "avg_output_tokens_per_task": 21216.74, "last_run_timestamp": "2026-09-14T19:42:46.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.76638602247191, "accuracy_std_dev": 0.0119351260850762, "run_count": 3, "total_questions": 356, "avg_cost_per_task_usd": 0.002961258426966292, "avg_duration_per_task_ms": 69864.89325842696, "avg_output_tokens_per_task": 3595.182584269663, "last_run_timestamp": "2026-08-15T14:54:48.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.765993, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08016306060606061, "avg_duration_per_task_ms": 425226.22222222225, "avg_output_tokens_per_task": 69822.58585858585, "last_run_timestamp": "2026-07-18T08:29:23.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.765512238095238, "accuracy_std_dev": 0.012007319862344141, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.011864828763828762, "avg_duration_per_task_ms": 112663.68951418951, "avg_output_tokens_per_task": 19644.012265512265, "last_run_timestamp": "2026-08-29T20:01:12.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.76443, "accuracy_std_dev": 0.03329192690428117, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.013529336940836939, "avg_duration_per_task_ms": 416657.21428571426, "avg_output_tokens_per_task": 23392.275974025975, "last_run_timestamp": "2026-08-29T18:36:10.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7643024133099825, "accuracy_std_dev": 0.031491855215101075, "run_count": 6, "total_questions": 1142, "avg_cost_per_task_usd": 0.04750308056042031, "avg_duration_per_task_ms": 265287.42907180387, "avg_output_tokens_per_task": 17893.702276707532, "last_run_timestamp": "2026-09-26T09:16:55.239Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.7642404121475055, "accuracy_std_dev": 0.012311945545143383, "run_count": 21, "total_questions": 4149, "avg_cost_per_task_usd": 0.009483342251144855, "avg_duration_per_task_ms": 98709.46348517716, "avg_output_tokens_per_task": 18943.602795854422, "last_run_timestamp": "2026-08-29T18:39:41.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.7638135731310943, "accuracy_std_dev": 0.022048436994096723, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.005248553990610328, "avg_duration_per_task_ms": 264357.4124232575, "avg_output_tokens_per_task": 16758.418201516793, "last_run_timestamp": "2026-07-18T16:28:34.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7637618470588236, "accuracy_std_dev": 0.12321473328330662, "run_count": 12, "total_questions": 2295, "avg_cost_per_task_usd": 0.0732439917211329, "avg_duration_per_task_ms": 793890.7620915032, "avg_output_tokens_per_task": 27412.28496732026, "last_run_timestamp": "2026-09-26T10:34:37.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.7636952078580483, "accuracy_std_dev": 0.023951539715497544, "run_count": 12, "total_questions": 2367, "avg_cost_per_task_usd": 0.010468032530629489, "avg_duration_per_task_ms": 136682.62822137726, "avg_output_tokens_per_task": 4070.7904520490074, "last_run_timestamp": "2026-08-01T17:44:11.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7636680000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.14155542328042328, "avg_duration_per_task_ms": 2288029.2486772486, "avg_output_tokens_per_task": 37245.06349206349, "last_run_timestamp": "2026-08-23T18:19:25.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "WandB", "accuracy": 0.7636362000000001, "accuracy_std_dev": 0.02440997912657852, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00031175454545454543, "avg_duration_per_task_ms": 34241.53636363636, "avg_output_tokens_per_task": 2848.0313131313133, "last_run_timestamp": "2026-06-20T16:16:12.961Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7632893237410072, "accuracy_std_dev": 0.11557017973361858, "run_count": 17, "total_questions": 3336, "avg_cost_per_task_usd": 0.0028636052158273377, "avg_duration_per_task_ms": 431031.65257793764, "avg_output_tokens_per_task": 14939.968824940048, "last_run_timestamp": "2026-08-29T19:07:25.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.7625483969176351, "accuracy_std_dev": 0.04157398706916983, "run_count": 20, "total_questions": 3958, "avg_cost_per_task_usd": 0.012629246336533605, "avg_duration_per_task_ms": 83704.65184436584, "avg_output_tokens_per_task": 7288.428499242042, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.760943, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08179280303030302, "avg_duration_per_task_ms": 426340.89898989897, "avg_output_tokens_per_task": 70994.31818181818, "last_run_timestamp": "2026-07-18T08:26:55.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.760674964876033, "accuracy_std_dev": 0.01645419404771925, "run_count": 5, "total_questions": 968, "avg_cost_per_task_usd": 0.010708528925619836, "avg_duration_per_task_ms": 425619.78925619833, "avg_output_tokens_per_task": 25673.311983471074, "last_run_timestamp": "2026-08-29T18:56:40.728Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.7606652169197398, "accuracy_std_dev": 0.04591514075032818, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.004993729573391178, "avg_duration_per_task_ms": 317882.5820679682, "avg_output_tokens_per_task": 12242.755603759942, "last_run_timestamp": "2026-08-29T18:46:19.350Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.7605443265306122, "accuracy_std_dev": 0.0343317834601635, "run_count": 3, "total_questions": 490, "avg_cost_per_task_usd": 0.0037374408163265305, "avg_duration_per_task_ms": 668153.0571428571, "avg_output_tokens_per_task": 14206.583673469388, "last_run_timestamp": "2026-08-08T11:42:26.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.7595521475625824, "accuracy_std_dev": 0.019571763499602556, "run_count": 8, "total_questions": 1518, "avg_cost_per_task_usd": 0.013668990118577072, "avg_duration_per_task_ms": 155525.03227931488, "avg_output_tokens_per_task": 18166.727272727272, "last_run_timestamp": "2026-08-29T18:11:51.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.759259, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018210444444444442, "avg_duration_per_task_ms": 254627.46717171717, "avg_output_tokens_per_task": 24079.401515151516, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7590111668383109, "accuracy_std_dev": 0.016611473377157127, "run_count": 5, "total_questions": 971, "avg_cost_per_task_usd": 0.004071515962924819, "avg_duration_per_task_ms": 4054.4541709577757, "avg_output_tokens_per_task": 2252.576725025747, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7587190035005834, "accuracy_std_dev": 0.01366534537530361, "run_count": 26, "total_questions": 5142, "avg_cost_per_task_usd": 0.015162635355892653, "avg_duration_per_task_ms": 166747.76215480358, "avg_output_tokens_per_task": 25091.703033838974, "last_run_timestamp": "2026-08-29T16:18:21.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.7585374615196078, "accuracy_std_dev": 0.038599957465404885, "run_count": 21, "total_questions": 4080, "avg_cost_per_task_usd": 0.01587172034313725, "avg_duration_per_task_ms": 44875.42181372549, "avg_output_tokens_per_task": 16664.9625, "last_run_timestamp": "2026-08-29T19:51:33.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.757881909090909, "accuracy_std_dev": 0.01539297995518821, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.010656316804407715, "avg_duration_per_task_ms": 94392.0583103765, "avg_output_tokens_per_task": 3877.1974288337924, "last_run_timestamp": "2026-08-01T17:45:27.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7575162329411763, "accuracy_std_dev": 0.07851061401993352, "run_count": 13, "total_questions": 2550, "avg_cost_per_task_usd": 0.004892281568627452, "avg_duration_per_task_ms": 717348.5560784314, "avg_output_tokens_per_task": 32493.26745098039, "last_run_timestamp": "2026-09-26T18:04:35.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.7572815631067961, "accuracy_std_dev": 0.10286820253600068, "run_count": 9, "total_questions": 721, "avg_cost_per_task_usd": 0.002934966712898752, "avg_duration_per_task_ms": 194322.5145631068, "avg_output_tokens_per_task": 13374.00693481276, "last_run_timestamp": "2026-09-26T16:00:24.612Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.7565450000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.020916916230366494, "avg_duration_per_task_ms": 124793.11518324607, "avg_output_tokens_per_task": 18115.47120418848, "last_run_timestamp": "2026-07-18T08:26:49.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7559323627118644, "accuracy_std_dev": 0.007798453236949536, "run_count": 3, "total_questions": 590, "avg_cost_per_task_usd": 0.002668813559322034, "avg_duration_per_task_ms": 91653.82203389831, "avg_output_tokens_per_task": 31189.77966101695, "last_run_timestamp": "2026-09-01T13:23:13.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "AtlasCloud", "accuracy": 0.7557626153846154, "accuracy_std_dev": 0.1999902666568237, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.005653325951825951, "avg_duration_per_task_ms": 41439.5710955711, "avg_output_tokens_per_task": 5907.5940170940175, "last_run_timestamp": "2026-07-01T02:27:21.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7552889455999998, "accuracy_std_dev": 0.020781831603270767, "run_count": 19, "total_questions": 3750, "avg_cost_per_task_usd": 0.010816569600000001, "avg_duration_per_task_ms": 208301.1416, "avg_output_tokens_per_task": 18668.501333333334, "last_run_timestamp": "2026-08-29T18:30:03.855Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.7551707142857145, "accuracy_std_dev": 0.008190904148841594, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02236280663780664, "avg_duration_per_task_ms": 98622.52741702742, "avg_output_tokens_per_task": 14763.670274170274, "last_run_timestamp": "2026-08-29T19:17:49.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.7551499775280899, "accuracy_std_dev": 0.03338976368963997, "run_count": 6, "total_questions": 1068, "avg_cost_per_task_usd": 0.006923780898876404, "avg_duration_per_task_ms": 301563.05337078654, "avg_output_tokens_per_task": 21238.912921348314, "last_run_timestamp": "2026-07-18T08:04:08.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7538529877807133, "accuracy_std_dev": 0.025619095703080515, "run_count": 17, "total_questions": 3028, "avg_cost_per_task_usd": 0.004401706406869221, "avg_duration_per_task_ms": 679926.5132100396, "avg_output_tokens_per_task": 11358.332892998678, "last_run_timestamp": "2026-08-29T19:19:08.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.7537320340136054, "accuracy_std_dev": 0.07541786806041245, "run_count": 18, "total_questions": 3528, "avg_cost_per_task_usd": 0.006723561224489798, "avg_duration_per_task_ms": 628214.5221088436, "avg_output_tokens_per_task": 7597.763888888889, "last_run_timestamp": "2026-08-30T01:20:44.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.7533753845344554, "accuracy_std_dev": 0.11640605414937877, "run_count": 11, "total_questions": 1901, "avg_cost_per_task_usd": 0.04040272172540768, "avg_duration_per_task_ms": 597850.3240399789, "avg_output_tokens_per_task": 17550.840610205156, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7525250000000001, "accuracy_std_dev": 0.010101000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.056976058080808085, "avg_duration_per_task_ms": 98919.65151515152, "avg_output_tokens_per_task": 22701.11111111111, "last_run_timestamp": "2026-09-01T17:54:55.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.7521143262182843, "accuracy_std_dev": 0.1621979257907336, "run_count": 14, "total_questions": 2483, "avg_cost_per_task_usd": 0.06938106121627065, "avg_duration_per_task_ms": 759326.9730165122, "avg_output_tokens_per_task": 24775.081755940395, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Inceptron", "accuracy": 0.7518907958115183, "accuracy_std_dev": 0.056756617575006676, "run_count": 6, "total_questions": 1146, "avg_cost_per_task_usd": 0.0764649572425829, "avg_duration_per_task_ms": 465218.3132635253, "avg_output_tokens_per_task": 17503.08987783595, "last_run_timestamp": "2026-06-30T23:33:15.910Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.75178689122807, "accuracy_std_dev": 0.06483392638966569, "run_count": 13, "total_questions": 2565, "avg_cost_per_task_usd": 0.01753287212475633, "avg_duration_per_task_ms": 827358.4358674464, "avg_output_tokens_per_task": 10936.669005847953, "last_run_timestamp": "2026-08-22T17:28:27.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7510363518653986, "accuracy_std_dev": 0.11063154442009523, "run_count": 7, "total_questions": 1367, "avg_cost_per_task_usd": 0.05419231089978055, "avg_duration_per_task_ms": 442904.11777615215, "avg_output_tokens_per_task": 22733.470373079737, "last_run_timestamp": "2026-08-15T10:07:37.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.750842, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01581958585858586, "avg_duration_per_task_ms": 326624.7373737374, "avg_output_tokens_per_task": 13062.823232323231, "last_run_timestamp": "2026-09-12T09:42:09.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.7504692996781979, "accuracy_std_dev": 0.12884102665725183, "run_count": 13, "total_questions": 2486, "avg_cost_per_task_usd": 0.03235196661303299, "avg_duration_per_task_ms": 504543.08045052295, "avg_output_tokens_per_task": 14950.808125502816, "last_run_timestamp": "2026-08-29T16:22:26.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.75, "accuracy_std_dev": null, "run_count": 1, "total_questions": 150, "avg_cost_per_task_usd": 0.0067602800000000005, "avg_duration_per_task_ms": 1824454.58, "avg_output_tokens_per_task": 23925.293333333335, "last_run_timestamp": "2026-09-22T21:48:57.116Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.7493782527075812, "accuracy_std_dev": 0.028699053688207896, "run_count": 21, "total_questions": 4155, "avg_cost_per_task_usd": 0.007139869554753309, "avg_duration_per_task_ms": 177604.2055354994, "avg_output_tokens_per_task": 14341.533814681106, "last_run_timestamp": "2026-08-29T19:17:18.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Cerebras", "accuracy": 0.7491580000000001, "accuracy_std_dev": 0.023808285322551064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0035041430976430975, "avg_duration_per_task_ms": 3687.0656565656564, "avg_output_tokens_per_task": 2784.715488215488, "last_run_timestamp": "2026-05-23T08:06:03.679Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "AtlasCloud", "accuracy": 0.7489477499999999, "accuracy_std_dev": 0.014071086487457193, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006045699494949496, "avg_duration_per_task_ms": 101503.2803030303, "avg_output_tokens_per_task": 2341.207702020202, "last_run_timestamp": "2026-07-01T05:04:09.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7485970000000001, "accuracy_std_dev": 0.07972918647312704, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006700525252525253, "avg_duration_per_task_ms": 93845.37542087543, "avg_output_tokens_per_task": 16482.319865319867, "last_run_timestamp": "2026-08-29T18:45:38.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.7479971828113618, "accuracy_std_dev": 0.07910913250219596, "run_count": 14, "total_questions": 2746, "avg_cost_per_task_usd": 0.006062376911871813, "avg_duration_per_task_ms": 285154.7793153678, "avg_output_tokens_per_task": 21130.692279679533, "last_run_timestamp": "2026-08-29T20:02:35.387Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.7475682777777778, "accuracy_std_dev": 0.015187563478808556, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.003526737373737373, "avg_duration_per_task_ms": 79157.85970819303, "avg_output_tokens_per_task": 5754.911335578002, "last_run_timestamp": "2026-08-29T18:15:48.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.7474749999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08105034848484848, "avg_duration_per_task_ms": 422601, "avg_output_tokens_per_task": 70367.68181818182, "last_run_timestamp": "2026-07-18T08:27:39.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.7467733333333335, "accuracy_std_dev": 0.012587939799224065, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.011146377104377103, "avg_duration_per_task_ms": 81852.9377104377, "avg_output_tokens_per_task": 12261.164983164983, "last_run_timestamp": "2026-08-29T18:17:28.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7465941264367815, "accuracy_std_dev": 0.10532480903970348, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.04816395977011494, "avg_duration_per_task_ms": 468882.4540229885, "avg_output_tokens_per_task": 36470.09642401022, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7464074370245138, "accuracy_std_dev": 0.026169100495032988, "run_count": 18, "total_questions": 3549, "avg_cost_per_task_usd": 0.10053423443223442, "avg_duration_per_task_ms": 535768.9887292195, "avg_output_tokens_per_task": 31443.99887292195, "last_run_timestamp": "2026-08-29T18:15:24.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7462122499999999, "accuracy_std_dev": 0.013222542538691302, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0095508547979798, "avg_duration_per_task_ms": 202963.16287878787, "avg_output_tokens_per_task": 8137.35101010101, "last_run_timestamp": "2026-08-29T18:16:24.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.746212, "accuracy_std_dev": 0.022409208720970065, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005678362373737374, "avg_duration_per_task_ms": 183239.09722222222, "avg_output_tokens_per_task": 11277.801767676769, "last_run_timestamp": "2026-05-23T09:41:28.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7460692307032589, "accuracy_std_dev": 0.11382303442613109, "run_count": 12, "total_questions": 2332, "avg_cost_per_task_usd": 0.03359197898799315, "avg_duration_per_task_ms": 1194136.795883362, "avg_output_tokens_per_task": 13708.225986277874, "last_run_timestamp": "2026-08-29T20:16:12.181Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.745544660208109, "accuracy_std_dev": 0.22950971531350736, "run_count": 17, "total_questions": 2787, "avg_cost_per_task_usd": 0.044433735916756374, "avg_duration_per_task_ms": 63269.68963042698, "avg_output_tokens_per_task": 14879.802655184787, "last_run_timestamp": "2026-08-15T14:54:40.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.7455108333333333, "accuracy_std_dev": 0.020304992673532824, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.08322730471380473, "avg_duration_per_task_ms": 907303.9099326599, "avg_output_tokens_per_task": 23781.781144781144, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.7453985513905684, "accuracy_std_dev": 0.1009705682494428, "run_count": 13, "total_questions": 2481, "avg_cost_per_task_usd": 0.1381599939540508, "avg_duration_per_task_ms": 992902.1555824265, "avg_output_tokens_per_task": 32901.31761386538, "last_run_timestamp": "2026-08-29T18:42:16.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.7450981176470588, "accuracy_std_dev": 0.012808884861607752, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.01028051633986928, "avg_duration_per_task_ms": 78818.64052287582, "avg_output_tokens_per_task": 10202.047534165182, "last_run_timestamp": "2026-08-29T18:11:58.511Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.7450696428571427, "accuracy_std_dev": 0.014631101665420764, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.00583946608946609, "avg_duration_per_task_ms": 147663.13888888888, "avg_output_tokens_per_task": 6788.261904761905, "last_run_timestamp": "2026-08-29T18:27:42.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.7449942438162545, "accuracy_std_dev": 0.01819967836712137, "run_count": 3, "total_questions": 283, "avg_cost_per_task_usd": 0.01679327561837456, "avg_duration_per_task_ms": 97120.16254416961, "avg_output_tokens_per_task": 9655.678445229682, "last_run_timestamp": "2026-08-30T00:01:09.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Together", "accuracy": 0.7431457142857144, "accuracy_std_dev": 0.01442199540101643, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0028307178932178937, "avg_duration_per_task_ms": 131645.240981241, "avg_output_tokens_per_task": 4593.8621933621935, "last_run_timestamp": "2026-07-04T17:56:44.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.7426647142857141, "accuracy_std_dev": 0.007278926328092368, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0053309747474747475, "avg_duration_per_task_ms": 143246.7821067821, "avg_output_tokens_per_task": 8614.1443001443, "last_run_timestamp": "2026-08-29T18:49:21.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.7425010399239544, "accuracy_std_dev": 0.040756824836413526, "run_count": 16, "total_questions": 3156, "avg_cost_per_task_usd": 0.008645365969581749, "avg_duration_per_task_ms": 476821.68187579216, "avg_output_tokens_per_task": 23199.611850443598, "last_run_timestamp": "2026-08-29T19:32:52.576Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Wafer", "accuracy": 0.7424240000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.048281424242424244, "avg_duration_per_task_ms": 987580.0707070708, "avg_output_tokens_per_task": 21326.60606060606, "last_run_timestamp": "2026-06-13T08:04:32.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.7423450257654967, "accuracy_std_dev": 0.057576989386154395, "run_count": 15, "total_questions": 2678, "avg_cost_per_task_usd": 0.08512316430171771, "avg_duration_per_task_ms": 515919.2300224048, "avg_output_tokens_per_task": 24342.80283793876, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.7411821798483208, "accuracy_std_dev": 0.01874609358231574, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.003938289635247381, "avg_duration_per_task_ms": 186658.11412062118, "avg_output_tokens_per_task": 6185.643553629469, "last_run_timestamp": "2026-08-29T18:12:05.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.7398376524390244, "accuracy_std_dev": 0.030475562908008763, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.004495964430894309, "avg_duration_per_task_ms": 781618.3089430894, "avg_output_tokens_per_task": 27377.12906504065, "last_run_timestamp": "2026-08-29T18:16:54.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.7393435345268542, "accuracy_std_dev": 0.04818327390886166, "run_count": 12, "total_questions": 2346, "avg_cost_per_task_usd": 0.008884832054560953, "avg_duration_per_task_ms": 737670.1547314578, "avg_output_tokens_per_task": 22523.651321398123, "last_run_timestamp": "2026-08-22T10:02:15.507Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Groq", "accuracy": 0.7389277692307694, "accuracy_std_dev": 0.024649382442424383, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.009822144133644132, "avg_duration_per_task_ms": 14839.177544677545, "avg_output_tokens_per_task": 3146.2645687645686, "last_run_timestamp": "2026-08-01T17:45:49.025Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7388467499999999, "accuracy_std_dev": 0.07747406520692911, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01745328409090909, "avg_duration_per_task_ms": 2275674.0732323234, "avg_output_tokens_per_task": 17131.02398989899, "last_run_timestamp": "2026-08-22T18:24:57.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.7384959999999999, "accuracy_std_dev": 0.0076941959727230055, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.058646202861952865, "avg_duration_per_task_ms": 123198.74579124579, "avg_output_tokens_per_task": 23489.968855218856, "last_run_timestamp": "2026-09-01T17:53:51.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.7382653844545136, "accuracy_std_dev": 0.017460726869285965, "run_count": 29, "total_questions": 5738, "avg_cost_per_task_usd": 0.005406186301847333, "avg_duration_per_task_ms": 69408.7223771349, "avg_output_tokens_per_task": 5668.429766469153, "last_run_timestamp": "2026-08-29T19:15:44.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.7378226666666667, "accuracy_std_dev": 0.013319255277813231, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.01630272525252525, "avg_duration_per_task_ms": 70473.10673400674, "avg_output_tokens_per_task": 10789.047138047137, "last_run_timestamp": "2026-08-29T18:28:10.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Nebius", "accuracy": 0.7373736666666667, "accuracy_std_dev": 0.18223849985542448, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04463949831649832, "avg_duration_per_task_ms": 228965.81313131313, "avg_output_tokens_per_task": 14731.36026936027, "last_run_timestamp": "2026-06-30T21:13:44.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.736111, "accuracy_std_dev": null, "run_count": 1, "total_questions": 168, "avg_cost_per_task_usd": 0.05987852380952381, "avg_duration_per_task_ms": 1235915.3511904762, "avg_output_tokens_per_task": 62102.07738095238, "last_run_timestamp": "2026-07-18T08:29:05.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7358002289156627, "accuracy_std_dev": 0.04992948859920135, "run_count": 9, "total_questions": 1743, "avg_cost_per_task_usd": 0.0068313775100401625, "avg_duration_per_task_ms": 1022403.1669535284, "avg_output_tokens_per_task": 18090.09982788296, "last_run_timestamp": "2026-08-29T20:11:32.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7355811297709923, "accuracy_std_dev": 0.10861849313307564, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.00556993893129771, "avg_duration_per_task_ms": 485638.58396946563, "avg_output_tokens_per_task": 24035.00954198473, "last_run_timestamp": "2026-09-26T09:37:20.718Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.735337781737194, "accuracy_std_dev": 0.017059479919011102, "run_count": 14, "total_questions": 2694, "avg_cost_per_task_usd": 0.0013270267260579063, "avg_duration_per_task_ms": 313987.4647364514, "avg_output_tokens_per_task": 7917.988864142539, "last_run_timestamp": "2026-07-18T16:27:55.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.7352940512333966, "accuracy_std_dev": 0.018881487682631787, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.0033387254901960785, "avg_duration_per_task_ms": 137055.96173308033, "avg_output_tokens_per_task": 5667.310562934851, "last_run_timestamp": "2026-08-29T18:16:39.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Fireworks", "accuracy": 0.7349234689468948, "accuracy_std_dev": 0.22406624102384445, "run_count": 6, "total_questions": 1111, "avg_cost_per_task_usd": 0.07603830513051305, "avg_duration_per_task_ms": 255861.0288028803, "avg_output_tokens_per_task": 17676.750675067506, "last_run_timestamp": "2026-07-18T08:32:17.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.7333755, "accuracy_std_dev": 0.033868640415951154, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.005042782196969697, "avg_duration_per_task_ms": 94336.22032828283, "avg_output_tokens_per_task": 5601.336805555556, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7330631323430115, "accuracy_std_dev": 0.07952342617106217, "run_count": 8, "total_questions": 1481, "avg_cost_per_task_usd": 0.03091582241728561, "avg_duration_per_task_ms": 150438.8203916273, "avg_output_tokens_per_task": 27968.933153274815, "last_run_timestamp": "2026-08-29T19:16:48.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "AtlasCloud", "accuracy": 0.7329965999999999, "accuracy_std_dev": 0.010941396457491143, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.007063433333333333, "avg_duration_per_task_ms": 42635.63636363636, "avg_output_tokens_per_task": 5195.6535353535355, "last_run_timestamp": "2026-07-01T02:32:36.197Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.7315366610778443, "accuracy_std_dev": 0.23240685150709303, "run_count": 5, "total_questions": 835, "avg_cost_per_task_usd": 0.05666666227544911, "avg_duration_per_task_ms": 351949.8934131737, "avg_output_tokens_per_task": 18214.095808383234, "last_run_timestamp": "2026-08-15T08:47:51.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7312871538461537, "accuracy_std_dev": 0.012238898554301565, "run_count": 26, "total_questions": 5148, "avg_cost_per_task_usd": 0.20442489957264962, "avg_duration_per_task_ms": 242818.47999223, "avg_output_tokens_per_task": 38189.194832944835, "last_run_timestamp": "2026-09-26T08:58:38.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7299561400595999, "accuracy_std_dev": 0.20101017137136507, "run_count": 13, "total_questions": 2349, "avg_cost_per_task_usd": 0.004135057471264368, "avg_duration_per_task_ms": 932978.9770114942, "avg_output_tokens_per_task": 27248.059174116646, "last_run_timestamp": "2026-09-26T21:21:20.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.7297043333333333, "accuracy_std_dev": 0.017207018593843885, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1851553911335578, "avg_duration_per_task_ms": 289835.51515151514, "avg_output_tokens_per_task": 36869.942199775534, "last_run_timestamp": "2026-09-26T09:21:00.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "WandB", "accuracy": 0.7296560582226762, "accuracy_std_dev": 0.15560230205957754, "run_count": 5, "total_questions": 979, "avg_cost_per_task_usd": 0.01869288764044944, "avg_duration_per_task_ms": 96519.51685393258, "avg_output_tokens_per_task": 14860.048008171603, "last_run_timestamp": "2026-07-18T16:00:08.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.729592, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.0030441326530612246, "avg_duration_per_task_ms": 200954.75, "avg_output_tokens_per_task": 31361.45408163265, "last_run_timestamp": "2026-09-01T13:20:55.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.727459888888889, "accuracy_std_dev": 0.010021023926885385, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1903540600448934, "avg_duration_per_task_ms": 241489.60381593715, "avg_output_tokens_per_task": 37880.08529741863, "last_run_timestamp": "2026-09-26T08:54:32.344Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7272725, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.058774820707070706, "avg_duration_per_task_ms": 121424.5, "avg_output_tokens_per_task": 23508.14898989899, "last_run_timestamp": "2026-09-01T17:49:53.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7263065279007972, "accuracy_std_dev": 0.1217642983358514, "run_count": 6, "total_questions": 1129, "avg_cost_per_task_usd": 0.2174638077945084, "avg_duration_per_task_ms": 1661844.5757307352, "avg_output_tokens_per_task": 59283.423383525245, "last_run_timestamp": "2026-08-22T09:29:12.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.7260814529667148, "accuracy_std_dev": 0.020878527218346984, "run_count": 21, "total_questions": 4146, "avg_cost_per_task_usd": 0.004243377713458756, "avg_duration_per_task_ms": 227026.06222865413, "avg_output_tokens_per_task": 16857.46454413893, "last_run_timestamp": "2026-08-29T18:36:32.803Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.7260439146496813, "accuracy_std_dev": 0.007747089236538362, "run_count": 12, "total_questions": 2355, "avg_cost_per_task_usd": 0.0025452929936305734, "avg_duration_per_task_ms": 422788.46751592355, "avg_output_tokens_per_task": 6748.217409766455, "last_run_timestamp": "2026-08-29T18:39:40.767Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7258483846153846, "accuracy_std_dev": 0.07587921757183326, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.11719855205905207, "avg_duration_per_task_ms": 909372.0038850039, "avg_output_tokens_per_task": 29889.70202020202, "last_run_timestamp": "2026-08-29T19:12:26.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7245957898383372, "accuracy_std_dev": 0.009250125041530539, "run_count": 9, "total_questions": 1732, "avg_cost_per_task_usd": 0.1865489249422633, "avg_duration_per_task_ms": 350199.4688221709, "avg_output_tokens_per_task": 37119.776558891455, "last_run_timestamp": "2026-09-26T08:53:09.141Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.7230691676829271, "accuracy_std_dev": 0.01328152332415223, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.005270103150406503, "avg_duration_per_task_ms": 358290.0543699187, "avg_output_tokens_per_task": 7038.59349593496, "last_run_timestamp": "2026-08-22T16:50:17.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7219416666666667, "accuracy_std_dev": 0.05789083119880804, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.011145885521885522, "avg_duration_per_task_ms": 86502.09595959596, "avg_output_tokens_per_task": 4478.992424242424, "last_run_timestamp": "2026-09-26T09:45:38.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.7219408075949366, "accuracy_std_dev": 0.025598999999999983, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.27939689620253166, "avg_duration_per_task_ms": 450249.90379746835, "avg_output_tokens_per_task": 45941.00253164557, "last_run_timestamp": "2026-08-15T08:16:15.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.7204764444444446, "accuracy_std_dev": 0.010281167113216102, "run_count": 27, "total_questions": 5346, "avg_cost_per_task_usd": 0.1972407962962963, "avg_duration_per_task_ms": 261625.21979049756, "avg_output_tokens_per_task": 36783.956041900485, "last_run_timestamp": "2026-09-26T09:21:19.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Phala", "accuracy": 0.7198153664772725, "accuracy_std_dev": 0.16422235834886253, "run_count": 15, "total_questions": 2816, "avg_cost_per_task_usd": 0.008276323508522728, "avg_duration_per_task_ms": 230801.09623579544, "avg_output_tokens_per_task": 6590.78515625, "last_run_timestamp": "2026-07-18T16:47:35.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "DeepInfra", "accuracy": 0.7196582307692306, "accuracy_std_dev": 0.022147329696075465, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.03072831965811966, "avg_duration_per_task_ms": 160744.22905982906, "avg_output_tokens_per_task": 13887.176068376068, "last_run_timestamp": "2026-06-30T21:06:56.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.7184962054677487, "accuracy_std_dev": 0.2646528685755041, "run_count": 12, "total_questions": 2341, "avg_cost_per_task_usd": 0.06396244126441693, "avg_duration_per_task_ms": 525060.6198205895, "avg_output_tokens_per_task": 18294.98761213157, "last_run_timestamp": "2026-08-29T18:36:08.021Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7184345000000002, "accuracy_std_dev": 0.00575544305418794, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006676434343434343, "avg_duration_per_task_ms": 1198975.5896464647, "avg_output_tokens_per_task": 54074.54671717172, "last_run_timestamp": "2026-09-26T17:35:27.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7183463229974161, "accuracy_std_dev": 0.029264511720208542, "run_count": 6, "total_questions": 1161, "avg_cost_per_task_usd": 0.17074779758828595, "avg_duration_per_task_ms": 1524939.6339362618, "avg_output_tokens_per_task": 41593.74849267872, "last_run_timestamp": "2026-08-22T16:49:30.177Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7181841912777354, "accuracy_std_dev": 0.06555394037846128, "run_count": 7, "total_questions": 1307, "avg_cost_per_task_usd": 0.060895005355776584, "avg_duration_per_task_ms": 455243.765876052, "avg_output_tokens_per_task": 23693.538638102524, "last_run_timestamp": "2026-08-15T09:29:41.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7173431778597786, "accuracy_std_dev": 0.1289599653571277, "run_count": 21, "total_questions": 4065, "avg_cost_per_task_usd": 0.07600008265682658, "avg_duration_per_task_ms": 666511.178597786, "avg_output_tokens_per_task": 41226.01648216482, "last_run_timestamp": "2026-08-29T19:10:33.460Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7170990662824207, "accuracy_std_dev": 0.10692249999999998, "run_count": 2, "total_questions": 347, "avg_cost_per_task_usd": 0.005610605187319885, "avg_duration_per_task_ms": 269295.4322766571, "avg_output_tokens_per_task": 27309.671469740635, "last_run_timestamp": "2026-09-01T13:46:27.106Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.7157249440677965, "accuracy_std_dev": 0.01597995721469033, "run_count": 18, "total_questions": 3540, "avg_cost_per_task_usd": 0.005597196045197738, "avg_duration_per_task_ms": 49724.024576271186, "avg_output_tokens_per_task": 7110.428813559322, "last_run_timestamp": "2026-08-29T18:12:18.859Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.7142573873954601, "accuracy_std_dev": 0.01848301628382384, "run_count": 17, "total_questions": 3348, "avg_cost_per_task_usd": 0.007225585424133809, "avg_duration_per_task_ms": 67629.55943847072, "avg_output_tokens_per_task": 6617.06541218638, "last_run_timestamp": "2026-08-29T19:10:03.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.7142555025380709, "accuracy_std_dev": 0.033662331581457615, "run_count": 4, "total_questions": 788, "avg_cost_per_task_usd": 0.0112786192893401, "avg_duration_per_task_ms": 562577.1395939087, "avg_output_tokens_per_task": 56130.946700507615, "last_run_timestamp": "2026-09-26T09:18:34.055Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.7136277774106337, "accuracy_std_dev": 0.03414979758617127, "run_count": 17, "total_questions": 3329, "avg_cost_per_task_usd": 0.0037028555121658147, "avg_duration_per_task_ms": 233018.78792430158, "avg_output_tokens_per_task": 9785.001501952538, "last_run_timestamp": "2026-08-29T19:25:16.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.7117850996055227, "accuracy_std_dev": 0.05549170236951472, "run_count": 31, "total_questions": 6084, "avg_cost_per_task_usd": 0.005924440664036818, "avg_duration_per_task_ms": 329601.12508218276, "avg_output_tokens_per_task": 18238.3634122288, "last_run_timestamp": "2026-08-29T19:12:34.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7117845333333335, "accuracy_std_dev": 0.015271985026475404, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.008412858585858586, "avg_duration_per_task_ms": 147396.4053872054, "avg_output_tokens_per_task": 4380.246127946128, "last_run_timestamp": "2026-08-29T18:30:44.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.7114771734605377, "accuracy_std_dev": 0.08659653364462867, "run_count": 7, "total_questions": 1153, "avg_cost_per_task_usd": 0.007400922810060712, "avg_duration_per_task_ms": 492060.8178664354, "avg_output_tokens_per_task": 32850.75888985256, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.7112795, "accuracy_std_dev": 0.016430227680254466, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.006384943181818182, "avg_duration_per_task_ms": 104150.93529040404, "avg_output_tokens_per_task": 4141.536616161616, "last_run_timestamp": "2026-08-29T18:10:22.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.7108986701231368, "accuracy_std_dev": 0.06666024694366084, "run_count": 16, "total_questions": 3086, "avg_cost_per_task_usd": 0.005044747245625405, "avg_duration_per_task_ms": 89902.54601425794, "avg_output_tokens_per_task": 6253.540181464679, "last_run_timestamp": "2026-08-08T16:46:11.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.7102692, "accuracy_std_dev": 0.01356127233558859, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.006768369696969696, "avg_duration_per_task_ms": 38096.08383838384, "avg_output_tokens_per_task": 4479.758080808081, "last_run_timestamp": "2026-06-30T19:34:58.612Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.709930669222344, "accuracy_std_dev": 0.17059505917300186, "run_count": 5, "total_questions": 913, "avg_cost_per_task_usd": 0.05822560240963856, "avg_duration_per_task_ms": 183232.56188389924, "avg_output_tokens_per_task": 16113.98247535597, "last_run_timestamp": "2026-07-18T08:37:00.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.70942765, "accuracy_std_dev": 0.01578725575986846, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.0038632861111111117, "avg_duration_per_task_ms": 70598.0202020202, "avg_output_tokens_per_task": 3586.676515151515, "last_run_timestamp": "2026-08-29T19:09:55.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7088045025693731, "accuracy_std_dev": 0.022963109072306993, "run_count": 15, "total_questions": 2919, "avg_cost_per_task_usd": 0.011898455292908531, "avg_duration_per_task_ms": 66441.3932853717, "avg_output_tokens_per_task": 7873.553956834532, "last_run_timestamp": "2026-08-29T19:12:26.066Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.708754, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01329097474747475, "avg_duration_per_task_ms": 223430.93434343435, "avg_output_tokens_per_task": 33132.76262626263, "last_run_timestamp": "2026-09-01T18:00:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.7084551708346867, "accuracy_std_dev": 0.05060517580742111, "run_count": 16, "total_questions": 3079, "avg_cost_per_task_usd": 0.00587215816823644, "avg_duration_per_task_ms": 124679.61123741475, "avg_output_tokens_per_task": 7701.0337772003895, "last_run_timestamp": "2026-08-29T18:11:53.163Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.7083467052631579, "accuracy_std_dev": 0.019685740692424424, "run_count": 16, "total_questions": 3135, "avg_cost_per_task_usd": 0.0076249078149920255, "avg_duration_per_task_ms": 77954.19457735248, "avg_output_tokens_per_task": 7319.329505582137, "last_run_timestamp": "2026-08-29T18:42:28.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Ambient", "accuracy": 0.707071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0038913585858585857, "avg_duration_per_task_ms": 62517.207070707074, "avg_output_tokens_per_task": 3515.439393939394, "last_run_timestamp": "2026-06-13T08:06:00.319Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.7065186345004272, "accuracy_std_dev": 0.01590026262623776, "run_count": 18, "total_questions": 3513, "avg_cost_per_task_usd": 0.007040196982635925, "avg_duration_per_task_ms": 99879.27754056362, "avg_output_tokens_per_task": 6356.3692001138625, "last_run_timestamp": "2026-08-29T18:09:55.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7052560216881594, "accuracy_std_dev": 0.14220511739766134, "run_count": 11, "total_questions": 1706, "avg_cost_per_task_usd": 0.04422351465416178, "avg_duration_per_task_ms": 329497.4015240328, "avg_output_tokens_per_task": 12749.240914419695, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.7050869311010947, "accuracy_std_dev": 0.036824859327421404, "run_count": 8, "total_questions": 1553, "avg_cost_per_task_usd": 0.01090031036703155, "avg_duration_per_task_ms": 296639.77012234385, "avg_output_tokens_per_task": 47139.87057308435, "last_run_timestamp": "2026-09-26T08:54:38.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.7047773804347826, "accuracy_std_dev": 0.10309159308121915, "run_count": 13, "total_questions": 2484, "avg_cost_per_task_usd": 0.10197293719806763, "avg_duration_per_task_ms": 616168.0583735909, "avg_output_tokens_per_task": 39185.06280193237, "last_run_timestamp": "2026-08-29T16:33:36.680Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.7037366126080326, "accuracy_std_dev": 0.08290095218562599, "run_count": 21, "total_questions": 3934, "avg_cost_per_task_usd": 0.006630117946110829, "avg_duration_per_task_ms": 234021.50737163192, "avg_output_tokens_per_task": 12083.909506863243, "last_run_timestamp": "2026-08-29T19:16:51.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.70328275, "accuracy_std_dev": 0.02153875190644758, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0019388005050505052, "avg_duration_per_task_ms": 636092.8409090909, "avg_output_tokens_per_task": 14000.35101010101, "last_run_timestamp": "2026-09-05T08:28:07.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.7026935333333333, "accuracy_std_dev": 0.2370147256769409, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.015571158585858585, "avg_duration_per_task_ms": 188449.88585858585, "avg_output_tokens_per_task": 15387.030976430977, "last_run_timestamp": "2026-08-15T15:24:35.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7024411666666667, "accuracy_std_dev": 0.14218853509503576, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.013173934343434343, "avg_duration_per_task_ms": 786438.8409090909, "avg_output_tokens_per_task": 27569.127104377105, "last_run_timestamp": "2026-09-22T02:20:03.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.7023595287107259, "accuracy_std_dev": 0.038764278307667294, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.009986093535572407, "avg_duration_per_task_ms": 174032.60635608522, "avg_output_tokens_per_task": 5593.384615384615, "last_run_timestamp": "2026-08-29T19:24:48.378Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.7020202666666666, "accuracy_std_dev": 0.017035677274342683, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.006572137373737374, "avg_duration_per_task_ms": 177727.7595959596, "avg_output_tokens_per_task": 4611.837373737374, "last_run_timestamp": "2026-08-29T18:14:24.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.6989537142857144, "accuracy_std_dev": 0.013756650596444363, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.0019099855699855697, "avg_duration_per_task_ms": 141976.2987012987, "avg_output_tokens_per_task": 4153.015151515152, "last_run_timestamp": "2026-08-29T16:33:36.891Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Nebius", "accuracy": 0.698653, "accuracy_std_dev": 0.25478202686610374, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.07541588215488215, "avg_duration_per_task_ms": 218316.69360269362, "avg_output_tokens_per_task": 18749.255892255893, "last_run_timestamp": "2026-07-11T08:07:17.820Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Io Net", "accuracy": 0.6969694999999999, "accuracy_std_dev": 0.02525250000000001, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008116111111111111, "avg_duration_per_task_ms": 29037.578282828283, "avg_output_tokens_per_task": 1510.050505050505, "last_run_timestamp": "2026-05-09T16:14:50.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.6961280000000001, "accuracy_std_dev": 0.01094299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004348992424242424, "avg_duration_per_task_ms": 356365.1767676768, "avg_output_tokens_per_task": 12056.626262626263, "last_run_timestamp": "2026-07-18T08:44:31.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.6953852941176472, "accuracy_std_dev": 0.04336719956205002, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.0073056770647653025, "avg_duration_per_task_ms": 36604.64824717766, "avg_output_tokens_per_task": 6041.8909685086155, "last_run_timestamp": "2026-08-29T18:10:09.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6948820649606299, "accuracy_std_dev": 0.027729042227588305, "run_count": 8, "total_questions": 1524, "avg_cost_per_task_usd": 0.015564356299212597, "avg_duration_per_task_ms": 765563.3963254594, "avg_output_tokens_per_task": 77634.76837270342, "last_run_timestamp": "2026-09-26T16:34:21.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Friendli", "accuracy": 0.69307675, "accuracy_std_dev": 0.04288574538162418, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.004385503156565657, "avg_duration_per_task_ms": 188413.82133838383, "avg_output_tokens_per_task": 5338.457702020202, "last_run_timestamp": "2026-08-01T17:52:16.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.6925813109756097, "accuracy_std_dev": 0.24072562924993626, "run_count": 9, "total_questions": 1312, "avg_cost_per_task_usd": 0.009318611280487804, "avg_duration_per_task_ms": 54896.76600609756, "avg_output_tokens_per_task": 9351.999237804877, "last_run_timestamp": "2026-08-29T18:12:40.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.6905726, "accuracy_std_dev": 0.11069382289468553, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.012376834343434341, "avg_duration_per_task_ms": 236253.0919191919, "avg_output_tokens_per_task": 10174.568686868686, "last_run_timestamp": "2026-08-15T15:26:03.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.6901446339066337, "accuracy_std_dev": 0.20455115646309735, "run_count": 13, "total_questions": 2442, "avg_cost_per_task_usd": 0.042357015970515964, "avg_duration_per_task_ms": 640141.3783783783, "avg_output_tokens_per_task": 14232.427927927927, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6898169582355701, "accuracy_std_dev": 0.20219361577785366, "run_count": 11, "total_questions": 2131, "avg_cost_per_task_usd": 0.015223487095260441, "avg_duration_per_task_ms": 380102.5335523228, "avg_output_tokens_per_task": 18493.574378226185, "last_run_timestamp": "2026-09-26T08:53:03.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.6896746666666668, "accuracy_std_dev": 0.04960523518294776, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04329939057239058, "avg_duration_per_task_ms": 1345214.5555555555, "avg_output_tokens_per_task": 31899.07744107744, "last_run_timestamp": "2026-08-22T16:20:35.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.6885525, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013761295454545455, "avg_duration_per_task_ms": 259532.94696969696, "avg_output_tokens_per_task": 32681.065656565657, "last_run_timestamp": "2026-09-01T18:15:18.821Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6863611111111112, "accuracy_std_dev": 0.022997260124245744, "run_count": 6, "total_questions": 1134, "avg_cost_per_task_usd": 0.010325405643738976, "avg_duration_per_task_ms": 361986.35802469135, "avg_output_tokens_per_task": 46144.19135802469, "last_run_timestamp": "2026-09-26T08:53:22.738Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Z.AI", "accuracy": 0.6858586, "accuracy_std_dev": 0.014069007564146077, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006483653535353535, "avg_duration_per_task_ms": 266458.26868686866, "avg_output_tokens_per_task": 16169.268686868687, "last_run_timestamp": "2026-05-23T17:26:25.014Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403:free", "model_slug": null, "model_name": "Gemma 4 26B A4B IT (free)", "provider_name": "Darkbloom", "accuracy": 0.684722, "accuracy_std_dev": null, "run_count": 1, "total_questions": 120, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 382535.98333333334, "avg_output_tokens_per_task": 6994.291666666667, "last_run_timestamp": "2026-08-04T05:36:23.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Ambient", "accuracy": 0.6843435, "accuracy_std_dev": 0.017676499999999984, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0009698787878787878, "avg_duration_per_task_ms": 30045.00505050505, "avg_output_tokens_per_task": 1560.3434343434344, "last_run_timestamp": "2026-05-23T16:14:51.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.6804360395869191, "accuracy_std_dev": 0.04000822543143897, "run_count": 9, "total_questions": 1743, "avg_cost_per_task_usd": 0.015301245553643147, "avg_duration_per_task_ms": 489170.94836488814, "avg_output_tokens_per_task": 18010.746987951807, "last_run_timestamp": "2026-08-01T17:59:38.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Ambient", "accuracy": 0.6792930000000001, "accuracy_std_dev": 0.07828270967780816, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.024729667929292928, "avg_duration_per_task_ms": 202686.41414141413, "avg_output_tokens_per_task": 6075.666666666667, "last_run_timestamp": "2026-06-13T09:16:40.692Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.678451, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0025699040404040404, "avg_duration_per_task_ms": 57532.25252525252, "avg_output_tokens_per_task": 2314.656565656566, "last_run_timestamp": "2026-09-01T13:44:51.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.678451, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01359871212121212, "avg_duration_per_task_ms": 222801.71212121213, "avg_output_tokens_per_task": 33900.580808080806, "last_run_timestamp": "2026-09-01T18:09:36.992Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.6765503619112165, "accuracy_std_dev": 0.12971631760570212, "run_count": 17, "total_questions": 2951, "avg_cost_per_task_usd": 0.025736548288715685, "avg_duration_per_task_ms": 193193.36191121655, "avg_output_tokens_per_task": 9449.462555066078, "last_run_timestamp": "2026-08-29T19:20:11.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.676548715835141, "accuracy_std_dev": 0.020437457580933318, "run_count": 14, "total_questions": 2766, "avg_cost_per_task_usd": 0.005609137382501808, "avg_duration_per_task_ms": 246410.7856109906, "avg_output_tokens_per_task": 35921.823571945046, "last_run_timestamp": "2026-07-18T17:04:25.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6750842, "accuracy_std_dev": 0.023641210809939493, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005652539393939394, "avg_duration_per_task_ms": 500294.62828282826, "avg_output_tokens_per_task": 16033.422222222222, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "Novita", "accuracy": 0.6747475999999999, "accuracy_std_dev": 0.013775898004848904, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0017936818181818182, "avg_duration_per_task_ms": 43213.43131313131, "avg_output_tokens_per_task": 5783.8050505050505, "last_run_timestamp": "2026-05-30T08:06:07.683Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.6742425, "accuracy_std_dev": 0.2011785, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03395365151515151, "avg_duration_per_task_ms": 248299.7626262626, "avg_output_tokens_per_task": 37528.89898989899, "last_run_timestamp": "2026-09-26T11:57:25.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.6741976266666667, "accuracy_std_dev": 0.04245737107063498, "run_count": 14, "total_questions": 2700, "avg_cost_per_task_usd": 0.01499391444444444, "avg_duration_per_task_ms": 350545.94703703705, "avg_output_tokens_per_task": 17958.764814814815, "last_run_timestamp": "2026-08-01T18:07:49.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.6731558206967214, "accuracy_std_dev": 0.0760641946679835, "run_count": 15, "total_questions": 2928, "avg_cost_per_task_usd": 0.09475717725409835, "avg_duration_per_task_ms": 785024.8715846995, "avg_output_tokens_per_task": 24226.864412568306, "last_run_timestamp": "2026-08-22T17:24:19.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.6731349473684209, "accuracy_std_dev": 0.085633655090479, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.009196745614035088, "avg_duration_per_task_ms": 10451.350345560872, "avg_output_tokens_per_task": 11969.038277511961, "last_run_timestamp": "2026-08-29T18:50:56.950Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.6729346307692308, "accuracy_std_dev": 0.008729468916008341, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.002463391452991453, "avg_duration_per_task_ms": 1210800.4, "avg_output_tokens_per_task": 14547.558974358974, "last_run_timestamp": "2026-07-18T09:58:41.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.6716106314888761, "accuracy_std_dev": 0.014749164820987887, "run_count": 18, "total_questions": 3506, "avg_cost_per_task_usd": 0.005318032515687393, "avg_duration_per_task_ms": 413726.3981745579, "avg_output_tokens_per_task": 37904.13605248146, "last_run_timestamp": "2026-08-29T21:19:16.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.671246097613883, "accuracy_std_dev": 0.027379954626458713, "run_count": 14, "total_questions": 2766, "avg_cost_per_task_usd": 0.0018295003615328993, "avg_duration_per_task_ms": 160145.0853217643, "avg_output_tokens_per_task": 3977.9674620390456, "last_run_timestamp": "2026-08-29T17:07:45.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6708755, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014901020202020204, "avg_duration_per_task_ms": 14005.674242424242, "avg_output_tokens_per_task": 1670.229797979798, "last_run_timestamp": "2026-09-01T13:21:57.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.6708310526315789, "accuracy_std_dev": 0.13484993870048123, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.043008509303561926, "avg_duration_per_task_ms": 55485.30568846358, "avg_output_tokens_per_task": 15637.817916002126, "last_run_timestamp": "2026-08-15T14:46:10.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.6689419784982935, "accuracy_std_dev": 0.1773412026363194, "run_count": 16, "total_questions": 2930, "avg_cost_per_task_usd": 0.0037005590443686008, "avg_duration_per_task_ms": 131354.5737201365, "avg_output_tokens_per_task": 4676.710921501706, "last_run_timestamp": "2026-08-29T19:47:54.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.6685414863824749, "accuracy_std_dev": 0.08727923360547458, "run_count": 19, "total_questions": 3378, "avg_cost_per_task_usd": 0.006985365304914151, "avg_duration_per_task_ms": 34927.65926583777, "avg_output_tokens_per_task": 4604.6335109532265, "last_run_timestamp": "2026-08-22T17:34:08.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.6658203026315791, "accuracy_std_dev": 0.02735758829038327, "run_count": 22, "total_questions": 4332, "avg_cost_per_task_usd": 0.0065892525392428425, "avg_duration_per_task_ms": 404554.2984764543, "avg_output_tokens_per_task": 35346.29085872576, "last_run_timestamp": "2026-08-29T21:14:10.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.664983, "accuracy_std_dev": 0.045452036199346076, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008639549663299663, "avg_duration_per_task_ms": 254582.55808080808, "avg_output_tokens_per_task": 42899.02777777778, "last_run_timestamp": "2026-09-26T09:38:02.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.6640851398698884, "accuracy_std_dev": 0.02270387404538052, "run_count": 33, "total_questions": 6456, "avg_cost_per_task_usd": 0.006331274473358116, "avg_duration_per_task_ms": 262668.35517348204, "avg_output_tokens_per_task": 41968.7614622057, "last_run_timestamp": "2026-08-29T22:02:09.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.661616, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003362974747474748, "avg_duration_per_task_ms": 65871.15656565657, "avg_output_tokens_per_task": 2362.121212121212, "last_run_timestamp": "2026-08-01T09:30:53.884Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.6615472218430034, "accuracy_std_dev": 0.011144726307502387, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.002312742320819112, "avg_duration_per_task_ms": 194982.48521046643, "avg_output_tokens_per_task": 3098.169510807736, "last_run_timestamp": "2026-08-29T18:47:21.578Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6603752184024266, "accuracy_std_dev": 0.09735619534024986, "run_count": 15, "total_questions": 2967, "avg_cost_per_task_usd": 0.0037568466464442197, "avg_duration_per_task_ms": 226109.69632625548, "avg_output_tokens_per_task": 4143.123019885406, "last_run_timestamp": "2026-08-29T19:11:19.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.6592700368271954, "accuracy_std_dev": 0.049300236020255434, "run_count": 11, "total_questions": 2118, "avg_cost_per_task_usd": 0.0190812327667611, "avg_duration_per_task_ms": 403168.8569405099, "avg_output_tokens_per_task": 18089.26912181303, "last_run_timestamp": "2026-08-01T18:05:37.080Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "NextBit", "accuracy": 0.6578283333333335, "accuracy_std_dev": 0.02342969883265444, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0015064617003367007, "avg_duration_per_task_ms": 42020.857323232325, "avg_output_tokens_per_task": 3276.22095959596, "last_run_timestamp": "2026-06-20T16:19:52.738Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.6575277142857143, "accuracy_std_dev": 0.021158523643569998, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005534338383838384, "avg_duration_per_task_ms": 453292.816017316, "avg_output_tokens_per_task": 34128.61327561327, "last_run_timestamp": "2026-08-29T22:11:02.681Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6571402410865875, "accuracy_std_dev": 0.017808193342575305, "run_count": 18, "total_questions": 3534, "avg_cost_per_task_usd": 0.023573010186757212, "avg_duration_per_task_ms": 539665.155065082, "avg_output_tokens_per_task": 12662.279852857951, "last_run_timestamp": "2026-08-29T18:31:43.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6569729516129034, "accuracy_std_dev": 0.17122119291573554, "run_count": 21, "total_questions": 4092, "avg_cost_per_task_usd": 0.01374512854349951, "avg_duration_per_task_ms": 244855.71725317693, "avg_output_tokens_per_task": 9153.359237536657, "last_run_timestamp": "2026-08-29T19:03:14.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.6566732189349114, "accuracy_std_dev": 0.013404939721861693, "run_count": 13, "total_questions": 2535, "avg_cost_per_task_usd": 0.002109493885601578, "avg_duration_per_task_ms": 61133.307297830375, "avg_output_tokens_per_task": 2302.72899408284, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.6566194539007094, "accuracy_std_dev": 0.021003345172555248, "run_count": 16, "total_questions": 3102, "avg_cost_per_task_usd": 0.007993988072211475, "avg_duration_per_task_ms": 399361.4635718891, "avg_output_tokens_per_task": 27495.45293359123, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Phala", "accuracy": 0.656566, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0025412272727272727, "avg_duration_per_task_ms": 58356.79797979798, "avg_output_tokens_per_task": 3959.469696969697, "last_run_timestamp": "2026-06-20T08:04:16.013Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.6556188125000002, "accuracy_std_dev": 0.01847027483288604, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.0016670344065656568, "avg_duration_per_task_ms": 63016.68939393939, "avg_output_tokens_per_task": 1545.2768308080808, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.655513375, "accuracy_std_dev": 0.1620210868528673, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0030583200757575758, "avg_duration_per_task_ms": 54577.18244949495, "avg_output_tokens_per_task": 1983.9665404040404, "last_run_timestamp": "2026-07-01T02:34:47.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.653757604130809, "accuracy_std_dev": 0.015316614138466363, "run_count": 3, "total_questions": 581, "avg_cost_per_task_usd": 0.01179434079173838, "avg_duration_per_task_ms": 1792911.3149741825, "avg_output_tokens_per_task": 50132.19449225473, "last_run_timestamp": "2026-08-29T20:13:41.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6536547169042769, "accuracy_std_dev": 0.019981737763823795, "run_count": 15, "total_questions": 2946, "avg_cost_per_task_usd": 0.004769026137135098, "avg_duration_per_task_ms": 99040.27562797013, "avg_output_tokens_per_task": 8986.41310251188, "last_run_timestamp": "2026-09-26T08:51:07.739Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.6519080986717266, "accuracy_std_dev": 0.0160056211434452, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.0014605863377609109, "avg_duration_per_task_ms": 61151.05566097407, "avg_output_tokens_per_task": 4710.759013282733, "last_run_timestamp": "2026-07-18T08:33:31.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.650363823580786, "accuracy_std_dev": 0.007103203700131033, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.0036315545851528385, "avg_duration_per_task_ms": 58503.03493449782, "avg_output_tokens_per_task": 2202.661135371179, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.6500593861209965, "accuracy_std_dev": 0.05876659054376805, "run_count": 6, "total_questions": 1124, "avg_cost_per_task_usd": 0.04040380338078292, "avg_duration_per_task_ms": 414098.88790035585, "avg_output_tokens_per_task": 20106.83896797153, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.6498354401756312, "accuracy_std_dev": 0.019005924168263848, "run_count": 14, "total_questions": 2733, "avg_cost_per_task_usd": 0.006061401024515184, "avg_duration_per_task_ms": 193666.52616172703, "avg_output_tokens_per_task": 9966.588730332967, "last_run_timestamp": "2026-09-26T08:55:02.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.6498317272727273, "accuracy_std_dev": 0.020163709926908996, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0017987033976124885, "avg_duration_per_task_ms": 59537.23829201102, "avg_output_tokens_per_task": 2292.5165289256197, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6489900000000001, "accuracy_std_dev": 0.01599299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.015694313131313132, "avg_duration_per_task_ms": 13831.43686868687, "avg_output_tokens_per_task": 1769.391414141414, "last_run_timestamp": "2026-09-01T13:20:53.165Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.6477275, "accuracy_std_dev": 0.015583769481418798, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01589866666666667, "avg_duration_per_task_ms": 19920.939393939392, "avg_output_tokens_per_task": 1699.0972222222222, "last_run_timestamp": "2026-09-01T13:27:17.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.6475160921501707, "accuracy_std_dev": 0.010423132030888464, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.002509189419795222, "avg_duration_per_task_ms": 70158.00284414107, "avg_output_tokens_per_task": 2711.532423208191, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "AtlasCloud", "accuracy": 0.6468015, "accuracy_std_dev": 0.02305589841342123, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0008760272727272729, "avg_duration_per_task_ms": 28080.331818181818, "avg_output_tokens_per_task": 923.8237373737373, "last_run_timestamp": "2026-07-01T02:41:54.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.646465, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0029047154882154884, "avg_duration_per_task_ms": 53981.06734006734, "avg_output_tokens_per_task": 2363.939393939394, "last_run_timestamp": "2026-08-29T16:33:11.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.6459677395788569, "accuracy_std_dev": 0.1480303961068657, "run_count": 12, "total_questions": 2327, "avg_cost_per_task_usd": 0.028217204555221313, "avg_duration_per_task_ms": 1621170.644177052, "avg_output_tokens_per_task": 20034.05242801891, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6450775350966429, "accuracy_std_dev": 0.07933459046326091, "run_count": 15, "total_questions": 2949, "avg_cost_per_task_usd": 0.006724317734825365, "avg_duration_per_task_ms": 642083.2651746355, "avg_output_tokens_per_task": 19917.21973550356, "last_run_timestamp": "2026-08-29T19:15:54.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.6447810909090911, "accuracy_std_dev": 0.013751933189420343, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.00262662580348944, "avg_duration_per_task_ms": 31615.61432506887, "avg_output_tokens_per_task": 2488.2291092745636, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.6440172000000001, "accuracy_std_dev": 0.016796480873153823, "run_count": 12, "total_questions": 2340, "avg_cost_per_task_usd": 0.0037662337606837607, "avg_duration_per_task_ms": 61315.614102564105, "avg_output_tokens_per_task": 7079.364957264957, "last_run_timestamp": "2026-08-29T18:43:21.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Ambient", "accuracy": 0.6439395, "accuracy_std_dev": 0.007575499999999957, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00040658080808080813, "avg_duration_per_task_ms": 74334.95202020202, "avg_output_tokens_per_task": 2837.530303030303, "last_run_timestamp": "2026-06-13T16:33:02.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Io Net", "accuracy": 0.6425364444444446, "accuracy_std_dev": 0.04264131881197732, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03682436419753086, "avg_duration_per_task_ms": 255794.32547699215, "avg_output_tokens_per_task": 12516.43265993266, "last_run_timestamp": "2026-06-20T09:54:07.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.6414145, "accuracy_std_dev": 0.001683499999999949, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004025537878787879, "avg_duration_per_task_ms": 31194.16414141414, "avg_output_tokens_per_task": 2323.563131313131, "last_run_timestamp": "2026-09-01T17:49:44.421Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.641414, "accuracy_std_dev": 0.003367000000000009, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004276166666666667, "avg_duration_per_task_ms": 30535.468434343435, "avg_output_tokens_per_task": 2347.165404040404, "last_run_timestamp": "2026-09-01T17:50:18.482Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.6408530000000001, "accuracy_std_dev": 0.00731669260435796, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00720101430976431, "avg_duration_per_task_ms": 742062.9217171717, "avg_output_tokens_per_task": 55987.864478114476, "last_run_timestamp": "2026-08-29T20:02:26.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6407338178368123, "accuracy_std_dev": 0.01094255224010119, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.0017736578115117015, "avg_duration_per_task_ms": 97733.33301707779, "avg_output_tokens_per_task": 5732.58697027198, "last_run_timestamp": "2026-08-29T18:23:33.062Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.6405006706484643, "accuracy_std_dev": 0.017184977086752887, "run_count": 18, "total_questions": 3516, "avg_cost_per_task_usd": 0.0015951729237770197, "avg_duration_per_task_ms": 87221.02275312855, "avg_output_tokens_per_task": 5581.319397042093, "last_run_timestamp": "2026-08-29T19:32:59.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6397305, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004182681818181818, "avg_duration_per_task_ms": 33306.957070707074, "avg_output_tokens_per_task": 2424.3232323232323, "last_run_timestamp": "2026-09-01T17:50:13.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.6393443457526082, "accuracy_std_dev": 0.041002002642534584, "run_count": 21, "total_questions": 4026, "avg_cost_per_task_usd": 0.01295243964232489, "avg_duration_per_task_ms": 568425.9622454048, "avg_output_tokens_per_task": 33286.69672131148, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.6379928049155147, "accuracy_std_dev": 0.03271804644949137, "run_count": 11, "total_questions": 1953, "avg_cost_per_task_usd": 0.01783217153097798, "avg_duration_per_task_ms": 896687.3640552995, "avg_output_tokens_per_task": 74524.547875064, "last_run_timestamp": "2026-09-26T16:32:03.216Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.6379416006600662, "accuracy_std_dev": 0.01704932395167573, "run_count": 14, "total_questions": 2727, "avg_cost_per_task_usd": 0.0018401910524385766, "avg_duration_per_task_ms": 156874.9156582325, "avg_output_tokens_per_task": 5961.517785111844, "last_run_timestamp": "2026-08-29T18:40:25.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.637803, "accuracy_std_dev": null, "run_count": 1, "total_questions": 179, "avg_cost_per_task_usd": 0.002030703910614525, "avg_duration_per_task_ms": 92260.82681564246, "avg_output_tokens_per_task": 6534.692737430168, "last_run_timestamp": "2026-07-18T09:17:11.788Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.6374862222222222, "accuracy_std_dev": 0.015120312553925432, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.001489671717171717, "avg_duration_per_task_ms": 106578.17789001123, "avg_output_tokens_per_task": 7528.87822671156, "last_run_timestamp": "2026-08-29T19:02:06.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.6370851428571429, "accuracy_std_dev": 0.022815823531228588, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.0012857987012987013, "avg_duration_per_task_ms": 143754.60606060605, "avg_output_tokens_per_task": 7026.642857142857, "last_run_timestamp": "2026-08-29T17:04:59.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6359762020460358, "accuracy_std_dev": 0.023468612837149117, "run_count": 4, "total_questions": 782, "avg_cost_per_task_usd": 0.006740489769820971, "avg_duration_per_task_ms": 885605.4833759591, "avg_output_tokens_per_task": 37192.53964194373, "last_run_timestamp": "2026-09-26T08:26:39.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.6358467809621291, "accuracy_std_dev": 0.02889634778968133, "run_count": 15, "total_questions": 2931, "avg_cost_per_task_usd": 0.006419597407028318, "avg_duration_per_task_ms": 105406.46980552713, "avg_output_tokens_per_task": 8959.577959740704, "last_run_timestamp": "2026-09-26T09:22:12.174Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.6334175, "accuracy_std_dev": 0.007154500000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001171128787878788, "avg_duration_per_task_ms": 257331.09848484848, "avg_output_tokens_per_task": 11528.555555555555, "last_run_timestamp": "2026-08-29T19:38:59.849Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.6333203317702227, "accuracy_std_dev": 0.07516813236760066, "run_count": 13, "total_questions": 2559, "avg_cost_per_task_usd": 0.014011412270418131, "avg_duration_per_task_ms": 375980.625244236, "avg_output_tokens_per_task": 16853.191871824933, "last_run_timestamp": "2026-08-01T18:25:56.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.6327843526405451, "accuracy_std_dev": 0.01928203380236522, "run_count": 9, "total_questions": 1761, "avg_cost_per_task_usd": 0.010741952299829643, "avg_duration_per_task_ms": 472791.5741056218, "avg_output_tokens_per_task": 11222.989210675752, "last_run_timestamp": "2026-09-19T10:52:13.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.6326842164948453, "accuracy_std_dev": 0.02330257323945676, "run_count": 9, "total_questions": 1746, "avg_cost_per_task_usd": 0.0025534261168384882, "avg_duration_per_task_ms": 45796.38602520046, "avg_output_tokens_per_task": 1364.9232531500572, "last_run_timestamp": "2026-07-18T08:26:39.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.6321081114302126, "accuracy_std_dev": 0.011032621401446443, "run_count": 9, "total_questions": 1741, "avg_cost_per_task_usd": 0.017073134979896612, "avg_duration_per_task_ms": 470536.2297530155, "avg_output_tokens_per_task": 85368.48765077542, "last_run_timestamp": "2026-09-26T16:13:51.663Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.6313133333333334, "accuracy_std_dev": 0.010910326968315643, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0011066590909090909, "avg_duration_per_task_ms": 52609.71380471381, "avg_output_tokens_per_task": 3573.563973063973, "last_run_timestamp": "2026-06-30T19:57:58.812Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6308680607902736, "accuracy_std_dev": 0.011456259937693471, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.0034108257345491387, "avg_duration_per_task_ms": 140057.4062816616, "avg_output_tokens_per_task": 11472.941236068895, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Venice", "accuracy": 0.6292411856287424, "accuracy_std_dev": 0.0117815, "run_count": 2, "total_questions": 334, "avg_cost_per_task_usd": 0.0071094161676646716, "avg_duration_per_task_ms": 112258.81736526947, "avg_output_tokens_per_task": 28232.077844311378, "last_run_timestamp": "2026-08-15T08:02:44.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.6286550506072877, "accuracy_std_dev": 0.02141189626560173, "run_count": 15, "total_questions": 2964, "avg_cost_per_task_usd": 0.00387920310391363, "avg_duration_per_task_ms": 404484.17341430497, "avg_output_tokens_per_task": 12837.82354925776, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "AtlasCloud", "accuracy": 0.626984142857143, "accuracy_std_dev": 0.01220184110731506, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.003761468975468975, "avg_duration_per_task_ms": 15627.561327561327, "avg_output_tokens_per_task": 918.3910533910534, "last_run_timestamp": "2026-06-30T19:49:03.931Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.626636888888889, "accuracy_std_dev": 0.010990047714429298, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.00803346857463524, "avg_duration_per_task_ms": 245092.09932659933, "avg_output_tokens_per_task": 19284.18069584736, "last_run_timestamp": "2026-07-18T16:20:00.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Cloudflare", "accuracy": 0.626263, "accuracy_std_dev": 0.021820653931539237, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02660411447811448, "avg_duration_per_task_ms": 192407.0723905724, "avg_output_tokens_per_task": 8826.355218855218, "last_run_timestamp": "2026-05-23T08:06:11.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.6248197142857143, "accuracy_std_dev": 0.11942661157585356, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.6020819559884559, "avg_duration_per_task_ms": 624611.841991342, "avg_output_tokens_per_task": 40107.266955266954, "last_run_timestamp": "2026-09-26T08:54:32.678Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "NextBit", "accuracy": 0.6235919324055667, "accuracy_std_dev": 0.017558878241772012, "run_count": 8, "total_questions": 1509, "avg_cost_per_task_usd": 0.005936661365142478, "avg_duration_per_task_ms": 567045.3445990722, "avg_output_tokens_per_task": 10721.506295559973, "last_run_timestamp": "2026-07-18T08:03:56.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6221339996023858, "accuracy_std_dev": 0.024618022699500797, "run_count": 13, "total_questions": 2515, "avg_cost_per_task_usd": 0.007214242942345925, "avg_duration_per_task_ms": 448969.5920477137, "avg_output_tokens_per_task": 11841.8648111332, "last_run_timestamp": "2026-09-26T09:22:23.002Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "AtlasCloud", "accuracy": 0.621595600867679, "accuracy_std_dev": 0.01951894601537237, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.008186916847433116, "avg_duration_per_task_ms": 220211.67172812726, "avg_output_tokens_per_task": 6891.177151120752, "last_run_timestamp": "2026-07-01T03:52:27.932Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.621418478413069, "accuracy_std_dev": 0.013260113971771842, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.004479952158693115, "avg_duration_per_task_ms": 91742.81602489304, "avg_output_tokens_per_task": 8509.088681446909, "last_run_timestamp": "2026-08-01T17:51:29.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6196523681241185, "accuracy_std_dev": 0.014509665172618149, "run_count": 11, "total_questions": 2127, "avg_cost_per_task_usd": 0.004821758345086977, "avg_duration_per_task_ms": 191599.74424071462, "avg_output_tokens_per_task": 9720.46450399624, "last_run_timestamp": "2026-08-01T18:16:00.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6194785090464547, "accuracy_std_dev": 0.03307467351712581, "run_count": 11, "total_questions": 2045, "avg_cost_per_task_usd": 0.014328027872860635, "avg_duration_per_task_ms": 570163.035207824, "avg_output_tokens_per_task": 69958.57506112469, "last_run_timestamp": "2026-09-26T16:16:58.507Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.6182419367816093, "accuracy_std_dev": 0.020441088895802338, "run_count": 16, "total_questions": 3132, "avg_cost_per_task_usd": 0.0025628135376756065, "avg_duration_per_task_ms": 24595.393358876117, "avg_output_tokens_per_task": 1512.9431673052363, "last_run_timestamp": "2026-08-29T18:49:22.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.6168832857142857, "accuracy_std_dev": 0.0274253947059201, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.009174507215007215, "avg_duration_per_task_ms": 639672.284992785, "avg_output_tokens_per_task": 16259.07142857143, "last_run_timestamp": "2026-07-18T17:46:02.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.6166527916666666, "accuracy_std_dev": 0.12145770743554425, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.6553831144781144, "avg_duration_per_task_ms": 484995.4349747475, "avg_output_tokens_per_task": 40791.21506734007, "last_run_timestamp": "2026-09-26T09:11:13.767Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Venice", "accuracy": 0.616162, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0017577828282828282, "avg_duration_per_task_ms": 47302.35858585859, "avg_output_tokens_per_task": 2472.939393939394, "last_run_timestamp": "2026-05-16T08:06:13.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6145900645161291, "accuracy_std_dev": 0.017021637149063382, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.0051391173308032905, "avg_duration_per_task_ms": 224377.06388361796, "avg_output_tokens_per_task": 12215.357368753954, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.614478375, "accuracy_std_dev": 0.12366286539715297, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.6154596931818181, "avg_duration_per_task_ms": 632046.8207070708, "avg_output_tokens_per_task": 40881.84659090909, "last_run_timestamp": "2026-09-26T09:34:52.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.6139495138993238, "accuracy_std_dev": 0.029024384950147974, "run_count": 8, "total_questions": 1331, "avg_cost_per_task_usd": 0.02793150338091661, "avg_duration_per_task_ms": 583500.4891059354, "avg_output_tokens_per_task": 29316.752817430504, "last_run_timestamp": "2026-09-26T16:19:07.764Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6129581867007673, "accuracy_std_dev": 0.0015394999999999714, "run_count": 2, "total_questions": 391, "avg_cost_per_task_usd": 0.0032793452685421997, "avg_duration_per_task_ms": 346994.51406649617, "avg_output_tokens_per_task": 13256.76726342711, "last_run_timestamp": "2026-09-01T13:34:17.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.6129154305135952, "accuracy_std_dev": 0.04617983854978681, "run_count": 8, "total_questions": 1324, "avg_cost_per_task_usd": 0.002775106495468278, "avg_duration_per_task_ms": 1032830.075528701, "avg_output_tokens_per_task": 25108.58836858006, "last_run_timestamp": "2026-07-18T16:00:09.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.6126859920896506, "accuracy_std_dev": 0.1240297456369234, "run_count": 23, "total_questions": 4551, "avg_cost_per_task_usd": 0.6596944438584925, "avg_duration_per_task_ms": 597097.6884201275, "avg_output_tokens_per_task": 41139.12920237311, "last_run_timestamp": "2026-09-26T09:32:36.461Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6122767692307693, "accuracy_std_dev": 0.02357255235406653, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.004552111888111887, "avg_duration_per_task_ms": 191391.34887334888, "avg_output_tokens_per_task": 9066.714452214452, "last_run_timestamp": "2026-08-01T18:08:49.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.6121935714285716, "accuracy_std_dev": 0.019793131697327618, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.00792913455988456, "avg_duration_per_task_ms": 24216.782467532466, "avg_output_tokens_per_task": 1511.1587301587301, "last_run_timestamp": "2026-08-29T18:32:24.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.6119012322274883, "accuracy_std_dev": 0.019905936820709, "run_count": 13, "total_questions": 2532, "avg_cost_per_task_usd": 0.0026039996050552924, "avg_duration_per_task_ms": 47372.351105845184, "avg_output_tokens_per_task": 1655.2235387045814, "last_run_timestamp": "2026-08-29T19:00:24.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.609428, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0023919898989898988, "avg_duration_per_task_ms": 73719.80808080808, "avg_output_tokens_per_task": 2205.409090909091, "last_run_timestamp": "2026-09-01T13:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.6087965, "accuracy_std_dev": 0.12831918473770787, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.6162571098484848, "avg_duration_per_task_ms": 633013.7184343435, "avg_output_tokens_per_task": 41027.7380050505, "last_run_timestamp": "2026-09-26T09:34:56.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.6086993033591731, "accuracy_std_dev": 0.019442174128425027, "run_count": 10, "total_questions": 1935, "avg_cost_per_task_usd": 0.001751839793281654, "avg_duration_per_task_ms": 39301.46408268734, "avg_output_tokens_per_task": 1636.5943152454781, "last_run_timestamp": "2026-08-29T18:37:17.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Together", "accuracy": 0.608586, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0020087575757575756, "avg_duration_per_task_ms": 10469.585858585859, "avg_output_tokens_per_task": 733.570707070707, "last_run_timestamp": "2026-05-23T08:09:18.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Wafer", "accuracy": 0.6085067890625, "accuracy_std_dev": 0.10062656908435016, "run_count": 4, "total_questions": 768, "avg_cost_per_task_usd": 0.010816417968749999, "avg_duration_per_task_ms": 300613.0859375, "avg_output_tokens_per_task": 26862.471354166668, "last_run_timestamp": "2026-07-18T08:04:42.524Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.607744, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003953611111111112, "avg_duration_per_task_ms": 57224.242424242424, "avg_output_tokens_per_task": 2094.1161616161617, "last_run_timestamp": "2026-08-01T17:44:33.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.6062243505658437, "accuracy_std_dev": 0.14728578802372175, "run_count": 21, "total_questions": 3888, "avg_cost_per_task_usd": 0.008590261574074075, "avg_duration_per_task_ms": 53306.474022633745, "avg_output_tokens_per_task": 28544.59079218107, "last_run_timestamp": "2026-08-29T19:48:18.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.6058966877256319, "accuracy_std_dev": 0.030100999151127752, "run_count": 17, "total_questions": 3324, "avg_cost_per_task_usd": 0.0021504960890493382, "avg_duration_per_task_ms": 36972.96871239471, "avg_output_tokens_per_task": 1435.0361010830325, "last_run_timestamp": "2026-08-29T18:47:46.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6057348861480076, "accuracy_std_dev": 0.07360041928198763, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.004612943074003796, "avg_duration_per_task_ms": 545569.0094876661, "avg_output_tokens_per_task": 18056.067678684376, "last_run_timestamp": "2026-08-22T22:18:10.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.6035355000000001, "accuracy_std_dev": 0.004208499999999948, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.002990320707070707, "avg_duration_per_task_ms": 555382.6085858586, "avg_output_tokens_per_task": 13221.70707070707, "last_run_timestamp": "2026-09-01T13:04:03.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6022078955765048, "accuracy_std_dev": 0.015661301991180567, "run_count": 21, "total_questions": 4137, "avg_cost_per_task_usd": 0.04455390645395214, "avg_duration_per_task_ms": 535138.5900410926, "avg_output_tokens_per_task": 16948.74981870921, "last_run_timestamp": "2026-08-30T00:22:07.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.6020365852036575, "accuracy_std_dev": 0.04030630297634977, "run_count": 13, "total_questions": 2406, "avg_cost_per_task_usd": 0.0031222069825436404, "avg_duration_per_task_ms": 36515.22194513716, "avg_output_tokens_per_task": 1660.556940980881, "last_run_timestamp": "2026-08-29T18:43:09.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.6009468851294903, "accuracy_std_dev": 0.08092483969818855, "run_count": 13, "total_questions": 2394, "avg_cost_per_task_usd": 0.006364984126984126, "avg_duration_per_task_ms": 31444.639097744363, "avg_output_tokens_per_task": 10538.471177944863, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.594276, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.002775611111111111, "avg_duration_per_task_ms": 509237.26767676766, "avg_output_tokens_per_task": 10972.9898989899, "last_run_timestamp": "2026-09-01T13:31:26.819Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.5922561000000001, "accuracy_std_dev": 0.11721268776753647, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.6146375166666666, "avg_duration_per_task_ms": 629521.5186868687, "avg_output_tokens_per_task": 40864.73838383838, "last_run_timestamp": "2026-09-26T09:23:13.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5900675, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017035757575757576, "avg_duration_per_task_ms": 123031.02272727272, "avg_output_tokens_per_task": 42470.74747474748, "last_run_timestamp": "2026-09-01T18:10:49.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.5872781331360947, "accuracy_std_dev": 0.004687499999999956, "run_count": 2, "total_questions": 338, "avg_cost_per_task_usd": 0.09507673668639052, "avg_duration_per_task_ms": 191708.50591715975, "avg_output_tokens_per_task": 21455.86094674556, "last_run_timestamp": "2026-08-01T18:26:20.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.5871213125, "accuracy_std_dev": 0.03216602090078666, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.003158161616161616, "avg_duration_per_task_ms": 217609.65056818182, "avg_output_tokens_per_task": 11118.01957070707, "last_run_timestamp": "2026-08-29T19:16:45.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.577862, "accuracy_std_dev": 0.03661599999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006390335858585858, "avg_duration_per_task_ms": 667369.1111111111, "avg_output_tokens_per_task": 32604.280303030304, "last_run_timestamp": "2026-09-26T08:13:27.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.5774372016509435, "accuracy_std_dev": 0.01609338000361728, "run_count": 13, "total_questions": 2544, "avg_cost_per_task_usd": 0.005421983490566038, "avg_duration_per_task_ms": 524101.0585691824, "avg_output_tokens_per_task": 35942.22051886792, "last_run_timestamp": "2026-08-29T16:53:28.602Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.5773608170445662, "accuracy_std_dev": 0.036921629989749905, "run_count": 20, "total_questions": 3837, "avg_cost_per_task_usd": 0.05988801198853271, "avg_duration_per_task_ms": 462586.3312483711, "avg_output_tokens_per_task": 19012.92468074016, "last_run_timestamp": "2026-08-29T19:11:06.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.5772384474734041, "accuracy_std_dev": 0.11649000927593507, "run_count": 11, "total_questions": 1504, "avg_cost_per_task_usd": 0.004710451462765957, "avg_duration_per_task_ms": 563810.3085106383, "avg_output_tokens_per_task": 32534.692819148935, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.5704424582524273, "accuracy_std_dev": 0.11994568871681048, "run_count": 16, "total_questions": 3090, "avg_cost_per_task_usd": 0.0021708019417475727, "avg_duration_per_task_ms": 65226.70291262136, "avg_output_tokens_per_task": 2007.0271844660194, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.5644344238110643, "accuracy_std_dev": 0.11775681330114128, "run_count": 16, "total_questions": 3091, "avg_cost_per_task_usd": 0.0663304202523455, "avg_duration_per_task_ms": 713891.3856357166, "avg_output_tokens_per_task": 21649.021675833064, "last_run_timestamp": "2026-08-22T17:02:13.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.5547767757009346, "accuracy_std_dev": 0.0771476524984634, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.007002486370716511, "avg_duration_per_task_ms": 230184.06152647975, "avg_output_tokens_per_task": 21060.570093457944, "last_run_timestamp": "2026-08-29T17:01:22.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Phala", "accuracy": 0.5536774896401309, "accuracy_std_dev": 0.11298267708866899, "run_count": 14, "total_questions": 2751, "avg_cost_per_task_usd": 0.03626482333696838, "avg_duration_per_task_ms": 497739.62849872775, "avg_output_tokens_per_task": 13499.503089785532, "last_run_timestamp": "2026-07-11T16:02:27.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.55303, "accuracy_std_dev": 0.061447999999999975, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006801247474747475, "avg_duration_per_task_ms": 137063.60858585857, "avg_output_tokens_per_task": 7268.926767676768, "last_run_timestamp": "2026-09-01T13:21:33.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.5514706253196932, "accuracy_std_dev": 0.20217034155010072, "run_count": 8, "total_questions": 1564, "avg_cost_per_task_usd": 0.02285590537084399, "avg_duration_per_task_ms": 745462.75, "avg_output_tokens_per_task": 32123.599104859335, "last_run_timestamp": "2026-09-26T16:19:31.961Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.5514001070195628, "accuracy_std_dev": 0.20533572929527444, "run_count": 6, "total_questions": 869, "avg_cost_per_task_usd": 0.05092492520138089, "avg_duration_per_task_ms": 1396586.3498273878, "avg_output_tokens_per_task": 19293.715765247412, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5503437185539289, "accuracy_std_dev": 0.037362835232183034, "run_count": 19, "total_questions": 3347, "avg_cost_per_task_usd": 0.031049354048401548, "avg_duration_per_task_ms": 403541.892739767, "avg_output_tokens_per_task": 14846.570959067822, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5496635000000001, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017014479797979796, "avg_duration_per_task_ms": 123100.59595959596, "avg_output_tokens_per_task": 42507.18939393939, "last_run_timestamp": "2026-09-01T17:51:18.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.5483589008379888, "accuracy_std_dev": 0.173972612705257, "run_count": 15, "total_questions": 2864, "avg_cost_per_task_usd": 0.03186034916201117, "avg_duration_per_task_ms": 397835.3034217877, "avg_output_tokens_per_task": 11537.356843575419, "last_run_timestamp": "2026-07-18T16:15:33.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Phala", "accuracy": 0.5475589166666667, "accuracy_std_dev": 0.10881973957303452, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.00421212079124579, "avg_duration_per_task_ms": 247814.2074915825, "avg_output_tokens_per_task": 11271.605218855218, "last_run_timestamp": "2026-06-20T18:20:35.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.5433505, "accuracy_std_dev": 0.016294800251920854, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.017506126262626265, "avg_duration_per_task_ms": 128742.19065656565, "avg_output_tokens_per_task": 43717.62373737374, "last_run_timestamp": "2026-09-01T18:06:03.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.5429295000000001, "accuracy_std_dev": 0.010942500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001760580808080808, "avg_duration_per_task_ms": 80209.29040404041, "avg_output_tokens_per_task": 1718.6565656565656, "last_run_timestamp": "2026-09-01T13:31:30.699Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.542088, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023943484848484846, "avg_duration_per_task_ms": 10793.646464646465, "avg_output_tokens_per_task": 1334.611111111111, "last_run_timestamp": "2026-08-01T08:53:34.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Novita", "accuracy": 0.53661625, "accuracy_std_dev": 0.018685337946836782, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004247876262626262, "avg_duration_per_task_ms": 201941.1426767677, "avg_output_tokens_per_task": 9385.565656565657, "last_run_timestamp": "2026-05-23T11:11:04.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.5341293636363637, "accuracy_std_dev": 0.015229376769154397, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.003491447199265381, "avg_duration_per_task_ms": 77240.00826446281, "avg_output_tokens_per_task": 7540.567033976125, "last_run_timestamp": "2026-08-01T17:48:17.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.53367, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01864569696969697, "avg_duration_per_task_ms": 17970.575757575756, "avg_output_tokens_per_task": 1670.050505050505, "last_run_timestamp": "2026-09-01T13:20:31.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Novita", "accuracy": 0.5323234, "accuracy_std_dev": 0.01905887848326865, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0038402525252525252, "avg_duration_per_task_ms": 124965.20303030303, "avg_output_tokens_per_task": 7619.874747474748, "last_run_timestamp": "2026-07-01T03:39:59.085Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.5315655, "accuracy_std_dev": 0.027285771058374, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004162521464646465, "avg_duration_per_task_ms": 147513.7563131313, "avg_output_tokens_per_task": 8281.712121212122, "last_run_timestamp": "2026-06-20T17:38:01.923Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5312429141104295, "accuracy_std_dev": 0.042893663569032354, "run_count": 15, "total_questions": 2934, "avg_cost_per_task_usd": 0.01798347137014315, "avg_duration_per_task_ms": 733666.4400136332, "avg_output_tokens_per_task": 45997.68916155419, "last_run_timestamp": "2026-08-30T00:49:28.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.5303269819620254, "accuracy_std_dev": 0.1650574482984897, "run_count": 16, "total_questions": 3160, "avg_cost_per_task_usd": 0.0020356806962025316, "avg_duration_per_task_ms": 82497.10664556963, "avg_output_tokens_per_task": 2227.1496835443036, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "ModelRun", "accuracy": 0.5290405, "accuracy_std_dev": 0.029959029561219083, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0006086212121212121, "avg_duration_per_task_ms": 25756.060606060608, "avg_output_tokens_per_task": 699.3156565656566, "last_run_timestamp": "2026-06-20T16:15:10.268Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.521886, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.016360343434343436, "avg_duration_per_task_ms": 13511.040404040405, "avg_output_tokens_per_task": 1443.3737373737374, "last_run_timestamp": "2026-09-01T14:02:01.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.521297721396731, "accuracy_std_dev": 0.03986710324587105, "run_count": 22, "total_questions": 4038, "avg_cost_per_task_usd": 0.009685821198613175, "avg_duration_per_task_ms": 422937.02204061416, "avg_output_tokens_per_task": 24523.735264982664, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.5202023333333333, "accuracy_std_dev": 0.0023810642345154617, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0035723956228956226, "avg_duration_per_task_ms": 90157.08922558922, "avg_output_tokens_per_task": 2721.4242424242425, "last_run_timestamp": "2026-08-15T08:46:42.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.5190796666666666, "accuracy_std_dev": 0.008837261503932561, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004016927609427609, "avg_duration_per_task_ms": 28916.838383838385, "avg_output_tokens_per_task": 2584.1296296296296, "last_run_timestamp": "2026-08-15T09:25:42.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Ambient", "accuracy": 0.5176769999999999, "accuracy_std_dev": 0.007575999999999972, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009877323232323232, "avg_duration_per_task_ms": 465993.38636363635, "avg_output_tokens_per_task": 18118.93181818182, "last_run_timestamp": "2026-06-13T11:17:16.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.517396, "accuracy_std_dev": 0.02012409515978297, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015192781144781144, "avg_duration_per_task_ms": 14836.205387205388, "avg_output_tokens_per_task": 1327.0218855218855, "last_run_timestamp": "2026-09-01T13:22:30.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.516835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018182969696969697, "avg_duration_per_task_ms": 17464.777777777777, "avg_output_tokens_per_task": 1627.171717171717, "last_run_timestamp": "2026-09-01T13:20:49.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.5157126666666666, "accuracy_std_dev": 0.02305568803763809, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004710382154882156, "avg_duration_per_task_ms": 211036.34006734006, "avg_output_tokens_per_task": 2694.722222222222, "last_run_timestamp": "2026-08-15T09:11:29.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.5137279018733274, "accuracy_std_dev": 0.06412042341716619, "run_count": 19, "total_questions": 3363, "avg_cost_per_task_usd": 0.06096582634552484, "avg_duration_per_task_ms": 510270.59768064227, "avg_output_tokens_per_task": 17365.03865596194, "last_run_timestamp": "2026-08-29T18:52:09.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.5134683333333333, "accuracy_std_dev": 0.00836132925370652, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008388316498316498, "avg_duration_per_task_ms": 89120.7138047138, "avg_output_tokens_per_task": 2792.7188552188554, "last_run_timestamp": "2026-08-15T08:49:03.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.5122718770053475, "accuracy_std_dev": 0.11548223967755335, "run_count": 14, "total_questions": 2431, "avg_cost_per_task_usd": 0.008256612093788564, "avg_duration_per_task_ms": 484397.99177293293, "avg_output_tokens_per_task": 41115.3813245578, "last_run_timestamp": "2026-09-26T16:17:38.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.511785, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01733130303030303, "avg_duration_per_task_ms": 13903.823232323231, "avg_output_tokens_per_task": 1536.9141414141413, "last_run_timestamp": "2026-09-01T13:47:27.116Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.5117335484247374, "accuracy_std_dev": 0.013140266237519679, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.004239138467522365, "avg_duration_per_task_ms": 118277.81796966161, "avg_output_tokens_per_task": 8576.858031894204, "last_run_timestamp": "2026-08-01T18:08:24.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.5092595, "accuracy_std_dev": 0.009259499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008195479797979797, "avg_duration_per_task_ms": 19948.4797979798, "avg_output_tokens_per_task": 1859.0176767676767, "last_run_timestamp": "2026-09-01T17:50:32.282Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.506734, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0237055303030303, "avg_duration_per_task_ms": 11478.515151515152, "avg_output_tokens_per_task": 1318.7474747474748, "last_run_timestamp": "2026-08-01T08:34:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.5053764679900744, "accuracy_std_dev": 0.20352934331044298, "run_count": 11, "total_questions": 2015, "avg_cost_per_task_usd": 0.01812188188585608, "avg_duration_per_task_ms": 1793360.2853598015, "avg_output_tokens_per_task": 13961.292803970224, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02292151515151515, "avg_duration_per_task_ms": 12797.353535353535, "avg_output_tokens_per_task": 1266.479797979798, "last_run_timestamp": "2026-08-01T08:47:47.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.5033673333333334, "accuracy_std_dev": 0.009622237242739144, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008440841750841751, "avg_duration_per_task_ms": 14338.025252525253, "avg_output_tokens_per_task": 1851.915824915825, "last_run_timestamp": "2026-09-01T17:51:18.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.503367, "accuracy_std_dev": 0.008418000000000037, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013204560606060607, "avg_duration_per_task_ms": 13084.876262626263, "avg_output_tokens_per_task": 1127.6338383838383, "last_run_timestamp": "2026-09-01T13:20:22.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Together", "accuracy": 0.5009998431876606, "accuracy_std_dev": 0.23720298614056273, "run_count": 6, "total_questions": 1167, "avg_cost_per_task_usd": 0.004498337617823479, "avg_duration_per_task_ms": 250896.1996572408, "avg_output_tokens_per_task": 23117.198800342758, "last_run_timestamp": "2026-07-04T16:00:23.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.4963358823529412, "accuracy_std_dev": 0.019652006075963045, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.0017086723113487819, "avg_duration_per_task_ms": 8000.373737373738, "avg_output_tokens_per_task": 1693.5855614973261, "last_run_timestamp": "2026-08-29T19:15:06.404Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.4951475294117647, "accuracy_std_dev": 0.023405011434501268, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.0007195903149138444, "avg_duration_per_task_ms": 45332.08585858586, "avg_output_tokens_per_task": 1560.3024361259656, "last_run_timestamp": "2026-08-29T19:21:45.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.4934997222222222, "accuracy_std_dev": 0.02101166846409543, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.0017448033108866444, "avg_duration_per_task_ms": 10151.609988776656, "avg_output_tokens_per_task": 1588.8221099887767, "last_run_timestamp": "2026-08-29T16:43:47.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.49248677460142937, "accuracy_std_dev": 0.1468475282129393, "run_count": 12, "total_questions": 1819, "avg_cost_per_task_usd": 0.02537227047828477, "avg_duration_per_task_ms": 43723.59538207806, "avg_output_tokens_per_task": 5430.2094557449145, "last_run_timestamp": "2026-08-29T16:54:06.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Inceptron", "accuracy": 0.489899, "accuracy_std_dev": 0.020202000000000025, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0003088434343434343, "avg_duration_per_task_ms": 18505.795454545456, "avg_output_tokens_per_task": 721.5959595959596, "last_run_timestamp": "2026-05-16T08:12:45.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.48945589473684215, "accuracy_std_dev": 0.024456503773659585, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.0008684388623072833, "avg_duration_per_task_ms": 44982.24348750665, "avg_output_tokens_per_task": 1513.1935140882508, "last_run_timestamp": "2026-08-29T18:24:23.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.48919281499107675, "accuracy_std_dev": 0.02018458247839152, "run_count": 17, "total_questions": 3362, "avg_cost_per_task_usd": 0.0008363173706127306, "avg_duration_per_task_ms": 58785.93456276026, "avg_output_tokens_per_task": 1903.9723378941108, "last_run_timestamp": "2026-08-29T19:00:36.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.4885771112490687, "accuracy_std_dev": 0.03899715161939785, "run_count": 21, "total_questions": 4027, "avg_cost_per_task_usd": 0.007579310404767817, "avg_duration_per_task_ms": 295718.4092376459, "avg_output_tokens_per_task": 16477.777253538614, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.4882155000000001, "accuracy_std_dev": 0.016835500000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008046262626262627, "avg_duration_per_task_ms": 20667.484848484848, "avg_output_tokens_per_task": 1820.4444444444443, "last_run_timestamp": "2026-09-01T17:50:43.769Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.48653191666666656, "accuracy_std_dev": 0.02063010630308019, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0013203434343434341, "avg_duration_per_task_ms": 19607.073232323233, "avg_output_tokens_per_task": 1411.3367003367002, "last_run_timestamp": "2026-07-18T16:13:31.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.48590056249999997, "accuracy_std_dev": 0.024073916665575915, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.0023842935606060607, "avg_duration_per_task_ms": 42122.70233585859, "avg_output_tokens_per_task": 1688.7894570707072, "last_run_timestamp": "2026-08-29T18:45:20.686Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.48383308151910986, "accuracy_std_dev": 0.0343561201621871, "run_count": 21, "total_questions": 4134, "avg_cost_per_task_usd": 0.006950208998548622, "avg_duration_per_task_ms": 299773.75689404935, "avg_output_tokens_per_task": 17335.253991291727, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.4829639762922659, "accuracy_std_dev": 0.0222392355589962, "run_count": 13, "total_questions": 2573, "avg_cost_per_task_usd": 0.0010490423630003885, "avg_duration_per_task_ms": 37775.31752817723, "avg_output_tokens_per_task": 1882.54061406918, "last_run_timestamp": "2026-08-29T17:00:17.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.481481, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.001115070707070707, "avg_duration_per_task_ms": 120135.48484848485, "avg_output_tokens_per_task": 2056.4848484848485, "last_run_timestamp": "2026-09-01T14:05:19.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.47643079999999993, "accuracy_std_dev": 0.007896357575490103, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0023637252525252526, "avg_duration_per_task_ms": 39731.30202020202, "avg_output_tokens_per_task": 2520.320202020202, "last_run_timestamp": "2026-08-29T18:16:38.845Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.47486757142857144, "accuracy_std_dev": 0.011496839122834002, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.000819739538239538, "avg_duration_per_task_ms": 122852.08513708513, "avg_output_tokens_per_task": 1852.6435786435786, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.4748159255751015, "accuracy_std_dev": 0.1131125589884548, "run_count": 12, "total_questions": 2217, "avg_cost_per_task_usd": 0.01081031709517366, "avg_duration_per_task_ms": 287884.16599007667, "avg_output_tokens_per_task": 14729.669373026612, "last_run_timestamp": "2026-08-01T18:13:08.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4747473529411765, "accuracy_std_dev": 0.01954779462494329, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.0006343000594177065, "avg_duration_per_task_ms": 120340.93464052287, "avg_output_tokens_per_task": 1809.9459298871063, "last_run_timestamp": "2026-08-29T18:48:55.759Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.46742483824768327, "accuracy_std_dev": 0.006948085005956674, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.002137747262005055, "avg_duration_per_task_ms": 34994.02274641955, "avg_output_tokens_per_task": 2587.217354675653, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.46464666666666665, "accuracy_std_dev": 0.24034588812283755, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0018368636363636363, "avg_duration_per_task_ms": 32878.122895622895, "avg_output_tokens_per_task": 3002.4276094276092, "last_run_timestamp": "2026-06-30T19:37:23.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.462542, "accuracy_std_dev": 0.005471000000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001783989898989899, "avg_duration_per_task_ms": 27644.777777777777, "avg_output_tokens_per_task": 1397.5252525252524, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.46128619685039374, "accuracy_std_dev": 0.075159, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.00731248031496063, "avg_duration_per_task_ms": 1016477.8543307086, "avg_output_tokens_per_task": 20586.740157480315, "last_run_timestamp": "2026-08-08T10:11:59.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Google", "accuracy": 0.4557567443438914, "accuracy_std_dev": 0.195794758968546, "run_count": 7, "total_questions": 1326, "avg_cost_per_task_usd": 0.002924125942684766, "avg_duration_per_task_ms": 340072.5972850679, "avg_output_tokens_per_task": 15733.245852187029, "last_run_timestamp": "2026-07-11T15:34:14.755Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.4461280000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0009776919191919191, "avg_duration_per_task_ms": 90556.41414141415, "avg_output_tokens_per_task": 1759.8131313131314, "last_run_timestamp": "2026-09-01T14:15:49.249Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.4444445, "accuracy_std_dev": 0.015151500000000012, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0015239469696969696, "avg_duration_per_task_ms": 59255.792929292926, "avg_output_tokens_per_task": 1592.9924242424242, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.4402355, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0013545, "avg_duration_per_task_ms": 68001.7196969697, "avg_output_tokens_per_task": 1535.1843434343434, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.4335015, "accuracy_std_dev": 0.009259500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010084242424242425, "avg_duration_per_task_ms": 21177.93181818182, "avg_output_tokens_per_task": 1486.2676767676767, "last_run_timestamp": "2026-09-01T17:49:46.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4301345, "accuracy_std_dev": 0.009259500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010125732323232322, "avg_duration_per_task_ms": 17946.272727272728, "avg_output_tokens_per_task": 1495.5707070707072, "last_run_timestamp": "2026-09-01T17:55:31.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.4297135, "accuracy_std_dev": 0.007469839539775939, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010784545454545454, "avg_duration_per_task_ms": 21933.41792929293, "avg_output_tokens_per_task": 1515.1376262626263, "last_run_timestamp": "2026-09-01T17:31:07.952Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.3889878823529412, "accuracy_std_dev": 0.2561627234269568, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.0011359013666072488, "avg_duration_per_task_ms": 33787.26500297088, "avg_output_tokens_per_task": 967.2875816993464, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy-mt2-30b-a3b-20260521", "model_slug": "tencent/hy-mt2-30b-a3b", "model_name": "Tencent: Hy-MT2-30B-A3B", "provider_name": "Reka", "accuracy": 0.388889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00023147979797979797, "avg_duration_per_task_ms": 8604.353535353535, "avg_output_tokens_per_task": 710.540404040404, "last_run_timestamp": "2026-09-24T06:00:03.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.3860826666666667, "accuracy_std_dev": 0.005722799567422292, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00015977104377104374, "avg_duration_per_task_ms": 6254.590909090909, "avg_output_tokens_per_task": 953.9612794612794, "last_run_timestamp": "2026-09-01T13:54:02.609Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.364571888888889, "accuracy_std_dev": 0.40360819680929966, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.003561915263748597, "avg_duration_per_task_ms": 170742.85016835018, "avg_output_tokens_per_task": 3450.5415263748596, "last_run_timestamp": "2026-08-15T14:54:58.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.36363599999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00041437878787878784, "avg_duration_per_task_ms": 34981.92424242424, "avg_output_tokens_per_task": 1665.6666666666667, "last_run_timestamp": "2026-08-01T08:55:19.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Together", "accuracy": 0.3417510000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0007054343434343434, "avg_duration_per_task_ms": 14064.868686868687, "avg_output_tokens_per_task": 1539, "last_run_timestamp": "2026-08-01T08:06:23.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.328283, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005318181818181818, "avg_duration_per_task_ms": 36388.32323232323, "avg_output_tokens_per_task": 2041.050505050505, "last_run_timestamp": "2026-08-01T08:36:11.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.3156760930232559, "accuracy_std_dev": 0.020895107938223238, "run_count": 8, "total_questions": 1548, "avg_cost_per_task_usd": 0.00008417894056847546, "avg_duration_per_task_ms": 8191.765503875969, "avg_output_tokens_per_task": 362.62467700258395, "last_run_timestamp": "2026-07-18T09:08:52.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.30923066923076925, "accuracy_std_dev": 0.02667595582842347, "run_count": 10, "total_questions": 1950, "avg_cost_per_task_usd": 0.0000934446153846154, "avg_duration_per_task_ms": 5045.05282051282, "avg_output_tokens_per_task": 300.8317948717949, "last_run_timestamp": "2026-07-18T09:09:45.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.30896005292479106, "accuracy_std_dev": 0.05550927626372292, "run_count": 11, "total_questions": 2154, "avg_cost_per_task_usd": 0.00260005756731662, "avg_duration_per_task_ms": 28138.977251624885, "avg_output_tokens_per_task": 1086.3834726090993, "last_run_timestamp": "2026-08-29T17:41:08.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.30662180000000006, "accuracy_std_dev": 0.048718266987787376, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.006156473063973065, "avg_duration_per_task_ms": 310379.08855218854, "avg_output_tokens_per_task": 13940.810774410775, "last_run_timestamp": "2026-08-29T19:48:22.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Novita", "accuracy": 0.3056295271111111, "accuracy_std_dev": 0.05792376099499717, "run_count": 6, "total_questions": 1125, "avg_cost_per_task_usd": 0.00008087733333333333, "avg_duration_per_task_ms": 36138.544, "avg_output_tokens_per_task": 344.34222222222223, "last_run_timestamp": "2026-07-18T08:00:15.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DeepInfra", "accuracy": 0.3026373112840467, "accuracy_std_dev": 0.026204171217336758, "run_count": 8, "total_questions": 1542, "avg_cost_per_task_usd": 0.00002046757457846952, "avg_duration_per_task_ms": 9360.384565499351, "avg_output_tokens_per_task": 275.0188067444877, "last_run_timestamp": "2026-07-18T08:00:15.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.29040400000000005, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.000060893939393939394, "avg_duration_per_task_ms": 9271.39393939394, "avg_output_tokens_per_task": 573.0479797979798, "last_run_timestamp": "2026-09-01T13:00:51.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.28953874164345406, "accuracy_std_dev": 0.024162932213900572, "run_count": 22, "total_questions": 4308, "avg_cost_per_task_usd": 0.00016581569173630456, "avg_duration_per_task_ms": 163891.54758588673, "avg_output_tokens_per_task": 3896.552460538533, "last_run_timestamp": "2026-08-29T16:57:41.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DekaLLM", "accuracy": 0.2874569642004773, "accuracy_std_dev": 0.027560829670429898, "run_count": 7, "total_questions": 1257, "avg_cost_per_task_usd": 0.000016271280827366746, "avg_duration_per_task_ms": 11521.354017501988, "avg_output_tokens_per_task": 197.71917263325378, "last_run_timestamp": "2026-07-18T08:04:03.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.28136913333333335, "accuracy_std_dev": 0.025692936660663933, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.0007517552188552191, "avg_duration_per_task_ms": 48215.582154882155, "avg_output_tokens_per_task": 9778.388552188551, "last_run_timestamp": "2026-08-29T18:21:00.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.2780711647307286, "accuracy_std_dev": 0.014628725522067875, "run_count": 5, "total_questions": 947, "avg_cost_per_task_usd": 0.0063300253431890175, "avg_duration_per_task_ms": 245873.9429778247, "avg_output_tokens_per_task": 28094.863780359028, "last_run_timestamp": "2026-08-29T18:18:26.937Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.275533, "accuracy_std_dev": 0.009152576176501717, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0007405521885521885, "avg_duration_per_task_ms": 48360.410774410775, "avg_output_tokens_per_task": 2532.1531986531986, "last_run_timestamp": "2026-08-15T09:28:07.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cerebras", "accuracy": 0.2727275, "accuracy_std_dev": 0.035353499999999996, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00008969696969696971, "avg_duration_per_task_ms": 633.3914141414141, "avg_output_tokens_per_task": 628.320707070707, "last_run_timestamp": "2026-05-23T08:04:59.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.27185566666666666, "accuracy_std_dev": 0.008747009330178078, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.00006098307952622674, "avg_duration_per_task_ms": 9307.97461928934, "avg_output_tokens_per_task": 612.7817258883249, "last_run_timestamp": "2026-09-01T12:41:31.862Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.26936, "accuracy_std_dev": 0.00673399999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0000632449494949495, "avg_duration_per_task_ms": 9455.388888888889, "avg_output_tokens_per_task": 592.1616161616162, "last_run_timestamp": "2026-09-01T13:21:46.100Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.2692480666666667, "accuracy_std_dev": 0.018385093311218802, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.0008740006734006734, "avg_duration_per_task_ms": 22071.978787878787, "avg_output_tokens_per_task": 10913.142760942761, "last_run_timestamp": "2026-08-29T18:45:59.793Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.268788525198939, "accuracy_std_dev": 0.004147499999999998, "run_count": 2, "total_questions": 377, "avg_cost_per_task_usd": 0.0000679681697612732, "avg_duration_per_task_ms": 9180.114058355437, "avg_output_tokens_per_task": 568.5623342175066, "last_run_timestamp": "2026-09-01T13:04:23.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "WandB", "accuracy": 0.25380578104993595, "accuracy_std_dev": 0.022641417314753803, "run_count": 12, "total_questions": 2343, "avg_cost_per_task_usd": 0.0007069568928723856, "avg_duration_per_task_ms": 20649.668373879642, "avg_output_tokens_per_task": 2769.859581732821, "last_run_timestamp": "2026-07-18T08:26:34.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.25143756923076926, "accuracy_std_dev": 0.08124746081830182, "run_count": 11, "total_questions": 2145, "avg_cost_per_task_usd": 0.0014683734265734264, "avg_duration_per_task_ms": 503997.0027972028, "avg_output_tokens_per_task": 5763.862937062937, "last_run_timestamp": "2026-08-29T17:16:14.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Ambient", "accuracy": 0.245614, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.03239533333333334, "avg_duration_per_task_ms": 323216.8245614035, "avg_output_tokens_per_task": 11659.74269005848, "last_run_timestamp": "2026-06-30T21:29:29.599Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.24149673724489795, "accuracy_std_dev": 0.024442688065230603, "run_count": 12, "total_questions": 2352, "avg_cost_per_task_usd": 0.00032777253401360546, "avg_duration_per_task_ms": 55241.83588435374, "avg_output_tokens_per_task": 6268.276785714285, "last_run_timestamp": "2026-08-29T16:55:14.828Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Parasail", "accuracy": 0.21734899999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.0003523450292397661, "avg_duration_per_task_ms": 72727.70760233919, "avg_output_tokens_per_task": 4269.023391812865, "last_run_timestamp": "2026-08-01T08:05:56.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.19696999999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0190694696969697, "avg_duration_per_task_ms": 2493159.101010101, "avg_output_tokens_per_task": 105666.86363636363, "last_run_timestamp": "2026-09-03T21:14:17.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "auto-routing", "accuracy": 0.116162, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0001537979797979798, "avg_duration_per_task_ms": 29722.909090909092, "avg_output_tokens_per_task": 1836.7929292929293, "last_run_timestamp": "2026-08-01T09:03:02.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.114478, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.017837661616161616, "avg_duration_per_task_ms": 71618.32323232324, "avg_output_tokens_per_task": 11767.727272727272, "last_run_timestamp": "2026-09-01T12:56:59.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.10606100000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.017886560606060607, "avg_duration_per_task_ms": 21072.777777777777, "avg_output_tokens_per_task": 11801.328282828283, "last_run_timestamp": "2026-09-01T13:00:01.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018059262626262628, "avg_duration_per_task_ms": 21777.424242424244, "avg_output_tokens_per_task": 11808.9898989899, "last_run_timestamp": "2026-09-01T13:21:37.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.060606, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.000040161616161616165, "avg_duration_per_task_ms": 3671.1060606060605, "avg_output_tokens_per_task": 93.72222222222223, "last_run_timestamp": "2026-08-01T08:05:34.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.0038196250000000005, "accuracy_std_dev": 0.0032291292324711936, "run_count": 5, "total_questions": 960, "avg_cost_per_task_usd": 0.003103454166666667, "avg_duration_per_task_ms": 152207.65520833334, "avg_output_tokens_per_task": 2930.8010416666666, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.822917, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.0087319375, "avg_duration_per_task_ms": 31283.875, "avg_output_tokens_per_task": 9903.5625, "last_run_timestamp": "2026-09-26T08:05:12.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20567228, "avg_duration_per_task_ms": 69455.26, "avg_output_tokens_per_task": 15737.28, "last_run_timestamp": "2026-08-15T00:15:11.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.805556, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.07709047916666667, "avg_duration_per_task_ms": 121300.85416666667, "avg_output_tokens_per_task": 14694.6875, "last_run_timestamp": "2026-08-15T00:15:15.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.804166625, "accuracy_std_dev": 0.021065488167483215, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.8715094075, "avg_duration_per_task_ms": 137048.4925, "avg_output_tokens_per_task": 5786.43, "last_run_timestamp": "2026-09-26T08:21:11.883Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8, "accuracy_std_dev": 0.033110611481316166, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.7322859599999999, "avg_duration_per_task_ms": 146639.07333333333, "avg_output_tokens_per_task": 5472.34, "last_run_timestamp": "2026-08-15T08:09:14.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Fireworks", "accuracy": 0.8, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.030081680000000003, "avg_duration_per_task_ms": 53399.74, "avg_output_tokens_per_task": 3282.63, "last_run_timestamp": "2026-04-25T14:38:47.693Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.7977776666666667, "accuracy_std_dev": 0.01911642045177097, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008363246666666668, "avg_duration_per_task_ms": 272879.14, "avg_output_tokens_per_task": 12625.026666666667, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Nebius", "accuracy": 0.7966666666666666, "accuracy_std_dev": 0.021343747458109477, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08247077333333334, "avg_duration_per_task_ms": 126277.50333333333, "avg_output_tokens_per_task": 8287.77, "last_run_timestamp": "2026-05-23T17:33:06.915Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.795833375, "accuracy_std_dev": 0.02846769733811245, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.9313334725, "avg_duration_per_task_ms": 130292.9175, "avg_output_tokens_per_task": 5748.73, "last_run_timestamp": "2026-09-26T08:18:08.440Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.7933333333333333, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.6872690333333334, "avg_duration_per_task_ms": 114143.74666666667, "avg_output_tokens_per_task": 6925.7266666666665, "last_run_timestamp": "2026-09-26T08:22:07.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7911111666666667, "accuracy_std_dev": 0.019116304205369344, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.8711063166666666, "avg_duration_per_task_ms": 151211.67333333334, "avg_output_tokens_per_task": 5696.93, "last_run_timestamp": "2026-09-26T08:09:55.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.790833375, "accuracy_std_dev": 0.01630862714897777, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.8205446474999999, "avg_duration_per_task_ms": 138181.955, "avg_output_tokens_per_task": 5731.4725, "last_run_timestamp": "2026-09-26T08:05:12.739Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.788889, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08521802666666665, "avg_duration_per_task_ms": 39816.48, "avg_output_tokens_per_task": 4684.933333333333, "last_run_timestamp": "2026-09-26T08:13:10.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.78833325, "accuracy_std_dev": 0.009860203404975974, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.8196274600000001, "avg_duration_per_task_ms": 183041.66, "avg_output_tokens_per_task": 7098.595, "last_run_timestamp": "2026-09-26T08:15:33.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7877778666666666, "accuracy_std_dev": 0.02049995242552745, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.9492467306666665, "avg_duration_per_task_ms": 98954.73466666667, "avg_output_tokens_per_task": 5602.965333333334, "last_run_timestamp": "2026-09-26T08:12:28.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.7875, "accuracy_std_dev": 0.024593980747329203, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.48769549000000006, "avg_duration_per_task_ms": 129773.455, "avg_output_tokens_per_task": 7130.5275, "last_run_timestamp": "2026-09-26T08:22:26.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.7870992438478748, "accuracy_std_dev": 0.008959550928450943, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.11607393959731543, "avg_duration_per_task_ms": 143483.8120805369, "avg_output_tokens_per_task": 11288.324384787473, "last_run_timestamp": "2026-09-26T08:27:30.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.8471384599999998, "avg_duration_per_task_ms": 331793.46, "avg_output_tokens_per_task": 22741.94, "last_run_timestamp": "2026-08-01T08:06:01.602Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7866669999999999, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2795196199999999, "avg_duration_per_task_ms": 104647.27, "avg_output_tokens_per_task": 5686.75, "last_run_timestamp": "2026-09-01T13:03:28.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.7866665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12536731, "avg_duration_per_task_ms": 118841.15, "avg_output_tokens_per_task": 19578.58, "last_run_timestamp": "2026-09-01T17:30:58.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.7859532642140468, "accuracy_std_dev": 0.022060806735148097, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.028781180602006688, "avg_duration_per_task_ms": 346700.6989966555, "avg_output_tokens_per_task": 11874.802675585284, "last_run_timestamp": "2026-08-29T17:06:21.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7857143728222997, "accuracy_std_dev": 0.022636034639976627, "run_count": 6, "total_questions": 287, "avg_cost_per_task_usd": 0.11757857142857142, "avg_duration_per_task_ms": 73705.68641114983, "avg_output_tokens_per_task": 8241.627177700348, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.784471297188755, "accuracy_std_dev": 0.02769278737577711, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.09832240562248996, "avg_duration_per_task_ms": 406639.9036144578, "avg_output_tokens_per_task": 17335.586345381525, "last_run_timestamp": "2026-09-26T08:23:09.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Morph", "accuracy": 0.784, "accuracy_std_dev": 0.04319121652836371, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.082875132, "avg_duration_per_task_ms": 213022.804, "avg_output_tokens_per_task": 11102.478666666666, "last_run_timestamp": "2026-07-04T16:00:23.544Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.7831884347826087, "accuracy_std_dev": 0.03796104117304211, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.10458759999999999, "avg_duration_per_task_ms": 387959.7686956522, "avg_output_tokens_per_task": 16737.08, "last_run_timestamp": "2026-08-29T16:14:58.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.7822223333333334, "accuracy_std_dev": 0.013698571612479267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06831790666666666, "avg_duration_per_task_ms": 97592.92, "avg_output_tokens_per_task": 11907.126666666667, "last_run_timestamp": "2026-07-18T08:04:03.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.78041675, "accuracy_std_dev": 0.02409111033789228, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.4685391825, "avg_duration_per_task_ms": 111064.53375, "avg_output_tokens_per_task": 7216.75875, "last_run_timestamp": "2026-09-26T08:13:10.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.78000025, "accuracy_std_dev": 0.008164965810553043, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.7813667750000001, "avg_duration_per_task_ms": 113768.35, "avg_output_tokens_per_task": 6852.055, "last_run_timestamp": "2026-09-26T08:13:59.150Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.78, "accuracy_std_dev": 0.021081640250859665, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.016256343333333336, "avg_duration_per_task_ms": 172479.31, "avg_output_tokens_per_task": 11984.07, "last_run_timestamp": "2026-08-08T08:29:14.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Together", "accuracy": 0.78, "accuracy_std_dev": 0.025071326821120347, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.11856992714285713, "avg_duration_per_task_ms": 98828.59428571428, "avg_output_tokens_per_task": 15445.625714285714, "last_run_timestamp": "2026-06-20T17:08:55.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.78, "accuracy_std_dev": 1.1102230246251565e-16, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.14591014000000002, "avg_duration_per_task_ms": 245617.41333333333, "avg_output_tokens_per_task": 6015.373333333333, "last_run_timestamp": "2026-07-18T16:00:09.009Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7788889166666667, "accuracy_std_dev": 0.02298676618897032, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.4834922383333334, "avg_duration_per_task_ms": 127667.31666666667, "avg_output_tokens_per_task": 7153.7075, "last_run_timestamp": "2026-09-26T08:12:13.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.7783335, "accuracy_std_dev": 0.028037782049406136, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.099593165, "avg_duration_per_task_ms": 81195.23, "avg_output_tokens_per_task": 4770.2, "last_run_timestamp": "2026-09-26T08:13:27.135Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.77833325, "accuracy_std_dev": 0.026404708125398767, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.807941035, "avg_duration_per_task_ms": 183353.72, "avg_output_tokens_per_task": 6938.485, "last_run_timestamp": "2026-09-26T08:11:40.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.77813332, "accuracy_std_dev": 0.02505255531991896, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.5623465103999999, "avg_duration_per_task_ms": 118279.7304, "avg_output_tokens_per_task": 7235.6968, "last_run_timestamp": "2026-09-26T08:16:10.311Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.777778, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.012714583333333333, "avg_duration_per_task_ms": 516568.7291666667, "avg_output_tokens_per_task": 14230.208333333334, "last_run_timestamp": "2026-08-03T17:20:47.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.7770961820040899, "accuracy_std_dev": 0.01908827114120082, "run_count": 10, "total_questions": 489, "avg_cost_per_task_usd": 0.07789318609406953, "avg_duration_per_task_ms": 97210.93251533742, "avg_output_tokens_per_task": 4258.01226993865, "last_run_timestamp": "2026-09-26T08:11:43.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7770371111111111, "accuracy_std_dev": 0.02224700963397825, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.4357714733333334, "avg_duration_per_task_ms": 133682.32888888888, "avg_output_tokens_per_task": 7264.266666666666, "last_run_timestamp": "2026-09-26T08:17:01.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.7766665, "accuracy_std_dev": 0.019148513142887433, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008114253333333333, "avg_duration_per_task_ms": 195047.99, "avg_output_tokens_per_task": 21425.716666666667, "last_run_timestamp": "2026-08-08T08:30:38.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.776, "accuracy_std_dev": 0.022939553421982742, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.007782092000000001, "avg_duration_per_task_ms": 211248.992, "avg_output_tokens_per_task": 4235.592, "last_run_timestamp": "2026-09-26T08:27:53.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.7755555000000001, "accuracy_std_dev": 0.0236225357991756, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.4783927666666667, "avg_duration_per_task_ms": 128906.20333333334, "avg_output_tokens_per_task": 7843.846666666666, "last_run_timestamp": "2026-09-26T08:22:36.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.77472225, "accuracy_std_dev": 0.029058032346567567, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.029999545833333332, "avg_duration_per_task_ms": 132057.45166666666, "avg_output_tokens_per_task": 5965.595833333334, "last_run_timestamp": "2026-08-29T17:11:32.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.7746666, "accuracy_std_dev": 0.02902873978387625, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.15693796000000002, "avg_duration_per_task_ms": 605970.636, "avg_output_tokens_per_task": 12427.38, "last_run_timestamp": "2026-09-26T08:23:03.870Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.7746666, "accuracy_std_dev": 0.022861927596770998, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.080879576, "avg_duration_per_task_ms": 126701.14, "avg_output_tokens_per_task": 4510.112, "last_run_timestamp": "2026-09-26T08:11:43.279Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.7746666, "accuracy_std_dev": 0.020395973461445754, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.120308296, "avg_duration_per_task_ms": 80105.144, "avg_output_tokens_per_task": 4494.34, "last_run_timestamp": "2026-09-26T08:09:25.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7746664000000001, "accuracy_std_dev": 0.016546172434735443, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.09735419199999999, "avg_duration_per_task_ms": 130067.476, "avg_output_tokens_per_task": 4561.928, "last_run_timestamp": "2026-09-26T08:10:40.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.7744443333333334, "accuracy_std_dev": 0.035986907770589154, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06337854666666667, "avg_duration_per_task_ms": 170014.64, "avg_output_tokens_per_task": 7038.76, "last_run_timestamp": "2026-08-29T16:51:02.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7733333333333333, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02996874, "avg_duration_per_task_ms": 113373.20666666667, "avg_output_tokens_per_task": 12010.053333333333, "last_run_timestamp": "2026-08-29T16:46:02.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.7733332727272727, "accuracy_std_dev": 0.018422746153456893, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4337744636363636, "avg_duration_per_task_ms": 96869.80363636363, "avg_output_tokens_per_task": 5661.309090909091, "last_run_timestamp": "2026-09-26T08:11:40.823Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.7733331666666667, "accuracy_std_dev": 0.00384890557676971, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.006986003333333334, "avg_duration_per_task_ms": 162086.69, "avg_output_tokens_per_task": 16532.63, "last_run_timestamp": "2026-08-08T08:27:33.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.773333, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12487065999999998, "avg_duration_per_task_ms": 149585.85, "avg_output_tokens_per_task": 20477.48, "last_run_timestamp": "2026-09-01T17:31:27.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02012318, "avg_duration_per_task_ms": 229027.46, "avg_output_tokens_per_task": 10940.04, "last_run_timestamp": "2026-07-18T08:04:46.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06535734, "avg_duration_per_task_ms": 368327.08, "avg_output_tokens_per_task": 13147.48, "last_run_timestamp": "2026-09-23T15:00:14.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7726666, "accuracy_std_dev": 0.03641728405359191, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.113975304, "avg_duration_per_task_ms": 177276.2, "avg_output_tokens_per_task": 9278.056, "last_run_timestamp": "2026-08-29T16:15:12.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.7725, "accuracy_std_dev": 0.030810865688909155, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.47031593, "avg_duration_per_task_ms": 117109.07, "avg_output_tokens_per_task": 7210.535, "last_run_timestamp": "2026-09-26T08:12:30.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7725, "accuracy_std_dev": 0.03302967521033168, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.22805288999999998, "avg_duration_per_task_ms": 411518.45, "avg_output_tokens_per_task": 13622.02, "last_run_timestamp": "2026-09-26T08:05:12.789Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7722223333333333, "accuracy_std_dev": 0.03890480468042762, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07840497666666667, "avg_duration_per_task_ms": 265427.81666666665, "avg_output_tokens_per_task": 11547.233333333334, "last_run_timestamp": "2026-09-26T08:13:10.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.77166675, "accuracy_std_dev": 0.023033966466666143, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.5286243174999999, "avg_duration_per_task_ms": 174531.3525, "avg_output_tokens_per_task": 10507.29, "last_run_timestamp": "2026-09-26T08:05:12.739Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.7716665, "accuracy_std_dev": 0.033124370015393, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.15296470833333334, "avg_duration_per_task_ms": 558733.8216666667, "avg_output_tokens_per_task": 36167.525, "last_run_timestamp": "2026-09-26T08:01:18.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.7716390570902395, "accuracy_std_dev": 0.03607105625734658, "run_count": 11, "total_questions": 543, "avg_cost_per_task_usd": 0.07918082872928178, "avg_duration_per_task_ms": 988188.6095764273, "avg_output_tokens_per_task": 25083.388581952117, "last_run_timestamp": "2026-09-26T08:24:45.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7715151818181818, "accuracy_std_dev": 0.029417231634840356, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5523802672727273, "avg_duration_per_task_ms": 183979.0109090909, "avg_output_tokens_per_task": 10526.685454545455, "last_run_timestamp": "2026-09-26T08:05:12.761Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.7713334, "accuracy_std_dev": 0.0331394434343729, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.122264977, "avg_duration_per_task_ms": 407781.64, "avg_output_tokens_per_task": 17384.362, "last_run_timestamp": "2026-08-29T17:01:22.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7712177472324724, "accuracy_std_dev": 0.052442826700422794, "run_count": 11, "total_questions": 542, "avg_cost_per_task_usd": 0.09838513653136532, "avg_duration_per_task_ms": 1843075.8265682657, "avg_output_tokens_per_task": 19082.356088560886, "last_run_timestamp": "2026-08-29T16:45:34.216Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7710438282828282, "accuracy_std_dev": 0.01667057037866429, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.1615302070707071, "avg_duration_per_task_ms": 338222.0707070707, "avg_output_tokens_per_task": 20443.252525252527, "last_run_timestamp": "2026-09-26T08:27:30.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.7706668, "accuracy_std_dev": 0.023702035544653124, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.074997112, "avg_duration_per_task_ms": 263018.86, "avg_output_tokens_per_task": 12780.056, "last_run_timestamp": "2026-09-26T08:10:10.216Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.7706668, "accuracy_std_dev": 0.01818435958069462, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.071001692, "avg_duration_per_task_ms": 34525.74, "avg_output_tokens_per_task": 4762.176, "last_run_timestamp": "2026-09-26T08:16:16.877Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.770303, "accuracy_std_dev": 0.018282610525752703, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.4931648818181818, "avg_duration_per_task_ms": 129914.10363636364, "avg_output_tokens_per_task": 8068.774545454546, "last_run_timestamp": "2026-09-26T08:08:19.694Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.7702221333333333, "accuracy_std_dev": 0.02970127552786618, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.5012201893333333, "avg_duration_per_task_ms": 129604.79333333333, "avg_output_tokens_per_task": 7921.828, "last_run_timestamp": "2026-09-26T08:23:02.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.77, "accuracy_std_dev": 0.03341663757396906, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.4393913750000001, "avg_duration_per_task_ms": 81234.3425, "avg_output_tokens_per_task": 5692.71875, "last_run_timestamp": "2026-09-26T08:16:17.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.77, "accuracy_std_dev": 0.007453485392754197, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.081881785, "avg_duration_per_task_ms": 99238.51, "avg_output_tokens_per_task": 4510.805, "last_run_timestamp": "2026-09-26T08:25:16.354Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.77, "accuracy_std_dev": 0.0033330000000000304, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.073975635, "avg_duration_per_task_ms": 45899.64, "avg_output_tokens_per_task": 4662.84, "last_run_timestamp": "2026-09-26T08:21:33.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "WandB", "accuracy": 0.7695238571428571, "accuracy_std_dev": 0.017036849485986643, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10181366857142858, "avg_duration_per_task_ms": 251127.43142857144, "avg_output_tokens_per_task": 7309.714285714285, "last_run_timestamp": "2026-07-18T08:04:04.764Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.7694109317269077, "accuracy_std_dev": 0.035180887422576496, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.05336053413654618, "avg_duration_per_task_ms": 321533.1927710843, "avg_output_tokens_per_task": 6818.439759036145, "last_run_timestamp": "2026-08-01T17:45:43.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "auto-routing", "accuracy": 0.7688889999999999, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10034226, "avg_duration_per_task_ms": 157612.40666666668, "avg_output_tokens_per_task": 8568.786666666667, "last_run_timestamp": "2026-06-30T19:14:03.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.7688888333333334, "accuracy_std_dev": 0.014229173651535601, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10475028333333332, "avg_duration_per_task_ms": 333223.94666666666, "avg_output_tokens_per_task": 20225.06333333333, "last_run_timestamp": "2026-09-26T08:21:56.720Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7688886666666668, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0562409, "avg_duration_per_task_ms": 124558.21333333333, "avg_output_tokens_per_task": 4508.253333333333, "last_run_timestamp": "2026-09-26T08:24:47.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.7687072040816327, "accuracy_std_dev": 0.0156945, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 1.7723489081632653, "avg_duration_per_task_ms": 313060.2448979592, "avg_output_tokens_per_task": 23077.85714285714, "last_run_timestamp": "2026-08-01T08:06:07.759Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.7684563154362417, "accuracy_std_dev": 0.006017520216464235, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1581116510067114, "avg_duration_per_task_ms": 537705.5637583892, "avg_output_tokens_per_task": 10979.71812080537, "last_run_timestamp": "2026-08-08T08:27:49.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.7683858338108882, "accuracy_std_dev": 0.03056258771966879, "run_count": 14, "total_questions": 698, "avg_cost_per_task_usd": 0.028291176217765036, "avg_duration_per_task_ms": 202040.56017191976, "avg_output_tokens_per_task": 8723.312320916906, "last_run_timestamp": "2026-08-29T16:33:11.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7683335, "accuracy_std_dev": 0.010929191289843909, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.09395701500000002, "avg_duration_per_task_ms": 284778.19, "avg_output_tokens_per_task": 21938.115, "last_run_timestamp": "2026-09-26T08:23:03.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.76833325, "accuracy_std_dev": 0.02048026828284971, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.4290680700000001, "avg_duration_per_task_ms": 101484.8275, "avg_output_tokens_per_task": 5562.66, "last_run_timestamp": "2026-09-26T08:11:43.281Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.76833325, "accuracy_std_dev": 0.02074978581064154, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.062534375, "avg_duration_per_task_ms": 51740.385, "avg_output_tokens_per_task": 4651.035, "last_run_timestamp": "2026-09-26T08:14:03.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.76799996, "accuracy_std_dev": 0.03859765703923492, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.0899618072, "avg_duration_per_task_ms": 144055.6648, "avg_output_tokens_per_task": 5923.9224, "last_run_timestamp": "2026-08-29T17:16:15.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7678431764705882, "accuracy_std_dev": 0.037450497276578024, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.13259897294117645, "avg_duration_per_task_ms": 176390.52823529413, "avg_output_tokens_per_task": 6495.158823529412, "last_run_timestamp": "2026-08-29T16:12:45.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.7673684210526316, "accuracy_std_dev": 0.027435852827077042, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.06853673368421052, "avg_duration_per_task_ms": 181102.96, "avg_output_tokens_per_task": 7826.743157894737, "last_run_timestamp": "2026-08-29T16:59:11.598Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7671691306532663, "accuracy_std_dev": 0.03712240837960737, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.11847029773869346, "avg_duration_per_task_ms": 600977.8140703518, "avg_output_tokens_per_task": 7982.221105527638, "last_run_timestamp": "2026-08-29T16:54:14.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.7671155218855219, "accuracy_std_dev": 0.020370521752577246, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.04689751515151515, "avg_duration_per_task_ms": 143803.91919191918, "avg_output_tokens_per_task": 4429.346801346801, "last_run_timestamp": "2026-09-26T08:24:40.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.7669025283018868, "accuracy_std_dev": 0.01965390363525289, "run_count": 17, "total_questions": 848, "avg_cost_per_task_usd": 0.06553669221698114, "avg_duration_per_task_ms": 138580.51533018867, "avg_output_tokens_per_task": 8122.042452830188, "last_run_timestamp": "2026-08-29T16:41:08.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.025223939999999997, "avg_duration_per_task_ms": 165137.1, "avg_output_tokens_per_task": 14240.82, "last_run_timestamp": "2026-09-15T00:00:05.846Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.86423468, "avg_duration_per_task_ms": 335268.18, "avg_output_tokens_per_task": 22931.04, "last_run_timestamp": "2026-08-01T08:05:35.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.020366930350175255, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.1362619733333333, "avg_duration_per_task_ms": 318797.34, "avg_output_tokens_per_task": 19441.493333333332, "last_run_timestamp": "2026-09-26T08:05:12.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12935825333333334, "avg_duration_per_task_ms": 58122.44, "avg_output_tokens_per_task": 4752.84, "last_run_timestamp": "2026-09-19T08:15:57.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7666665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.21096532, "avg_duration_per_task_ms": 124417.44, "avg_output_tokens_per_task": 5140.16, "last_run_timestamp": "2026-09-01T12:59:37.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.7661904285714286, "accuracy_std_dev": 0.01872337959311179, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.019808859999999998, "avg_duration_per_task_ms": 510174.85142857145, "avg_output_tokens_per_task": 13353.934285714286, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AkashML", "accuracy": 0.766, "accuracy_std_dev": 0.0200997512422418, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.073923902, "avg_duration_per_task_ms": 82060.868, "avg_output_tokens_per_task": 4072.032, "last_run_timestamp": "2026-06-13T11:18:01.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7659570000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.006887468085106384, "avg_duration_per_task_ms": 773525.340425532, "avg_output_tokens_per_task": 9139.936170212766, "last_run_timestamp": "2026-09-22T21:00:04.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.7658824117647058, "accuracy_std_dev": 0.041720086264163854, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.010050117647058824, "avg_duration_per_task_ms": 111496.44235294117, "avg_output_tokens_per_task": 10075.014117647059, "last_run_timestamp": "2026-08-29T16:53:19.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.7653968095238095, "accuracy_std_dev": 0.03808201053443309, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.051203074285714294, "avg_duration_per_task_ms": 167458.42666666667, "avg_output_tokens_per_task": 7624.500952380952, "last_run_timestamp": "2026-08-29T17:02:01.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.7653333999999999, "accuracy_std_dev": 0.01995535461574161, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.233724332, "avg_duration_per_task_ms": 181237.788, "avg_output_tokens_per_task": 6561.384, "last_run_timestamp": "2026-07-18T08:04:09.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.765217347826087, "accuracy_std_dev": 0.020893702830421756, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.5270942373913046, "avg_duration_per_task_ms": 123051.67565217392, "avg_output_tokens_per_task": 8091.90347826087, "last_run_timestamp": "2026-09-26T08:26:39.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.7650793809523809, "accuracy_std_dev": 0.034969675378037926, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.010378357142857146, "avg_duration_per_task_ms": 113551.27047619048, "avg_output_tokens_per_task": 9004.36761904762, "last_run_timestamp": "2026-08-29T17:17:18.830Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7644446666666667, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13677892666666666, "avg_duration_per_task_ms": 241869.39333333334, "avg_output_tokens_per_task": 11667.466666666667, "last_run_timestamp": "2026-08-08T08:29:31.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.7644444444444445, "accuracy_std_dev": 0.020608041101101583, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.027644917777777778, "avg_duration_per_task_ms": 219174.32, "avg_output_tokens_per_task": 5612.551111111111, "last_run_timestamp": "2026-06-20T10:37:11.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.764166625, "accuracy_std_dev": 0.030081035231676705, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.56641688625, "avg_duration_per_task_ms": 181084.34625, "avg_output_tokens_per_task": 10579.67875, "last_run_timestamp": "2026-09-26T08:14:04.078Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.7641026153846153, "accuracy_std_dev": 0.022423789261676347, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.042344887692307695, "avg_duration_per_task_ms": 152484.15384615384, "avg_output_tokens_per_task": 9627.38153846154, "last_run_timestamp": "2026-08-29T17:07:19.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.7639931779448622, "accuracy_std_dev": 0.027369852484074517, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.0707108245614035, "avg_duration_per_task_ms": 77146.8395989975, "avg_output_tokens_per_task": 13251.315789473685, "last_run_timestamp": "2026-08-29T17:41:16.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.7633335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.34846716, "avg_duration_per_task_ms": 136967.1, "avg_output_tokens_per_task": 12613.18, "last_run_timestamp": "2026-09-01T17:30:16.052Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.017743016363840187, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.018957181666666663, "avg_duration_per_task_ms": 239454.035, "avg_output_tokens_per_task": 17272.2, "last_run_timestamp": "2026-09-26T08:11:43.368Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.005773358354328977, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.039311875, "avg_duration_per_task_ms": 88152.73, "avg_output_tokens_per_task": 4482.07, "last_run_timestamp": "2026-09-26T08:07:18.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7633331666666667, "accuracy_std_dev": 0.02631724543853748, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.5436579066666667, "avg_duration_per_task_ms": 181008.31666666668, "avg_output_tokens_per_task": 7997.5633333333335, "last_run_timestamp": "2026-09-01T13:02:58.914Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.7631372941176471, "accuracy_std_dev": 0.035211220731239695, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.05713043764705882, "avg_duration_per_task_ms": 206801.39411764705, "avg_output_tokens_per_task": 9987.18, "last_run_timestamp": "2026-08-29T17:42:47.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.7626666, "accuracy_std_dev": 0.016110821761784848, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.09137509199999999, "avg_duration_per_task_ms": 332031.4, "avg_output_tokens_per_task": 19933.684, "last_run_timestamp": "2026-09-26T08:21:56.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7626666, "accuracy_std_dev": 0.03227647784749755, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.146569104, "avg_duration_per_task_ms": 373419.58, "avg_output_tokens_per_task": 24831.152, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.7626262929292928, "accuracy_std_dev": 0.02553140404849484, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.10055969696969695, "avg_duration_per_task_ms": 35430.38888888889, "avg_output_tokens_per_task": 3961.580808080808, "last_run_timestamp": "2026-09-26T08:30:51.947Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.7626261818181818, "accuracy_std_dev": 0.028670609588697295, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.021812868686868686, "avg_duration_per_task_ms": 396305.23232323234, "avg_output_tokens_per_task": 9161.318181818182, "last_run_timestamp": "2026-08-01T08:05:58.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.025141417307348132, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08369306, "avg_duration_per_task_ms": 261571.12, "avg_output_tokens_per_task": 19541.486666666668, "last_run_timestamp": "2026-09-26T08:16:17.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.013698571612479268, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06797058666666667, "avg_duration_per_task_ms": 364277.74, "avg_output_tokens_per_task": 20919.446666666667, "last_run_timestamp": "2026-09-26T08:05:25.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.006285707880227596, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.1522076066666667, "avg_duration_per_task_ms": 298029.9066666667, "avg_output_tokens_per_task": 19805.7, "last_run_timestamp": "2026-09-26T08:05:12.730Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.021998563367234276, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3364002133333333, "avg_duration_per_task_ms": 240112.26, "avg_output_tokens_per_task": 11849.62, "last_run_timestamp": "2026-09-01T13:04:16.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.013698571612479268, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09941037999999999, "avg_duration_per_task_ms": 137232.76666666666, "avg_output_tokens_per_task": 5643.006666666667, "last_run_timestamp": "2026-09-26T08:22:37.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.76222225, "accuracy_std_dev": 0.015234723173970049, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.016681795000000003, "avg_duration_per_task_ms": 202557.275, "avg_output_tokens_per_task": 19520.191666666666, "last_run_timestamp": "2026-09-26T08:22:26.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7622222083333333, "accuracy_std_dev": 0.03818408546434319, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.03669861666666666, "avg_duration_per_task_ms": 129387.95, "avg_output_tokens_per_task": 6015.355, "last_run_timestamp": "2026-08-29T16:11:46.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.7619999000000001, "accuracy_std_dev": 0.014313938818159046, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.008990316, "avg_duration_per_task_ms": 356282.446, "avg_output_tokens_per_task": 14140.9, "last_run_timestamp": "2026-09-26T08:22:54.418Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.7619049795918367, "accuracy_std_dev": 0.01847199999999999, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.1408940612244898, "avg_duration_per_task_ms": 345711.98979591834, "avg_output_tokens_per_task": 8276.57142857143, "last_run_timestamp": "2026-08-01T08:06:00.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7619047142857143, "accuracy_std_dev": 0.02487158128532752, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03577183142857143, "avg_duration_per_task_ms": 462004.73142857145, "avg_output_tokens_per_task": 13726.342857142858, "last_run_timestamp": "2026-09-26T08:07:18.969Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.76166675, "accuracy_std_dev": 0.028431218171008805, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.051254930000000004, "avg_duration_per_task_ms": 163368.405, "avg_output_tokens_per_task": 4534.255, "last_run_timestamp": "2026-09-26T08:24:09.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.76166675, "accuracy_std_dev": 0.025110926131616508, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06906778, "avg_duration_per_task_ms": 159996.095, "avg_output_tokens_per_task": 6808.63, "last_run_timestamp": "2026-09-26T08:22:24.923Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Ambient", "accuracy": 0.7616666666666667, "accuracy_std_dev": 0.03782268572636741, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09442261499999999, "avg_duration_per_task_ms": 188047.78333333333, "avg_output_tokens_per_task": 3614.1383333333333, "last_run_timestamp": "2026-06-20T09:40:45.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.761666625, "accuracy_std_dev": 0.020749836011023693, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.4161910325, "avg_duration_per_task_ms": 98455.6425, "avg_output_tokens_per_task": 5629.3925, "last_run_timestamp": "2026-09-26T08:10:40.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.7614156780821918, "accuracy_std_dev": 0.0228848139730987, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.024887965753424657, "avg_duration_per_task_ms": 458119.4315068493, "avg_output_tokens_per_task": 13217.239726027397, "last_run_timestamp": "2026-08-29T16:59:36.100Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.7612120909090909, "accuracy_std_dev": 0.02825828791202695, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5180262836363636, "avg_duration_per_task_ms": 222126.0781818182, "avg_output_tokens_per_task": 10227.196363636363, "last_run_timestamp": "2026-09-26T08:22:25.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7610441927710843, "accuracy_std_dev": 0.04002571797891325, "run_count": 20, "total_questions": 996, "avg_cost_per_task_usd": 0.016136098393574293, "avg_duration_per_task_ms": 328656.0351405622, "avg_output_tokens_per_task": 8275.835341365462, "last_run_timestamp": "2026-08-29T16:33:19.761Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.7610153534482759, "accuracy_std_dev": 0.044412661393182805, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.06370122844827586, "avg_duration_per_task_ms": 278513.2284482759, "avg_output_tokens_per_task": 9221.025862068966, "last_run_timestamp": "2026-08-29T17:06:46.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.7609524285714285, "accuracy_std_dev": 0.018662793573288653, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.021895571428571427, "avg_duration_per_task_ms": 241457.5, "avg_output_tokens_per_task": 8715.474285714286, "last_run_timestamp": "2026-07-18T08:03:40.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.7609522857142857, "accuracy_std_dev": 0.017969598920035443, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08491382571428571, "avg_duration_per_task_ms": 105368.18285714286, "avg_output_tokens_per_task": 24364.49142857143, "last_run_timestamp": "2026-09-01T17:31:46.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.7609424343434344, "accuracy_std_dev": 0.012517, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.02651870707070707, "avg_duration_per_task_ms": 84229.25252525252, "avg_output_tokens_per_task": 17462.434343434343, "last_run_timestamp": "2026-09-26T08:10:56.841Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.760833625, "accuracy_std_dev": 0.018390020316312206, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.4986317424999999, "avg_duration_per_task_ms": 132839.435, "avg_output_tokens_per_task": 8114.2475, "last_run_timestamp": "2026-09-26T08:16:12.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.7605556666666666, "accuracy_std_dev": 0.013391060222734018, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.12239992333333333, "avg_duration_per_task_ms": 458072.06, "avg_output_tokens_per_task": 22242.516666666666, "last_run_timestamp": "2026-09-26T08:22:24.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.7604167291666667, "accuracy_std_dev": 0.017826499999999967, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.2660930625, "avg_duration_per_task_ms": 146365.54166666666, "avg_output_tokens_per_task": 6163.291666666667, "last_run_timestamp": "2026-09-26T08:05:12.774Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.7603703333333334, "accuracy_std_dev": 0.028822460188771784, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.056350817777777784, "avg_duration_per_task_ms": 98825.95333333334, "avg_output_tokens_per_task": 4242.228888888889, "last_run_timestamp": "2026-09-26T08:32:07.081Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.7602777916666666, "accuracy_std_dev": 0.03482648069713999, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.11371839333333329, "avg_duration_per_task_ms": 294570.1925, "avg_output_tokens_per_task": 17313.15166666667, "last_run_timestamp": "2026-08-29T16:25:47.585Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.7601780858778625, "accuracy_std_dev": 0.02811995722322186, "run_count": 21, "total_questions": 1048, "avg_cost_per_task_usd": 0.0422277356870229, "avg_duration_per_task_ms": 162724.19942748093, "avg_output_tokens_per_task": 7589.8158396946565, "last_run_timestamp": "2026-08-29T16:25:15.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.7600423065539111, "accuracy_std_dev": 0.030704280834836183, "run_count": 19, "total_questions": 946, "avg_cost_per_task_usd": 0.08771633932346722, "avg_duration_per_task_ms": 364680.0031712474, "avg_output_tokens_per_task": 16580.900634249472, "last_run_timestamp": "2026-08-15T14:15:18.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01747962, "avg_duration_per_task_ms": 589736.28, "avg_output_tokens_per_task": 13675.16, "last_run_timestamp": "2026-08-01T08:06:01.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.76, "accuracy_std_dev": 0.034318637633216124, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03080999, "avg_duration_per_task_ms": 262760.8, "avg_output_tokens_per_task": 18663.095, "last_run_timestamp": "2026-09-26T08:09:25.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.76, "accuracy_std_dev": 0.033333, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11355728, "avg_duration_per_task_ms": 147828.6, "avg_output_tokens_per_task": 17720.73, "last_run_timestamp": "2026-09-01T17:30:58.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Venice", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19103682, "avg_duration_per_task_ms": 233230.24, "avg_output_tokens_per_task": 15236.66, "last_run_timestamp": "2026-08-04T03:12:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.76, "accuracy_std_dev": 0.020727419665747104, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.0731778, "avg_duration_per_task_ms": 258357.41, "avg_output_tokens_per_task": 11852.263333333334, "last_run_timestamp": "2026-09-26T08:07:46.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.76, "accuracy_std_dev": 0.038682027601113875, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07101471333333334, "avg_duration_per_task_ms": 318099.35333333333, "avg_output_tokens_per_task": 12493.926666666666, "last_run_timestamp": "2026-09-26T08:05:12.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.35068044, "avg_duration_per_task_ms": 248200.34, "avg_output_tokens_per_task": 11571.02, "last_run_timestamp": "2026-09-01T12:41:31.788Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.027619090000000002, "avg_duration_per_task_ms": 42870.03, "avg_output_tokens_per_task": 4628.08, "last_run_timestamp": "2026-09-26T09:28:27.720Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.76, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04223252, "avg_duration_per_task_ms": 69942.84, "avg_output_tokens_per_task": 4621.51, "last_run_timestamp": "2026-09-26T08:05:12.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.76, "accuracy_std_dev": 0.027371610829992666, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.009037317142857142, "avg_duration_per_task_ms": 270209.39714285714, "avg_output_tokens_per_task": 6316.0257142857145, "last_run_timestamp": "2026-09-26T08:08:19.666Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.76, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014824000000000002, "avg_duration_per_task_ms": 335920.77, "avg_output_tokens_per_task": 4448.51, "last_run_timestamp": "2026-09-12T08:11:26.740Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7599999583333333, "accuracy_std_dev": 0.04023080402655736, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.05313657, "avg_duration_per_task_ms": 148959.73916666667, "avg_output_tokens_per_task": 5980.9625, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7594871538461538, "accuracy_std_dev": 0.03003836972132133, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.05324420692307693, "avg_duration_per_task_ms": 141495.42, "avg_output_tokens_per_task": 5929.143846153846, "last_run_timestamp": "2026-08-29T16:32:40.387Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.7592915452961673, "accuracy_std_dev": 0.05377860847392117, "run_count": 23, "total_questions": 1148, "avg_cost_per_task_usd": 0.15847917944250875, "avg_duration_per_task_ms": 312268.7691637631, "avg_output_tokens_per_task": 17697.43205574913, "last_run_timestamp": "2026-08-29T16:13:54.412Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.759166625, "accuracy_std_dev": 0.02319920826201565, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0076381949999999995, "avg_duration_per_task_ms": 261034.725, "avg_output_tokens_per_task": 15531.6525, "last_run_timestamp": "2026-09-26T08:24:41.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Phala", "accuracy": 0.759040841981132, "accuracy_std_dev": 0.036093876622690174, "run_count": 17, "total_questions": 848, "avg_cost_per_task_usd": 0.08171993514150944, "avg_duration_per_task_ms": 269188.51061320753, "avg_output_tokens_per_task": 4527.392688679245, "last_run_timestamp": "2026-07-11T15:34:16.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.758888875, "accuracy_std_dev": 0.0209643399476995, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.6800024841666666, "avg_duration_per_task_ms": 74777.63666666667, "avg_output_tokens_per_task": 5635.594166666667, "last_run_timestamp": "2026-09-26T08:23:22.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.7588888333333333, "accuracy_std_dev": 0.03578052612067383, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.4306518199999999, "avg_duration_per_task_ms": 94862.28333333334, "avg_output_tokens_per_task": 5554.29, "last_run_timestamp": "2026-09-26T08:25:11.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7588406086956522, "accuracy_std_dev": 0.021414040391536566, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.59698552, "avg_duration_per_task_ms": 123337.09304347826, "avg_output_tokens_per_task": 8161.509565217391, "last_run_timestamp": "2026-09-26T08:22:57.486Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7586668, "accuracy_std_dev": 0.01654634971708262, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.27091952399999997, "avg_duration_per_task_ms": 154225.472, "avg_output_tokens_per_task": 23085.08, "last_run_timestamp": "2026-09-01T17:32:40.487Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7586668, "accuracy_std_dev": 0.012927091279943853, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08354813200000001, "avg_duration_per_task_ms": 248803.664, "avg_output_tokens_per_task": 12064.656, "last_run_timestamp": "2026-09-26T08:08:19.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7585878072519083, "accuracy_std_dev": 0.03764932431268607, "run_count": 21, "total_questions": 1048, "avg_cost_per_task_usd": 0.0847812709923664, "avg_duration_per_task_ms": 181956.90935114503, "avg_output_tokens_per_task": 7087.411259541985, "last_run_timestamp": "2026-08-29T16:48:46.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.7585185555555556, "accuracy_std_dev": 0.01743499783711622, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.07257207555555555, "avg_duration_per_task_ms": 58870.41333333333, "avg_output_tokens_per_task": 4565.811111111111, "last_run_timestamp": "2026-09-26T08:26:14.308Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.015184156584660873, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.163021, "avg_duration_per_task_ms": 227043.59, "avg_output_tokens_per_task": 11333.075, "last_run_timestamp": "2026-08-29T16:55:33.128Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.022298623526744884, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025212460000000003, "avg_duration_per_task_ms": 129050.295, "avg_output_tokens_per_task": 16100.67, "last_run_timestamp": "2026-09-26T08:22:17.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7583331666666667, "accuracy_std_dev": 0.013844473427047428, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09591996333333334, "avg_duration_per_task_ms": 214673.9, "avg_output_tokens_per_task": 9689.54, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.758, "accuracy_std_dev": 0.03709447398198281, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.079319272, "avg_duration_per_task_ms": 620750.224, "avg_output_tokens_per_task": 10895.356, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.7579964141414142, "accuracy_std_dev": 0.023533015514113357, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.20713992929292932, "avg_duration_per_task_ms": 664979.7727272727, "avg_output_tokens_per_task": 53099.67929292929, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.7577778333333334, "accuracy_std_dev": 0.037646776132079215, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08977579333333333, "avg_duration_per_task_ms": 61527.54, "avg_output_tokens_per_task": 12239.113333333333, "last_run_timestamp": "2026-08-29T16:25:14.949Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7575, "accuracy_std_dev": 0.023318483371565994, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.34416930749999997, "avg_duration_per_task_ms": 221562.1725, "avg_output_tokens_per_task": 11914.215, "last_run_timestamp": "2026-09-26T08:22:18.965Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.7574602857142857, "accuracy_std_dev": 0.030548852532643923, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.010414794285714285, "avg_duration_per_task_ms": 137613.95523809525, "avg_output_tokens_per_task": 9270.931428571428, "last_run_timestamp": "2026-08-29T16:48:14.107Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.7573335999999999, "accuracy_std_dev": 0.017178865569064807, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.151421216, "avg_duration_per_task_ms": 315167.644, "avg_output_tokens_per_task": 20253, "last_run_timestamp": "2026-09-26T08:10:25.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.031439640936880967, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.081021904, "avg_duration_per_task_ms": 42035.076, "avg_output_tokens_per_task": 4841.128, "last_run_timestamp": "2026-09-26T08:26:16.831Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.756944, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.2442584375, "avg_duration_per_task_ms": 645873.5208333334, "avg_output_tokens_per_task": 12350.729166666666, "last_run_timestamp": "2026-09-26T08:23:04.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7568027346938776, "accuracy_std_dev": 0.038809665680729605, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.3020268469387756, "avg_duration_per_task_ms": 145185.1581632653, "avg_output_tokens_per_task": 20600.117346938776, "last_run_timestamp": "2026-09-01T17:31:47.860Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7567114127516779, "accuracy_std_dev": 0.02249334300085622, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.020425387583892618, "avg_duration_per_task_ms": 519535.90939597314, "avg_output_tokens_per_task": 17284.703020134228, "last_run_timestamp": "2026-09-26T08:07:18.967Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01828589, "avg_duration_per_task_ms": 240722.54, "avg_output_tokens_per_task": 15147.8, "last_run_timestamp": "2026-09-19T08:02:39.158Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.7566669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05416404, "avg_duration_per_task_ms": 1008004.72, "avg_output_tokens_per_task": 38151.6, "last_run_timestamp": "2026-09-26T00:00:04.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.7566667499999999, "accuracy_std_dev": 0.013743584364622665, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.33711718500000004, "avg_duration_per_task_ms": 415294.135, "avg_output_tokens_per_task": 10214.705, "last_run_timestamp": "2026-09-26T08:13:27.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7566665, "accuracy_std_dev": 0.02687425451214104, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.54838445, "avg_duration_per_task_ms": 178864.39333333334, "avg_output_tokens_per_task": 10795.433333333332, "last_run_timestamp": "2026-09-26T08:26:47.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7566665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10920926999999998, "avg_duration_per_task_ms": 101251.93, "avg_output_tokens_per_task": 25100.99, "last_run_timestamp": "2026-09-01T17:23:16.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.7566665, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.42697405000000005, "avg_duration_per_task_ms": 522179.69, "avg_output_tokens_per_task": 37590.96, "last_run_timestamp": "2026-09-01T12:41:24.474Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.7566665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29550982, "avg_duration_per_task_ms": 152697.48, "avg_output_tokens_per_task": 10806.55, "last_run_timestamp": "2026-09-01T17:30:15.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.7564705294117647, "accuracy_std_dev": 0.03521125545336429, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.008393454117647059, "avg_duration_per_task_ms": 146412.67529411765, "avg_output_tokens_per_task": 9095.61294117647, "last_run_timestamp": "2026-08-29T16:54:19.718Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.756, "accuracy_std_dev": 0.012365076271499492, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.0694329, "avg_duration_per_task_ms": 122849.852, "avg_output_tokens_per_task": 4849.072, "last_run_timestamp": "2026-09-26T08:26:14.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.7559259444444444, "accuracy_std_dev": 0.03443646283531493, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.05180537888888888, "avg_duration_per_task_ms": 157203.02777777778, "avg_output_tokens_per_task": 6346.41, "last_run_timestamp": "2026-08-29T17:22:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.755919868852459, "accuracy_std_dev": 0.04303119426679743, "run_count": 22, "total_questions": 1098, "avg_cost_per_task_usd": 0.054472353369763214, "avg_duration_per_task_ms": 164605.7941712204, "avg_output_tokens_per_task": 6348.341530054645, "last_run_timestamp": "2026-08-29T16:45:34.327Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.7556390701754386, "accuracy_std_dev": 0.0748655823238389, "run_count": 16, "total_questions": 798, "avg_cost_per_task_usd": 0.1314042468671679, "avg_duration_per_task_ms": 282979.45112781954, "avg_output_tokens_per_task": 13309.800751879699, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.7555556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05382074666666667, "avg_duration_per_task_ms": 90716.92666666667, "avg_output_tokens_per_task": 14804.206666666667, "last_run_timestamp": "2026-09-26T08:22:43.308Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.7555556666666666, "accuracy_std_dev": 0.03823241466202323, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0327312, "avg_duration_per_task_ms": 153585.12, "avg_output_tokens_per_task": 15537.24, "last_run_timestamp": "2026-09-26T08:17:02.789Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.7555556666666666, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05866996, "avg_duration_per_task_ms": 356562.5266666667, "avg_output_tokens_per_task": 13311.113333333333, "last_run_timestamp": "2026-09-26T08:12:56.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.7555556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06505471333333333, "avg_duration_per_task_ms": 157291.16, "avg_output_tokens_per_task": 14450.066666666668, "last_run_timestamp": "2026-09-26T08:05:12.747Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7553623043478261, "accuracy_std_dev": 0.032979826821380405, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.19905160782608697, "avg_duration_per_task_ms": 122704.35391304348, "avg_output_tokens_per_task": 7790.118260869565, "last_run_timestamp": "2026-09-26T08:11:40.828Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.7552381428571429, "accuracy_std_dev": 0.009897295726302106, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03504835714285714, "avg_duration_per_task_ms": 271800.3542857143, "avg_output_tokens_per_task": 13667.517142857143, "last_run_timestamp": "2026-09-26T08:10:40.746Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7551515454545454, "accuracy_std_dev": 0.023968159933786534, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.024316309090909087, "avg_duration_per_task_ms": 165689.23272727273, "avg_output_tokens_per_task": 18155.507272727275, "last_run_timestamp": "2026-09-26T08:08:49.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.755, "accuracy_std_dev": 0.04773577934491767, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.04855596, "avg_duration_per_task_ms": 135012.90833333333, "avg_output_tokens_per_task": 6055.553333333333, "last_run_timestamp": "2026-08-29T16:58:32.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.755, "accuracy_std_dev": 0.017240198505237704, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.055826839999999996, "avg_duration_per_task_ms": 122326.96, "avg_output_tokens_per_task": 4308.39, "last_run_timestamp": "2026-09-26T08:26:39.685Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.7547709770992367, "accuracy_std_dev": 0.037708741308906626, "run_count": 21, "total_questions": 1048, "avg_cost_per_task_usd": 0.06855190171755726, "avg_duration_per_task_ms": 232773.2967557252, "avg_output_tokens_per_task": 7463.7099236641225, "last_run_timestamp": "2026-08-29T16:33:36.999Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7546664000000001, "accuracy_std_dev": 0.04509740371950474, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.17806932799999997, "avg_duration_per_task_ms": 249010.144, "avg_output_tokens_per_task": 9041.408, "last_run_timestamp": "2026-09-26T08:26:47.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7544445, "accuracy_std_dev": 0.025939233358948773, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.19473402666666667, "avg_duration_per_task_ms": 127608.55, "avg_output_tokens_per_task": 7689.33, "last_run_timestamp": "2026-09-26T08:22:38.150Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7543859845201238, "accuracy_std_dev": 0.03746648424278414, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.06285631269349846, "avg_duration_per_task_ms": 163970.43962848297, "avg_output_tokens_per_task": 7998.698142414861, "last_run_timestamp": "2026-08-29T16:44:10.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7543103448275862, "accuracy_std_dev": 0.016399321098441732, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.04285599137931035, "avg_duration_per_task_ms": 524163.9051724138, "avg_output_tokens_per_task": 22101.97988505747, "last_run_timestamp": "2026-09-26T16:00:25.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.7542953608247424, "accuracy_std_dev": 0.02652500000000002, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.021299783505154636, "avg_duration_per_task_ms": 633495.5257731959, "avg_output_tokens_per_task": 26267.113402061856, "last_run_timestamp": "2026-08-08T08:30:53.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.754166625, "accuracy_std_dev": 0.02895150279319496, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.1985393725, "avg_duration_per_task_ms": 132141.125, "avg_output_tokens_per_task": 7833.895, "last_run_timestamp": "2026-09-26T08:08:19.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.754166625, "accuracy_std_dev": 0.0239066485989646, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02336587, "avg_duration_per_task_ms": 124888.855, "avg_output_tokens_per_task": 7646.1525, "last_run_timestamp": "2026-09-26T16:00:25.687Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7539393636363636, "accuracy_std_dev": 0.01999089692049736, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.024043370909090908, "avg_duration_per_task_ms": 299110.4909090909, "avg_output_tokens_per_task": 17382.84909090909, "last_run_timestamp": "2026-09-26T08:12:56.930Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.7538889166666667, "accuracy_std_dev": 0.03686387162127335, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.06654354, "avg_duration_per_task_ms": 244757.80833333332, "avg_output_tokens_per_task": 10635.376666666667, "last_run_timestamp": "2026-08-29T16:33:11.313Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.7538888333333333, "accuracy_std_dev": 0.033934516643169634, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.1837781666666667, "avg_duration_per_task_ms": 552485.2983333333, "avg_output_tokens_per_task": 26570.453333333335, "last_run_timestamp": "2026-09-26T08:13:27.139Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.7538536705202312, "accuracy_std_dev": 0.022320967608497434, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.20567977745664742, "avg_duration_per_task_ms": 493851.28612716764, "avg_output_tokens_per_task": 20258.49710982659, "last_run_timestamp": "2026-09-26T08:08:00.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.7533668080808081, "accuracy_std_dev": 0.00967513458239728, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.0209760505050505, "avg_duration_per_task_ms": 99025.76767676767, "avg_output_tokens_per_task": 14135.353535353535, "last_run_timestamp": "2026-09-26T08:05:09.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5044717400000001, "avg_duration_per_task_ms": 153665.59, "avg_output_tokens_per_task": 8278.2, "last_run_timestamp": "2026-09-01T12:41:31.807Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.014142371326973414, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.19823152, "avg_duration_per_task_ms": 359205.05, "avg_output_tokens_per_task": 12583.985, "last_run_timestamp": "2026-09-26T08:10:43.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.7533333999999999, "accuracy_std_dev": 0.01738441192102859, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.019357872, "avg_duration_per_task_ms": 107397.768, "avg_output_tokens_per_task": 16129.496, "last_run_timestamp": "2026-09-26T08:24:48.615Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.7533333999999999, "accuracy_std_dev": 0.0193218356643462, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.037210308, "avg_duration_per_task_ms": 202748.952, "avg_output_tokens_per_task": 17567.136, "last_run_timestamp": "2026-09-26T08:23:54.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021082753333333332, "avg_duration_per_task_ms": 915986.2133333334, "avg_output_tokens_per_task": 13268.026666666667, "last_run_timestamp": "2026-08-15T08:15:52.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.753333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.36418883999999996, "avg_duration_per_task_ms": 138377.24, "avg_output_tokens_per_task": 12769.81, "last_run_timestamp": "2026-09-01T17:31:47.883Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.41880896, "avg_duration_per_task_ms": 447303.41, "avg_output_tokens_per_task": 36551.01, "last_run_timestamp": "2026-09-01T12:59:50.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.6857741799999999, "avg_duration_per_task_ms": 105619.36, "avg_output_tokens_per_task": 7597.08, "last_run_timestamp": "2026-09-01T13:02:16.941Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0196027, "avg_duration_per_task_ms": 76385.02, "avg_output_tokens_per_task": 9950, "last_run_timestamp": "2026-07-18T08:04:02.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10139794, "avg_duration_per_task_ms": 119629.34, "avg_output_tokens_per_task": 9506.02, "last_run_timestamp": "2026-08-01T17:46:04.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2831028, "avg_duration_per_task_ms": 544501.1, "avg_output_tokens_per_task": 8096.24, "last_run_timestamp": "2026-08-23T18:00:01.881Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09723594, "avg_duration_per_task_ms": 178772.1, "avg_output_tokens_per_task": 8310.58, "last_run_timestamp": "2026-08-01T08:05:58.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7525925555555556, "accuracy_std_dev": 0.02463451135257706, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021416855555555555, "avg_duration_per_task_ms": 123014.89111111111, "avg_output_tokens_per_task": 15302.471111111112, "last_run_timestamp": "2026-09-26T08:21:23.410Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.752499875, "accuracy_std_dev": 0.04128993437400179, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.19582053000000002, "avg_duration_per_task_ms": 106281.8925, "avg_output_tokens_per_task": 7612.88, "last_run_timestamp": "2026-09-26T08:23:00.199Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.7524445333333333, "accuracy_std_dev": 0.03730686310384309, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.19597836800000001, "avg_duration_per_task_ms": 104969.73733333334, "avg_output_tokens_per_task": 7646.229333333334, "last_run_timestamp": "2026-09-26T08:12:13.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.752, "accuracy_std_dev": 0.015434478002187193, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.036405735999999994, "avg_duration_per_task_ms": 202702.796, "avg_output_tokens_per_task": 18338.232, "last_run_timestamp": "2026-09-26T08:01:18.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Ambient", "accuracy": 0.752, "accuracy_std_dev": 0.02400000000000002, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.031335448, "avg_duration_per_task_ms": 314672.816, "avg_output_tokens_per_task": 20249.848, "last_run_timestamp": "2026-06-13T19:57:55.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.7519999333333334, "accuracy_std_dev": 0.042457799179054144, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.07472457066666666, "avg_duration_per_task_ms": 97051.48533333333, "avg_output_tokens_per_task": 11508.92, "last_run_timestamp": "2026-07-18T08:03:41.319Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.7517460476190476, "accuracy_std_dev": 0.03886781145721899, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.0710823885714286, "avg_duration_per_task_ms": 177196.37523809524, "avg_output_tokens_per_task": 5993.54, "last_run_timestamp": "2026-08-29T16:12:10.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.7516667499999999, "accuracy_std_dev": 0.009860203404975986, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07850260499999999, "avg_duration_per_task_ms": 143951.215, "avg_output_tokens_per_task": 4587.75, "last_run_timestamp": "2026-09-26T08:22:24.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.7516667499999999, "accuracy_std_dev": 0.0172401743375031, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.14700563, "avg_duration_per_task_ms": 224982.975, "avg_output_tokens_per_task": 8550.835, "last_run_timestamp": "2026-09-26T08:08:19.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.751666625, "accuracy_std_dev": 0.02598077013358873, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.54754274, "avg_duration_per_task_ms": 191361.1975, "avg_output_tokens_per_task": 10587.185, "last_run_timestamp": "2026-09-26T08:16:15.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.7516665, "accuracy_std_dev": 0.03378863221188454, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025036024999999996, "avg_duration_per_task_ms": 332301.835, "avg_output_tokens_per_task": 13337.575, "last_run_timestamp": "2026-08-29T17:19:05.764Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7516665, "accuracy_std_dev": 0.03484088434942489, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017483123333333336, "avg_duration_per_task_ms": 78588.8, "avg_output_tokens_per_task": 4208.683333333333, "last_run_timestamp": "2026-09-26T08:25:00.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.7513043478260869, "accuracy_std_dev": 0.033096829546845286, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.008512693043478257, "avg_duration_per_task_ms": 141943.57217391304, "avg_output_tokens_per_task": 8289.525217391305, "last_run_timestamp": "2026-08-29T16:33:30.698Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.7511110833333333, "accuracy_std_dev": 0.016629606947942455, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.051246248333333334, "avg_duration_per_task_ms": 276941.145, "avg_output_tokens_per_task": 29769.108333333334, "last_run_timestamp": "2026-09-26T16:00:20.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7510040481927711, "accuracy_std_dev": 0.022883894021997232, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.007396576305220883, "avg_duration_per_task_ms": 255494.20682730924, "avg_output_tokens_per_task": 11511.991967871487, "last_run_timestamp": "2026-08-29T17:00:01.341Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.7506668, "accuracy_std_dev": 0.027840878020637227, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08836137599999999, "avg_duration_per_task_ms": 290555.676, "avg_output_tokens_per_task": 12011.224, "last_run_timestamp": "2026-09-26T08:10:43.592Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.7506666, "accuracy_std_dev": 0.029090015088342618, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.008165608, "avg_duration_per_task_ms": 191715.892, "avg_output_tokens_per_task": 4404.68, "last_run_timestamp": "2026-09-26T08:23:40.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.7505555833333333, "accuracy_std_dev": 0.037757344555857254, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.016059435, "avg_duration_per_task_ms": 293963.405, "avg_output_tokens_per_task": 5996.211666666667, "last_run_timestamp": "2026-07-18T08:03:38.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.7505555, "accuracy_std_dev": 0.03274012621004996, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02124632666666667, "avg_duration_per_task_ms": 354797.86666666664, "avg_output_tokens_per_task": 18494.39, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7502796040268457, "accuracy_std_dev": 0.04400657815221114, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.11341485906040269, "avg_duration_per_task_ms": 467292.23825503356, "avg_output_tokens_per_task": 23614.795302013423, "last_run_timestamp": "2026-09-26T08:23:03.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.7500002567567566, "accuracy_std_dev": 0.007287030686241284, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.009324743243243245, "avg_duration_per_task_ms": 215748.13513513515, "avg_output_tokens_per_task": 3777.3783783783783, "last_run_timestamp": "2026-09-26T08:06:43.931Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2253275, "avg_duration_per_task_ms": 98359.26, "avg_output_tokens_per_task": 5434.86, "last_run_timestamp": "2026-09-01T13:03:25.552Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.75, "accuracy_std_dev": 0.027688836500474354, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.5338496725, "avg_duration_per_task_ms": 189310.245, "avg_output_tokens_per_task": 10232.6975, "last_run_timestamp": "2026-09-26T08:09:25.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Nebius", "accuracy": 0.75, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11403106000000002, "avg_duration_per_task_ms": 126507.38, "avg_output_tokens_per_task": 4905.84, "last_run_timestamp": "2026-06-20T10:57:18.552Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.75, "accuracy_std_dev": 0.02839314819146025, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.035507067777777776, "avg_duration_per_task_ms": 259194.36444444445, "avg_output_tokens_per_task": 5673.533333333334, "last_run_timestamp": "2026-08-29T16:10:33.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.75, "accuracy_std_dev": 0.020275820303997587, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0059813, "avg_duration_per_task_ms": 146386.56, "avg_output_tokens_per_task": 4218.225, "last_run_timestamp": "2026-09-26T08:22:40.372Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.7499999583333333, "accuracy_std_dev": 0.0368179040090995, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.2266885883333333, "avg_duration_per_task_ms": 100830.78416666666, "avg_output_tokens_per_task": 7512.6875, "last_run_timestamp": "2026-09-26T08:24:10.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.7496296666666666, "accuracy_std_dev": 0.01780858133971749, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021206060000000002, "avg_duration_per_task_ms": 122947.83777777778, "avg_output_tokens_per_task": 16271.093333333334, "last_run_timestamp": "2026-09-26T08:08:19.683Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.7493332, "accuracy_std_dev": 0.01162353169049754, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.172436168, "avg_duration_per_task_ms": 308861.672, "avg_output_tokens_per_task": 10652.584, "last_run_timestamp": "2026-09-26T08:22:58.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.748889, "accuracy_std_dev": 0.01257094435593444, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05290565333333333, "avg_duration_per_task_ms": 112293.75333333333, "avg_output_tokens_per_task": 4801.486666666667, "last_run_timestamp": "2026-09-26T08:17:02.746Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.748889, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01121336, "avg_duration_per_task_ms": 201242.51333333334, "avg_output_tokens_per_task": 4454.833333333333, "last_run_timestamp": "2026-09-26T08:24:26.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.7488585479452055, "accuracy_std_dev": 0.02333557629219577, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.13015020547945205, "avg_duration_per_task_ms": 94301.3698630137, "avg_output_tokens_per_task": 9432.931506849314, "last_run_timestamp": "2026-07-18T08:04:45.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7485715714285714, "accuracy_std_dev": 0.030386743739697415, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.27195099142857143, "avg_duration_per_task_ms": 529335.9085714286, "avg_output_tokens_per_task": 9553.417142857143, "last_run_timestamp": "2026-08-29T16:51:55.567Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.748333375, "accuracy_std_dev": 0.027233626580651628, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.488940975, "avg_duration_per_task_ms": 194699.49, "avg_output_tokens_per_task": 10000.255, "last_run_timestamp": "2026-09-26T08:11:43.276Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.7483332500000001, "accuracy_std_dev": 0.012802094015726486, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17867437000000003, "avg_duration_per_task_ms": 156388.555, "avg_output_tokens_per_task": 12772.235, "last_run_timestamp": "2026-09-26T08:22:52.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.7481202957393484, "accuracy_std_dev": 0.01771080940943976, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.01360957142857143, "avg_duration_per_task_ms": 218299.9649122807, "avg_output_tokens_per_task": 16333.967418546366, "last_run_timestamp": "2026-09-26T08:20:57.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.748116, "accuracy_std_dev": 0.022837863998193896, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.37004660695652175, "avg_duration_per_task_ms": 167935.71652173914, "avg_output_tokens_per_task": 11884.72, "last_run_timestamp": "2026-09-26T08:22:24.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.7480002, "accuracy_std_dev": 0.014236085605249795, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.14549154800000003, "avg_duration_per_task_ms": 139795.724, "avg_output_tokens_per_task": 19960.988, "last_run_timestamp": "2026-09-26T08:09:25.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.748, "accuracy_std_dev": 0.015434218839967281, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011429584, "avg_duration_per_task_ms": 130555.472, "avg_output_tokens_per_task": 4185.616, "last_run_timestamp": "2026-09-26T08:24:45.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7479998000000001, "accuracy_std_dev": 0.023247453386554, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.025147304000000002, "avg_duration_per_task_ms": 339787.244, "avg_output_tokens_per_task": 4397.436, "last_run_timestamp": "2026-09-26T08:13:28.807Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.747916625, "accuracy_std_dev": 0.029459786032986997, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.56374661625, "avg_duration_per_task_ms": 203094.54, "avg_output_tokens_per_task": 9850.0775, "last_run_timestamp": "2026-09-26T08:23:07.274Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.7478787272727273, "accuracy_std_dev": 0.029242020409898036, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.33650350909090904, "avg_duration_per_task_ms": 209916.4818181818, "avg_output_tokens_per_task": 11777.303636363637, "last_run_timestamp": "2026-09-26T08:09:55.575Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.7477703913043479, "accuracy_std_dev": 0.01921375665647317, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.0067899197324414705, "avg_duration_per_task_ms": 182508.58026755851, "avg_output_tokens_per_task": 12210.97491638796, "last_run_timestamp": "2026-09-26T08:06:10.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.747500125, "accuracy_std_dev": 0.03323948977661021, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0216457225, "avg_duration_per_task_ms": 167096.8075, "avg_output_tokens_per_task": 13839.2125, "last_run_timestamp": "2026-09-26T08:10:10.199Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7475, "accuracy_std_dev": 0.028565091361660317, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.012148102500000002, "avg_duration_per_task_ms": 152199.3025, "avg_output_tokens_per_task": 16425.225, "last_run_timestamp": "2026-09-26T08:04:53.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.7474746666666666, "accuracy_std_dev": 0.013351080637442798, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.14427683333333333, "avg_duration_per_task_ms": 274328.10101010103, "avg_output_tokens_per_task": 7734.318181818182, "last_run_timestamp": "2026-07-18T08:04:29.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.7473334, "accuracy_std_dev": 0.02318052631067726, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.019461324000000002, "avg_duration_per_task_ms": 260422.68, "avg_output_tokens_per_task": 9034.204, "last_run_timestamp": "2026-08-08T08:29:47.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Friendli", "accuracy": 0.7471111333333333, "accuracy_std_dev": 0.1405273225224033, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.04440916933333333, "avg_duration_per_task_ms": 42335.25866666667, "avg_output_tokens_per_task": 3598.3533333333335, "last_run_timestamp": "2026-07-01T02:07:35.472Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7469239563758389, "accuracy_std_dev": 0.04294748693624446, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.1647119630872483, "avg_duration_per_task_ms": 412023.6426174497, "avg_output_tokens_per_task": 24209.687919463086, "last_run_timestamp": "2026-09-26T08:12:00.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7466670000000001, "accuracy_std_dev": 0.03999999999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.44994164999999997, "avg_duration_per_task_ms": 128787.73, "avg_output_tokens_per_task": 20836.09, "last_run_timestamp": "2026-09-01T17:32:23.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.746667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.3847332, "avg_duration_per_task_ms": 155324.72, "avg_output_tokens_per_task": 13079.53, "last_run_timestamp": "2026-09-01T17:31:47.956Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02698528, "avg_duration_per_task_ms": 147907.84, "avg_output_tokens_per_task": 11065.38, "last_run_timestamp": "2026-07-18T08:04:04.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.7466668333333333, "accuracy_std_dev": 0.02143021404790186, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.24169485, "avg_duration_per_task_ms": 277144.9666666667, "avg_output_tokens_per_task": 10442, "last_run_timestamp": "2026-09-26T08:09:25.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7466667499999999, "accuracy_std_dev": 0.03349941956493424, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.54667351, "avg_duration_per_task_ms": 183079.6925, "avg_output_tokens_per_task": 10525.565, "last_run_timestamp": "2026-09-26T08:07:18.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7466667499999999, "accuracy_std_dev": 0.014907082584714572, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12188539999999999, "avg_duration_per_task_ms": 449678.28, "avg_output_tokens_per_task": 19415.325, "last_run_timestamp": "2026-09-26T08:23:24.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.7466667499999999, "accuracy_std_dev": 0.02494415986132826, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0097815, "avg_duration_per_task_ms": 206039.625, "avg_output_tokens_per_task": 4442.965, "last_run_timestamp": "2026-09-26T08:26:29.786Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.2640489466666667, "avg_duration_per_task_ms": 261052.74666666667, "avg_output_tokens_per_task": 16845.666666666668, "last_run_timestamp": "2026-09-26T08:22:21.004Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.42183384, "avg_duration_per_task_ms": 170456.6, "avg_output_tokens_per_task": 28298.04, "last_run_timestamp": "2026-09-01T17:30:52.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.011547101609639251, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04422623666666667, "avg_duration_per_task_ms": 266653.04, "avg_output_tokens_per_task": 13360.296666666667, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.024366269999999995, "avg_duration_per_task_ms": 118191.69, "avg_output_tokens_per_task": 9023.81, "last_run_timestamp": "2026-09-26T16:00:24.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04198429, "avg_duration_per_task_ms": 312370.3, "avg_output_tokens_per_task": 24895.75, "last_run_timestamp": "2026-09-26T16:00:24.612Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.7466665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03717445, "avg_duration_per_task_ms": 350241.71, "avg_output_tokens_per_task": 12501.07, "last_run_timestamp": "2026-09-26T08:10:43.553Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.7466665, "accuracy_std_dev": 0.019999888889274717, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06283585333333332, "avg_duration_per_task_ms": 361191.69, "avg_output_tokens_per_task": 8869.28, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.7466664000000001, "accuracy_std_dev": 0.03126950946593184, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.172299032, "avg_duration_per_task_ms": 325786.904, "avg_output_tokens_per_task": 11139.324, "last_run_timestamp": "2026-09-26T08:08:19.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.7463991265432098, "accuracy_std_dev": 0.02001099180406232, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.03682306790123457, "avg_duration_per_task_ms": 266463.73302469135, "avg_output_tokens_per_task": 23459.353395061727, "last_run_timestamp": "2026-09-26T16:00:20.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7461684252873563, "accuracy_std_dev": 0.022865583595063955, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.14753891091954022, "avg_duration_per_task_ms": 476020.4597701149, "avg_output_tokens_per_task": 10344.031609195403, "last_run_timestamp": "2026-08-22T08:22:21.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.7457142857142857, "accuracy_std_dev": 0.01866272796407924, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.041890954285714294, "avg_duration_per_task_ms": 192572.89428571428, "avg_output_tokens_per_task": 12803.885714285714, "last_run_timestamp": "2026-09-26T08:05:12.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.7457142857142857, "accuracy_std_dev": 0.048328140467653255, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.08099248857142857, "avg_duration_per_task_ms": 179157.66714285716, "avg_output_tokens_per_task": 7300.001428571429, "last_run_timestamp": "2026-09-26T08:09:25.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7456140464396285, "accuracy_std_dev": 0.047057992260714526, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.011993541795665636, "avg_duration_per_task_ms": 632093.0835913313, "avg_output_tokens_per_task": 8861.049535603715, "last_run_timestamp": "2026-08-29T16:44:33.448Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.7455555833333333, "accuracy_std_dev": 0.043276290803892344, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11665106833333336, "avg_duration_per_task_ms": 700763.9033333333, "avg_output_tokens_per_task": 27544.08, "last_run_timestamp": "2026-09-26T08:01:16.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.7455555, "accuracy_std_dev": 0.027599586239350292, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11502652333333335, "avg_duration_per_task_ms": 272510.09, "avg_output_tokens_per_task": 12128.78, "last_run_timestamp": "2026-09-26T08:22:52.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7454546363636363, "accuracy_std_dev": 0.026180138836609183, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.35241780545454543, "avg_duration_per_task_ms": 231797.27818181817, "avg_output_tokens_per_task": 10791.974545454545, "last_run_timestamp": "2026-09-26T08:23:01.142Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.7450981176470588, "accuracy_std_dev": 0.03403410890628669, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.08884740352941176, "avg_duration_per_task_ms": 247677.12588235294, "avg_output_tokens_per_task": 7114.02, "last_run_timestamp": "2026-08-29T16:43:42.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Io Net", "accuracy": 0.745, "accuracy_std_dev": 0.051071844820148544, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.05017629, "avg_duration_per_task_ms": 100886.12833333333, "avg_output_tokens_per_task": 4164.488333333334, "last_run_timestamp": "2026-06-20T12:09:06.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.745, "accuracy_std_dev": 0.009860245661239886, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.45607220499999995, "avg_duration_per_task_ms": 481202.88, "avg_output_tokens_per_task": 36078.37, "last_run_timestamp": "2026-09-01T12:59:55.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.7448276551724138, "accuracy_std_dev": 0.0035438860340341496, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.022715406896551726, "avg_duration_per_task_ms": 289516.0620689655, "avg_output_tokens_per_task": 10103.63448275862, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.7447618571428571, "accuracy_std_dev": 0.035405124090119165, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08685644000000001, "avg_duration_per_task_ms": 256492.48857142858, "avg_output_tokens_per_task": 12140.245714285715, "last_run_timestamp": "2026-09-26T08:23:17.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.744221408759124, "accuracy_std_dev": 0.0694836949766747, "run_count": 22, "total_questions": 1096, "avg_cost_per_task_usd": 0.010320726277372263, "avg_duration_per_task_ms": 190073.67335766423, "avg_output_tokens_per_task": 8391.975364963504, "last_run_timestamp": "2026-08-29T16:29:07.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.7441947955056181, "accuracy_std_dev": 0.013929720069329346, "run_count": 9, "total_questions": 445, "avg_cost_per_task_usd": 0.10463494606741572, "avg_duration_per_task_ms": 943614.8921348314, "avg_output_tokens_per_task": 21227.357303370787, "last_run_timestamp": "2026-09-26T08:24:43.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.74416675, "accuracy_std_dev": 0.021586450870569266, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.32272964499999995, "avg_duration_per_task_ms": 199123.9275, "avg_output_tokens_per_task": 11446.9275, "last_run_timestamp": "2026-09-26T08:10:43.577Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.7441077474747474, "accuracy_std_dev": 0.03625850000000003, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.012172767676767676, "avg_duration_per_task_ms": 161557.24242424243, "avg_output_tokens_per_task": 3853.3636363636365, "last_run_timestamp": "2026-09-26T08:10:43.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.7441045812619503, "accuracy_std_dev": 0.03256003617627679, "run_count": 21, "total_questions": 1046, "avg_cost_per_task_usd": 0.061009545889101346, "avg_duration_per_task_ms": 226154.17782026768, "avg_output_tokens_per_task": 8148.751434034417, "last_run_timestamp": "2026-08-29T16:33:12.146Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.7440306848137535, "accuracy_std_dev": 0.019647453058182128, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.028864432664756447, "avg_duration_per_task_ms": 650673.9828080229, "avg_output_tokens_per_task": 21865.07736389685, "last_run_timestamp": "2026-09-26T16:00:24.615Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.744, "accuracy_std_dev": 0.019137165704461043, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.006426796, "avg_duration_per_task_ms": 131863.972, "avg_output_tokens_per_task": 4262.824, "last_run_timestamp": "2026-09-26T08:07:18.986Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7439998000000001, "accuracy_std_dev": 0.025508095447524134, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.010447856000000002, "avg_duration_per_task_ms": 427694.796, "avg_output_tokens_per_task": 12591.384, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7433335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0032184600000000002, "avg_duration_per_task_ms": 64941.97, "avg_output_tokens_per_task": 11846.7, "last_run_timestamp": "2026-08-19T18:00:02.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7433330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.34578612, "avg_duration_per_task_ms": 121381.44, "avg_output_tokens_per_task": 6958.96, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.7430555, "accuracy_std_dev": 0.035484932031074834, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.068885335, "avg_duration_per_task_ms": 169463.79083333333, "avg_output_tokens_per_task": 5588.91, "last_run_timestamp": "2026-08-29T16:44:49.542Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Together", "accuracy": 0.7428571428571429, "accuracy_std_dev": 0.042666241494480235, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.09274233999999999, "avg_duration_per_task_ms": 75968.94571428571, "avg_output_tokens_per_task": 4836.024285714286, "last_run_timestamp": "2026-06-20T18:29:02.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.7427776666666667, "accuracy_std_dev": 0.036434422742175517, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3671939166666667, "avg_duration_per_task_ms": 137596.83, "avg_output_tokens_per_task": 5212.336666666667, "last_run_timestamp": "2026-09-01T13:00:30.911Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.7426668, "accuracy_std_dev": 0.03690245640821219, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011645404, "avg_duration_per_task_ms": 585840.916, "avg_output_tokens_per_task": 18794.172, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.7426666000000001, "accuracy_std_dev": 0.01236473120775379, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07381416, "avg_duration_per_task_ms": 152125.172, "avg_output_tokens_per_task": 4690.728, "last_run_timestamp": "2026-09-26T08:27:29.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.7425, "accuracy_std_dev": 0.029988405167664416, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.5165171675, "avg_duration_per_task_ms": 228220.77, "avg_output_tokens_per_task": 10000.7275, "last_run_timestamp": "2026-09-26T08:11:12.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.742499875, "accuracy_std_dev": 0.021327474782762628, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0092532975, "avg_duration_per_task_ms": 294444.1, "avg_output_tokens_per_task": 4179.7575, "last_run_timestamp": "2026-09-26T08:24:45.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7423371321839081, "accuracy_std_dev": 0.08048438687076966, "run_count": 21, "total_questions": 1044, "avg_cost_per_task_usd": 0.051994346743295033, "avg_duration_per_task_ms": 148657.38218390805, "avg_output_tokens_per_task": 19702.44827586207, "last_run_timestamp": "2026-08-29T16:10:33.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7422839058641976, "accuracy_std_dev": 0.037889381261823826, "run_count": 26, "total_questions": 1296, "avg_cost_per_task_usd": 0.05804707330246913, "avg_duration_per_task_ms": 156059.7939814815, "avg_output_tokens_per_task": 6063.099537037037, "last_run_timestamp": "2026-08-29T16:07:02.345Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7422221666666666, "accuracy_std_dev": 0.009937943254964244, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.34229832999999993, "avg_duration_per_task_ms": 115494.89, "avg_output_tokens_per_task": 7015.326666666667, "last_run_timestamp": "2026-09-26T08:16:17.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7422221666666666, "accuracy_std_dev": 0.029731460662720387, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.34941253333333333, "avg_duration_per_task_ms": 221865.17666666667, "avg_output_tokens_per_task": 10845.116666666667, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7422221666666666, "accuracy_std_dev": 0.01791636261835036, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3279842533333333, "avg_duration_per_task_ms": 196889.34333333332, "avg_output_tokens_per_task": 10978.62, "last_run_timestamp": "2026-09-01T17:31:47.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7418405617977527, "accuracy_std_dev": 0.038139869614963295, "run_count": 25, "total_questions": 1246, "avg_cost_per_task_usd": 0.07584754895666132, "avg_duration_per_task_ms": 258067.6573033708, "avg_output_tokens_per_task": 5980.402889245586, "last_run_timestamp": "2026-08-29T16:31:41.307Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.741791919866444, "accuracy_std_dev": 0.04646084840389512, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.06980493155258766, "avg_duration_per_task_ms": 1045800.3572621035, "avg_output_tokens_per_task": 6776.651085141903, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.7416666666666667, "accuracy_std_dev": 0.0371060392419108, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.15347501333333333, "avg_duration_per_task_ms": 347171.24333333335, "avg_output_tokens_per_task": 21806.65, "last_run_timestamp": "2026-09-26T08:23:03.682Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.7414969591836733, "accuracy_std_dev": 0.023315275936604284, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.01141882142857143, "avg_duration_per_task_ms": 212540.27551020408, "avg_output_tokens_per_task": 4189.09693877551, "last_run_timestamp": "2026-09-26T08:26:46.238Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7413501746835442, "accuracy_std_dev": 0.03375155758853722, "run_count": 16, "total_questions": 790, "avg_cost_per_task_usd": 0.08565632531645569, "avg_duration_per_task_ms": 345925.9759493671, "avg_output_tokens_per_task": 10900.69746835443, "last_run_timestamp": "2026-08-29T16:54:12.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.7411113333333333, "accuracy_std_dev": 0.028588182830595485, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018602686666666667, "avg_duration_per_task_ms": 242873.4, "avg_output_tokens_per_task": 16988.533333333333, "last_run_timestamp": "2026-09-26T08:21:38.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7411111666666667, "accuracy_std_dev": 0.02732958194836176, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.34793083666666663, "avg_duration_per_task_ms": 202714.10666666666, "avg_output_tokens_per_task": 11886.45, "last_run_timestamp": "2026-09-26T08:09:25.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7411110000000001, "accuracy_std_dev": 0.033425712732366185, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03801252333333333, "avg_duration_per_task_ms": 199405.32333333333, "avg_output_tokens_per_task": 13535.723333333333, "last_run_timestamp": "2026-09-26T08:09:22.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.7409571552587647, "accuracy_std_dev": 0.034434709574561184, "run_count": 24, "total_questions": 1198, "avg_cost_per_task_usd": 0.24114451085141902, "avg_duration_per_task_ms": 25708.291318864776, "avg_output_tokens_per_task": 5412.737061769616, "last_run_timestamp": "2026-08-15T14:16:11.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.740909090909091, "accuracy_std_dev": 0.034424213616038424, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.3756752018181817, "avg_duration_per_task_ms": 205007.96545454545, "avg_output_tokens_per_task": 11919.866363636364, "last_run_timestamp": "2026-09-26T08:26:37.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7406779661016949, "accuracy_std_dev": 0.0192449897289312, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.02004586440677966, "avg_duration_per_task_ms": 1117850.5423728814, "avg_output_tokens_per_task": 22657.922033898307, "last_run_timestamp": "2026-09-12T08:11:08.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7402778333333333, "accuracy_std_dev": 0.039085846085536155, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.11041305000000001, "avg_duration_per_task_ms": 227804.8675, "avg_output_tokens_per_task": 5546.4675, "last_run_timestamp": "2026-08-29T16:32:10.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05110574, "avg_duration_per_task_ms": 158677.44, "avg_output_tokens_per_task": 8249.06, "last_run_timestamp": "2026-07-18T16:00:09.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AkashML", "accuracy": 0.74, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06375521, "avg_duration_per_task_ms": 66151.97, "avg_output_tokens_per_task": 3290.09, "last_run_timestamp": "2026-06-20T10:18:25.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.74, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.054267579999999996, "avg_duration_per_task_ms": 797443.04, "avg_output_tokens_per_task": 42406.53, "last_run_timestamp": "2026-08-29T16:33:37.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.74, "accuracy_std_dev": 0.02380486645100116, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09328288000000001, "avg_duration_per_task_ms": 826540.2925, "avg_output_tokens_per_task": 38601.0775, "last_run_timestamp": "2026-09-26T16:00:20.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02561014, "avg_duration_per_task_ms": 147210.62, "avg_output_tokens_per_task": 10787.18, "last_run_timestamp": "2026-07-18T08:04:42.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.74, "accuracy_std_dev": 0.01115542718142162, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.014306968, "avg_duration_per_task_ms": 188774.736, "avg_output_tokens_per_task": 4205.028, "last_run_timestamp": "2026-09-26T08:25:40.376Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7398561463790447, "accuracy_std_dev": 0.12410568998523018, "run_count": 26, "total_questions": 1298, "avg_cost_per_task_usd": 0.03732514175654854, "avg_duration_per_task_ms": 125480.83204930663, "avg_output_tokens_per_task": 5421.03312788906, "last_run_timestamp": "2026-08-29T16:11:58.048Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.7396135507246376, "accuracy_std_dev": 0.02509467552355785, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.0175928115942029, "avg_duration_per_task_ms": 309043.7826086957, "avg_output_tokens_per_task": 14060.257971014493, "last_run_timestamp": "2026-09-26T08:21:25.884Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.7394443333333334, "accuracy_std_dev": 0.04158517957825306, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09852801333333332, "avg_duration_per_task_ms": 155104.675, "avg_output_tokens_per_task": 8559.225, "last_run_timestamp": "2026-09-26T08:22:32.817Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.7391975308641975, "accuracy_std_dev": 0.03518784605833174, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.03759480401234569, "avg_duration_per_task_ms": 156986.57098765433, "avg_output_tokens_per_task": 7739.7793209876545, "last_run_timestamp": "2026-09-26T08:12:13.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.739160162601626, "accuracy_std_dev": 0.016384230901693223, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.02238267073170732, "avg_duration_per_task_ms": 118042.10162601626, "avg_output_tokens_per_task": 3823.4349593495936, "last_run_timestamp": "2026-09-26T08:17:01.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.7391112, "accuracy_std_dev": 0.02063675876746798, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.5042409586666666, "avg_duration_per_task_ms": 734521.768, "avg_output_tokens_per_task": 9377.424, "last_run_timestamp": "2026-09-26T08:10:56.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.7390475000000001, "accuracy_std_dev": 0.022797592068180002, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.022234177142857144, "avg_duration_per_task_ms": 351589.08714285714, "avg_output_tokens_per_task": 11306.812857142857, "last_run_timestamp": "2026-09-26T08:08:19.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.738889, "accuracy_std_dev": 0.017812306747489716, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.022993023333333338, "avg_duration_per_task_ms": 282964.01666666666, "avg_output_tokens_per_task": 13071.093333333334, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.7386666000000001, "accuracy_std_dev": 0.02777674167068556, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.008449704, "avg_duration_per_task_ms": 262399.948, "avg_output_tokens_per_task": 5465.996, "last_run_timestamp": "2026-09-26T08:30:17.131Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7383335, "accuracy_std_dev": 0.007264793338423339, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.029026915000000004, "avg_duration_per_task_ms": 190934.755, "avg_output_tokens_per_task": 18429.705, "last_run_timestamp": "2026-09-26T08:10:43.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.7383335, "accuracy_std_dev": 0.01092919128984391, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07321903, "avg_duration_per_task_ms": 107189.01, "avg_output_tokens_per_task": 4495.765, "last_run_timestamp": "2026-09-26T08:22:24.359Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.7383335, "accuracy_std_dev": 0.01092919128984391, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.023440455, "avg_duration_per_task_ms": 211636.695, "avg_output_tokens_per_task": 4181.36, "last_run_timestamp": "2026-09-26T08:23:00.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.73833325, "accuracy_std_dev": 0.03345812889175814, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0298002275, "avg_duration_per_task_ms": 280974.21, "avg_output_tokens_per_task": 17047.3125, "last_run_timestamp": "2026-09-26T08:28:05.592Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.7382865983935742, "accuracy_std_dev": 0.027461257286584682, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.07498479919678715, "avg_duration_per_task_ms": 91080.29718875502, "avg_output_tokens_per_task": 4367.955823293173, "last_run_timestamp": "2026-09-26T08:11:26.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7380952857142857, "accuracy_std_dev": 0.034862897001639706, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10877247142857141, "avg_duration_per_task_ms": 324815.4542857143, "avg_output_tokens_per_task": 18549.505714285715, "last_run_timestamp": "2026-09-26T08:13:27.148Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.7377122627345845, "accuracy_std_dev": 0.03780777470286003, "run_count": 15, "total_questions": 746, "avg_cost_per_task_usd": 0.027386981233243967, "avg_duration_per_task_ms": 394334.3485254692, "avg_output_tokens_per_task": 12507.815013404826, "last_run_timestamp": "2026-09-26T08:22:26.272Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.737695687919463, "accuracy_std_dev": 0.024690397243074747, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.12595012919463086, "avg_duration_per_task_ms": 492774.60067114094, "avg_output_tokens_per_task": 24465.090604026846, "last_run_timestamp": "2026-09-26T08:23:04.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7376066307692307, "accuracy_std_dev": 0.012768517269048922, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.012177446153846153, "avg_duration_per_task_ms": 65592.95897435897, "avg_output_tokens_per_task": 3712.5128205128203, "last_run_timestamp": "2026-09-26T08:16:16.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.7375757272727272, "accuracy_std_dev": 0.015640496284214565, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.008148763636363637, "avg_duration_per_task_ms": 115345.35272727272, "avg_output_tokens_per_task": 10256.063636363637, "last_run_timestamp": "2026-08-29T17:19:58.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7375155805243446, "accuracy_std_dev": 0.026231812733573265, "run_count": 11, "total_questions": 534, "avg_cost_per_task_usd": 0.017135816479400746, "avg_duration_per_task_ms": 93129.53558052435, "avg_output_tokens_per_task": 5183.471910112359, "last_run_timestamp": "2026-09-26T16:00:25.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.7368888666666666, "accuracy_std_dev": 0.02741899242706698, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.06124634666666667, "avg_duration_per_task_ms": 229793.576, "avg_output_tokens_per_task": 9405.036, "last_run_timestamp": "2026-08-29T16:16:50.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.73680004, "accuracy_std_dev": 0.029461548394447975, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.033284238400000006, "avg_duration_per_task_ms": 120902.7176, "avg_output_tokens_per_task": 5596.316, "last_run_timestamp": "2026-08-22T16:12:21.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02745565, "avg_duration_per_task_ms": 321964.94, "avg_output_tokens_per_task": 12731.03, "last_run_timestamp": "2026-09-26T08:24:06.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.7366665, "accuracy_std_dev": 0.05666649999999995, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5401649000000001, "avg_duration_per_task_ms": 161301.85, "avg_output_tokens_per_task": 8843.07, "last_run_timestamp": "2026-09-01T13:02:17.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.7365591666666667, "accuracy_std_dev": 0.037933644459181036, "run_count": 15, "total_questions": 744, "avg_cost_per_task_usd": 0.020751344086021507, "avg_duration_per_task_ms": 306670.2688172043, "avg_output_tokens_per_task": 12001.696236559139, "last_run_timestamp": "2026-09-26T08:22:59.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7365404329896906, "accuracy_std_dev": 0.04686504205487875, "run_count": 6, "total_questions": 291, "avg_cost_per_task_usd": 0.08916013058419243, "avg_duration_per_task_ms": 682362.1065292096, "avg_output_tokens_per_task": 7674.209621993127, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.7363080617180984, "accuracy_std_dev": 0.1346197222374049, "run_count": 24, "total_questions": 1199, "avg_cost_per_task_usd": 0.09551945788156796, "avg_duration_per_task_ms": 256553.70475396165, "avg_output_tokens_per_task": 13626.88156797331, "last_run_timestamp": "2026-08-29T17:19:03.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.7360844606525911, "accuracy_std_dev": 0.10956776688246402, "run_count": 21, "total_questions": 1042, "avg_cost_per_task_usd": 0.0232232245681382, "avg_duration_per_task_ms": 366903.96257197694, "avg_output_tokens_per_task": 8722.57773512476, "last_run_timestamp": "2026-08-29T17:21:09.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.7357541832402235, "accuracy_std_dev": 0.09788376088289748, "run_count": 18, "total_questions": 895, "avg_cost_per_task_usd": 0.09288570949720674, "avg_duration_per_task_ms": 339472.96424581006, "avg_output_tokens_per_task": 14305.882681564246, "last_run_timestamp": "2026-08-29T16:30:43.592Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7357256822966507, "accuracy_std_dev": 0.0866496604884399, "run_count": 21, "total_questions": 1045, "avg_cost_per_task_usd": 0.04395789377990429, "avg_duration_per_task_ms": 207924.5062200957, "avg_output_tokens_per_task": 6683.989473684211, "last_run_timestamp": "2026-08-29T17:16:16.996Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13620342666666665, "avg_duration_per_task_ms": 101042.81333333334, "avg_output_tokens_per_task": 8769.966666666667, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020994060000000002, "avg_duration_per_task_ms": 93700.16666666667, "avg_output_tokens_per_task": 4193.253333333333, "last_run_timestamp": "2026-09-26T08:26:42.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7355553333333333, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13248254666666667, "avg_duration_per_task_ms": 94960.21333333333, "avg_output_tokens_per_task": 24879.886666666665, "last_run_timestamp": "2026-09-01T17:31:46.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.7355553333333333, "accuracy_std_dev": 0.013698571612479268, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.011050406666666665, "avg_duration_per_task_ms": 621248.44, "avg_output_tokens_per_task": 4366.6466666666665, "last_run_timestamp": "2026-09-26T08:07:18.968Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.7351751940700809, "accuracy_std_dev": 0.08409205356369888, "run_count": 15, "total_questions": 742, "avg_cost_per_task_usd": 0.05205572641509434, "avg_duration_per_task_ms": 112166.10916442049, "avg_output_tokens_per_task": 4389.22371967655, "last_run_timestamp": "2026-08-01T08:06:01.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7350002499999999, "accuracy_std_dev": 0.008660205926391138, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03314667, "avg_duration_per_task_ms": 134389.755, "avg_output_tokens_per_task": 16623.845, "last_run_timestamp": "2026-09-26T08:05:12.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.735, "accuracy_std_dev": 0.15570928990590127, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.1830062116666667, "avg_duration_per_task_ms": 302480.2633333333, "avg_output_tokens_per_task": 14547.835833333333, "last_run_timestamp": "2026-08-29T16:58:32.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7346031428571429, "accuracy_std_dev": 0.06504760378682801, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.05404573238095239, "avg_duration_per_task_ms": 148223.61523809523, "avg_output_tokens_per_task": 6767.127619047619, "last_run_timestamp": "2026-08-29T17:19:05.156Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.7341239911045219, "accuracy_std_dev": 0.02480538350740109, "run_count": 27, "total_questions": 1349, "avg_cost_per_task_usd": 0.10910542920681986, "avg_duration_per_task_ms": 159024.67679762788, "avg_output_tokens_per_task": 8159.174944403262, "last_run_timestamp": "2026-09-26T08:08:19.667Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.7341176470588235, "accuracy_std_dev": 0.06332332647233949, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.07183540352941177, "avg_duration_per_task_ms": 480891.6094117647, "avg_output_tokens_per_task": 22660.84, "last_run_timestamp": "2026-07-18T16:00:08.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7339180701754385, "accuracy_std_dev": 0.02156426040350099, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.009527433583959901, "avg_duration_per_task_ms": 516946.6215538847, "avg_output_tokens_per_task": 13232.288220551378, "last_run_timestamp": "2026-09-26T08:07:18.969Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.34495834000000003, "avg_duration_per_task_ms": 101333.53, "avg_output_tokens_per_task": 6956.68, "last_run_timestamp": "2026-09-26T08:22:24.747Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09979016999999998, "avg_duration_per_task_ms": 95640.78, "avg_output_tokens_per_task": 4171.89, "last_run_timestamp": "2026-09-01T12:57:13.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.733333375, "accuracy_std_dev": 0.05196139592749965, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0075224875, "avg_duration_per_task_ms": 325334.9425, "avg_output_tokens_per_task": 15635.5625, "last_run_timestamp": "2026-09-26T08:07:31.852Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.733333347826087, "accuracy_std_dev": 0.1334925998058413, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.07098599043478261, "avg_duration_per_task_ms": 169848.0147826087, "avg_output_tokens_per_task": 6912.339130434782, "last_run_timestamp": "2026-08-29T17:05:17.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008480786666666665, "avg_duration_per_task_ms": 397846.0266666667, "avg_output_tokens_per_task": 19127.326666666668, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Fireworks", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.024944382578492966, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.035478496666666665, "avg_duration_per_task_ms": 63670.043333333335, "avg_output_tokens_per_task": 4095.423333333333, "last_run_timestamp": "2026-05-23T14:30:04.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.03250838448428001, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06780395555555555, "avg_duration_per_task_ms": 133879.79333333333, "avg_output_tokens_per_task": 7446.308888888889, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "SiliconFlow", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.031972210155418145, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.019457343333333335, "avg_duration_per_task_ms": 71958.22, "avg_output_tokens_per_task": 3419.1466666666665, "last_run_timestamp": "2026-06-06T18:57:18.338Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.73333325, "accuracy_std_dev": 0.004713927363409425, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.093701365, "avg_duration_per_task_ms": 409506.12, "avg_output_tokens_per_task": 12562.605, "last_run_timestamp": "2026-09-26T08:23:24.418Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.7333332, "accuracy_std_dev": 0.01633001326882499, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.016486864, "avg_duration_per_task_ms": 85790.632, "avg_output_tokens_per_task": 4187.072, "last_run_timestamp": "2026-09-26T08:05:55.055Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.7333331666666667, "accuracy_std_dev": 0.012171581962597242, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3451316866666666, "avg_duration_per_task_ms": 86668.75666666667, "avg_output_tokens_per_task": 6980.716666666666, "last_run_timestamp": "2026-09-26T08:17:01.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.34216192, "avg_duration_per_task_ms": 118971.98, "avg_output_tokens_per_task": 7077.54, "last_run_timestamp": "2026-09-26T08:22:25.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2412942, "avg_duration_per_task_ms": 207134.84, "avg_output_tokens_per_task": 12538.34, "last_run_timestamp": "2026-08-01T08:06:12.346Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.7329933877551019, "accuracy_std_dev": 0.02075268657909138, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.05900951530612245, "avg_duration_per_task_ms": 460991.887755102, "avg_output_tokens_per_task": 12569.994897959185, "last_run_timestamp": "2026-09-26T08:23:24.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7329792361275088, "accuracy_std_dev": 0.03497211277654473, "run_count": 17, "total_questions": 847, "avg_cost_per_task_usd": 0.052468580873671775, "avg_duration_per_task_ms": 188219.26328217238, "avg_output_tokens_per_task": 7981.263282172373, "last_run_timestamp": "2026-08-29T16:55:12.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.7328571428571429, "accuracy_std_dev": 0.02503509736024079, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06745117142857142, "avg_duration_per_task_ms": 76562.92857142857, "avg_output_tokens_per_task": 8496.594285714285, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.7327989038076153, "accuracy_std_dev": 0.1632218869480668, "run_count": 20, "total_questions": 998, "avg_cost_per_task_usd": 0.015171186372745492, "avg_duration_per_task_ms": 223130.1102204409, "avg_output_tokens_per_task": 8631.145290581162, "last_run_timestamp": "2026-08-29T16:47:28.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.7326667, "accuracy_std_dev": 0.04049139881271083, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.06109215, "avg_duration_per_task_ms": 223095.216, "avg_output_tokens_per_task": 7046.56, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7326621543624161, "accuracy_std_dev": 0.030789543369281885, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.09465771812080537, "avg_duration_per_task_ms": 390427.067114094, "avg_output_tokens_per_task": 11230.369127516778, "last_run_timestamp": "2026-08-29T16:33:38.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.7318549798387096, "accuracy_std_dev": 0.04649888648139865, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.08042021975806452, "avg_duration_per_task_ms": 208719.5060483871, "avg_output_tokens_per_task": 8252.131048387097, "last_run_timestamp": "2026-09-19T08:12:24.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.73166675, "accuracy_std_dev": 0.03210208110368392, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.00461813, "avg_duration_per_task_ms": 254807.5, "avg_output_tokens_per_task": 4301.06, "last_run_timestamp": "2026-09-26T08:25:43.420Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7314287142857142, "accuracy_std_dev": 0.02301532369478017, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07236491428571429, "avg_duration_per_task_ms": 89556.96, "avg_output_tokens_per_task": 10594.671428571428, "last_run_timestamp": "2026-09-26T16:00:24.624Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.7314287142857142, "accuracy_std_dev": 0.009897460680314293, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.39489586285714284, "avg_duration_per_task_ms": 142184.54857142858, "avg_output_tokens_per_task": 11046.07142857143, "last_run_timestamp": "2026-09-26T08:21:22.720Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7311556834170854, "accuracy_std_dev": 0.027058284160943663, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.02928007286432161, "avg_duration_per_task_ms": 481200.60552763817, "avg_output_tokens_per_task": 8314.90703517588, "last_run_timestamp": "2026-08-29T16:49:08.522Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012396426666666667, "avg_duration_per_task_ms": 337807.9, "avg_output_tokens_per_task": 15691.053333333333, "last_run_timestamp": "2026-09-26T08:22:46.339Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.025903600000000002, "avg_duration_per_task_ms": 117172.05333333333, "avg_output_tokens_per_task": 15158.646666666667, "last_run_timestamp": "2026-09-26T08:07:18.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.035416115465514666, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3332058333333333, "avg_duration_per_task_ms": 117249.60666666667, "avg_output_tokens_per_task": 11086.966666666667, "last_run_timestamp": "2026-09-01T17:31:47.546Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.7310255769230769, "accuracy_std_dev": 0.04276519757499896, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.03939189384615385, "avg_duration_per_task_ms": 165400.37076923077, "avg_output_tokens_per_task": 5925.508461538461, "last_run_timestamp": "2026-08-30T00:01:02.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.730909, "accuracy_std_dev": 0.028180123769003633, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06749359090909092, "avg_duration_per_task_ms": 291533.3, "avg_output_tokens_per_task": 9954.196363636363, "last_run_timestamp": "2026-09-26T08:23:16.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.7305882352941176, "accuracy_std_dev": 0.10471396142743807, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.06950796235294117, "avg_duration_per_task_ms": 400129.06588235294, "avg_output_tokens_per_task": 38127.568235294115, "last_run_timestamp": "2026-08-15T14:08:35.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7305263684210526, "accuracy_std_dev": 0.033390586105116714, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.35496904315789474, "avg_duration_per_task_ms": 206890.78210526315, "avg_output_tokens_per_task": 10297.44, "last_run_timestamp": "2026-09-26T08:17:01.551Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.7304761428571429, "accuracy_std_dev": 0.040293813213262725, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.006170451428571428, "avg_duration_per_task_ms": 328862.59714285715, "avg_output_tokens_per_task": 15818.588571428572, "last_run_timestamp": "2026-09-26T08:13:27.139Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.7302799198473282, "accuracy_std_dev": 0.08316610177772252, "run_count": 21, "total_questions": 1048, "avg_cost_per_task_usd": 0.009317546755725192, "avg_duration_per_task_ms": 128886.75190839695, "avg_output_tokens_per_task": 7970.051526717557, "last_run_timestamp": "2026-08-29T17:11:34.630Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7302564615384616, "accuracy_std_dev": 0.04434636251775576, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.09595046461538462, "avg_duration_per_task_ms": 158127.4846153846, "avg_output_tokens_per_task": 8436.901538461538, "last_run_timestamp": "2026-09-26T08:22:58.695Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.7300089191374662, "accuracy_std_dev": 0.028871736576028047, "run_count": 15, "total_questions": 742, "avg_cost_per_task_usd": 0.05800620754716981, "avg_duration_per_task_ms": 288668.6940700809, "avg_output_tokens_per_task": 8401.919137466308, "last_run_timestamp": "2026-08-29T16:47:08.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7300001666666667, "accuracy_std_dev": 0.03605552816336059, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11662036, "avg_duration_per_task_ms": 82058.93666666666, "avg_output_tokens_per_task": 9160.97, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.73, "accuracy_std_dev": 0.030489986214056124, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.49638783, "avg_duration_per_task_ms": 192253.77333333335, "avg_output_tokens_per_task": 9911.976666666667, "last_run_timestamp": "2026-09-26T08:22:57.969Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.73, "accuracy_std_dev": 0.0247205998233255, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.36644244, "avg_duration_per_task_ms": 230988.18, "avg_output_tokens_per_task": 10810.57375, "last_run_timestamp": "2026-09-26T08:22:54.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.73, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23651355, "avg_duration_per_task_ms": 142691.29, "avg_output_tokens_per_task": 22898.27, "last_run_timestamp": "2026-09-01T17:31:43.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.73, "accuracy_std_dev": 0.026874047787782167, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.40094411999999996, "avg_duration_per_task_ms": 212371.995, "avg_output_tokens_per_task": 8595.31, "last_run_timestamp": "2026-09-26T08:23:04.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.73, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.30719247000000005, "avg_duration_per_task_ms": 162813.34, "avg_output_tokens_per_task": 10978.12, "last_run_timestamp": "2026-09-01T17:32:02.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.73, "accuracy_std_dev": 0.01374388753228139, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.243089465, "avg_duration_per_task_ms": 228014.985, "avg_output_tokens_per_task": 12589.82, "last_run_timestamp": "2026-09-26T08:11:43.368Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.72986668, "accuracy_std_dev": 0.038093252876298195, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.037141388, "avg_duration_per_task_ms": 176289.5272, "avg_output_tokens_per_task": 6180.716, "last_run_timestamp": "2026-08-30T00:01:09.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7296182105263157, "accuracy_std_dev": 0.10571754149171865, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.13478412538699688, "avg_duration_per_task_ms": 559761.580495356, "avg_output_tokens_per_task": 26074.630030959754, "last_run_timestamp": "2026-09-26T08:05:12.733Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.7295868703170029, "accuracy_std_dev": 0.031298136757478015, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.07363949279538905, "avg_duration_per_task_ms": 211055.38616714696, "avg_output_tokens_per_task": 8076.533141210374, "last_run_timestamp": "2026-09-26T08:14:46.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Phala", "accuracy": 0.729167, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.044939625000000004, "avg_duration_per_task_ms": 321431, "avg_output_tokens_per_task": 27916.854166666668, "last_run_timestamp": "2026-07-11T08:03:26.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.72916675, "accuracy_std_dev": 0.019346895388860176, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.059703584999999997, "avg_duration_per_task_ms": 1171969.47, "avg_output_tokens_per_task": 26770.295, "last_run_timestamp": "2026-09-26T08:10:43.579Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.7288886666666667, "accuracy_std_dev": 0.02572412999932599, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.041228259999999996, "avg_duration_per_task_ms": 125095.52, "avg_output_tokens_per_task": 23895.02666666667, "last_run_timestamp": "2026-09-26T08:16:16.958Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.728581156626506, "accuracy_std_dev": 0.046862578058083824, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.017320277108433733, "avg_duration_per_task_ms": 435636.6847389558, "avg_output_tokens_per_task": 8247.0140562249, "last_run_timestamp": "2026-08-29T16:32:40.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.7285714285714285, "accuracy_std_dev": 0.03504452237962912, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04105087428571428, "avg_duration_per_task_ms": 218582.77142857143, "avg_output_tokens_per_task": 20493.322857142855, "last_run_timestamp": "2026-09-26T08:07:18.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Together", "accuracy": 0.7285714285714285, "accuracy_std_dev": 0.04323641700120444, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.09354192, "avg_duration_per_task_ms": 71979.10142857143, "avg_output_tokens_per_task": 3596.4542857142856, "last_run_timestamp": "2026-06-20T10:33:11.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.7283335, "accuracy_std_dev": 0.02279116544299565, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.345777245, "avg_duration_per_task_ms": 85393.745, "avg_output_tokens_per_task": 6868.74, "last_run_timestamp": "2026-09-26T08:25:13.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.728187812080537, "accuracy_std_dev": 0.031197835362943585, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.09276904697986577, "avg_duration_per_task_ms": 219176.42953020133, "avg_output_tokens_per_task": 8491.71812080537, "last_run_timestamp": "2026-09-26T08:23:18.761Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.7280700247678018, "accuracy_std_dev": 0.022449281062274982, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.007610120743034056, "avg_duration_per_task_ms": 323992.3544891641, "avg_output_tokens_per_task": 14009.664086687306, "last_run_timestamp": "2026-09-26T08:05:12.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.727891030612245, "accuracy_std_dev": 0.023240919929888738, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.32119672959183676, "avg_duration_per_task_ms": 168213.4693877551, "avg_output_tokens_per_task": 61497.88775510204, "last_run_timestamp": "2026-07-04T08:06:25.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.7276787589285714, "accuracy_std_dev": 0.023663327303270934, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.019942993303571428, "avg_duration_per_task_ms": 481307.24776785716, "avg_output_tokens_per_task": 14518.11830357143, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.7275, "accuracy_std_dev": 0.011273272229038026, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.25819418250000004, "avg_duration_per_task_ms": 292629.7475, "avg_output_tokens_per_task": 10249.3025, "last_run_timestamp": "2026-09-26T08:16:17.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.7274073888888889, "accuracy_std_dev": 0.03589348401581003, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.018933201111111114, "avg_duration_per_task_ms": 205864.55777777778, "avg_output_tokens_per_task": 6492.102222222222, "last_run_timestamp": "2026-07-18T16:00:09.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.7272255559006211, "accuracy_std_dev": 0.03191875859800738, "run_count": 13, "total_questions": 644, "avg_cost_per_task_usd": 0.04242405434782608, "avg_duration_per_task_ms": 215017.30279503105, "avg_output_tokens_per_task": 8513.728260869566, "last_run_timestamp": "2026-09-26T08:25:16.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.7272222083333333, "accuracy_std_dev": 0.033550248154794085, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.03448022916666667, "avg_duration_per_task_ms": 350584.57416666666, "avg_output_tokens_per_task": 8108.9575, "last_run_timestamp": "2026-08-29T17:14:29.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.7272108979591837, "accuracy_std_dev": 0.019828664901097113, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.020857016326530614, "avg_duration_per_task_ms": 240921.06530612244, "avg_output_tokens_per_task": 10264.273469387756, "last_run_timestamp": "2026-09-26T08:13:28.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7271692397260274, "accuracy_std_dev": 0.01097209078627325, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.12186630821917807, "avg_duration_per_task_ms": 191102.53424657535, "avg_output_tokens_per_task": 9680.431506849314, "last_run_timestamp": "2026-09-26T16:00:20.148Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.7270795428051002, "accuracy_std_dev": 0.09660785679619506, "run_count": 22, "total_questions": 1098, "avg_cost_per_task_usd": 0.0763265227686703, "avg_duration_per_task_ms": 209098.53916211292, "avg_output_tokens_per_task": 22814.218579234974, "last_run_timestamp": "2026-08-29T16:12:10.428Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.7269444166666666, "accuracy_std_dev": 0.03653912012363903, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.03204830166666667, "avg_duration_per_task_ms": 247789.1025, "avg_output_tokens_per_task": 8437.178333333333, "last_run_timestamp": "2026-08-29T16:13:54.386Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.029598939999999997, "avg_duration_per_task_ms": 159012.9, "avg_output_tokens_per_task": 16200.32, "last_run_timestamp": "2026-09-26T08:25:06.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.27023132, "avg_duration_per_task_ms": 228964.68, "avg_output_tokens_per_task": 14058.24, "last_run_timestamp": "2026-08-01T08:05:59.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06664494, "avg_duration_per_task_ms": 229624.28, "avg_output_tokens_per_task": 13080.68, "last_run_timestamp": "2026-09-23T06:00:08.551Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.72666675, "accuracy_std_dev": 0.01414201777284626, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.19420406499999998, "avg_duration_per_task_ms": 374699.79, "avg_output_tokens_per_task": 9514.245, "last_run_timestamp": "2026-09-26T08:17:01.582Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.7266666363636364, "accuracy_std_dev": 0.025897973870460233, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.028845547272727274, "avg_duration_per_task_ms": 194541.59636363637, "avg_output_tokens_per_task": 9348.309090909092, "last_run_timestamp": "2026-09-26T16:00:24.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.7266665, "accuracy_std_dev": 0.020000166668055548, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.46161753000000005, "avg_duration_per_task_ms": 792082.91, "avg_output_tokens_per_task": 9376.13, "last_run_timestamp": "2026-09-26T08:24:06.679Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.7266665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20674873, "avg_duration_per_task_ms": 122950.24, "avg_output_tokens_per_task": 5001.27, "last_run_timestamp": "2026-09-01T13:00:51.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.7266665, "accuracy_std_dev": 0.022607791184677924, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.022968040000000002, "avg_duration_per_task_ms": 247273.5, "avg_output_tokens_per_task": 4817.96, "last_run_timestamp": "2026-09-26T08:11:43.349Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.7261538461538461, "accuracy_std_dev": 0.03891341591641658, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.04461130461538462, "avg_duration_per_task_ms": 117488.42923076924, "avg_output_tokens_per_task": 8622.856923076923, "last_run_timestamp": "2026-09-26T08:17:01.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.7260000999999999, "accuracy_std_dev": 0.02958217406293864, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.009236828, "avg_duration_per_task_ms": 354448.192, "avg_output_tokens_per_task": 13566.026, "last_run_timestamp": "2026-09-26T08:23:24.666Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.725238, "accuracy_std_dev": 0.02095781531348834, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.07056013857142857, "avg_duration_per_task_ms": 238159.57714285713, "avg_output_tokens_per_task": 8221.92, "last_run_timestamp": "2026-09-26T08:23:24.553Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.7251281538461538, "accuracy_std_dev": 0.02890059864923259, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.0717470676923077, "avg_duration_per_task_ms": 210149.61076923076, "avg_output_tokens_per_task": 7836.80923076923, "last_run_timestamp": "2026-09-26T08:23:24.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.725, "accuracy_std_dev": 0.025000000000000022, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00440039, "avg_duration_per_task_ms": 1062673.02, "avg_output_tokens_per_task": 13355.81, "last_run_timestamp": "2026-09-26T08:07:18.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Cloudflare", "accuracy": 0.725, "accuracy_std_dev": 0.04555216789572149, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0219137175, "avg_duration_per_task_ms": 61697.205, "avg_output_tokens_per_task": 2955.5725, "last_run_timestamp": "2026-05-23T09:28:32.982Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DekaLLM", "accuracy": 0.725, "accuracy_std_dev": 0.05545268253204706, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06306274499999999, "avg_duration_per_task_ms": 307709.445, "avg_output_tokens_per_task": 29336.17, "last_run_timestamp": "2026-05-16T11:09:22.970Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.7247474713804714, "accuracy_std_dev": 0.08284079836897833, "run_count": 12, "total_questions": 594, "avg_cost_per_task_usd": 0.06815684343434343, "avg_duration_per_task_ms": 133143.7861952862, "avg_output_tokens_per_task": 12155.594276094276, "last_run_timestamp": "2026-08-29T16:57:02.331Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7246489750390015, "accuracy_std_dev": 0.032220821203342284, "run_count": 13, "total_questions": 641, "avg_cost_per_task_usd": 0.09729576131045242, "avg_duration_per_task_ms": 301974.6053042122, "avg_output_tokens_per_task": 13658.625585023401, "last_run_timestamp": "2026-09-26T08:12:00.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.7244446666666666, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02107129333333333, "avg_duration_per_task_ms": 190752.33333333334, "avg_output_tokens_per_task": 8303.633333333333, "last_run_timestamp": "2026-09-26T16:00:20.149Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7244444666666666, "accuracy_std_dev": 0.030663471653563444, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.09078783866666666, "avg_duration_per_task_ms": 211616.812, "avg_output_tokens_per_task": 7390.202666666667, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.724127, "accuracy_std_dev": 0.08098533957157808, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.06660713428571428, "avg_duration_per_task_ms": 276096.2704761905, "avg_output_tokens_per_task": 27588.389523809525, "last_run_timestamp": "2026-08-29T17:00:54.109Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7241015454545454, "accuracy_std_dev": 0.06989016607046616, "run_count": 19, "total_questions": 946, "avg_cost_per_task_usd": 0.05174934883720931, "avg_duration_per_task_ms": 224979.1617336152, "avg_output_tokens_per_task": 6867.136363636364, "last_run_timestamp": "2026-08-29T16:53:19.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Together", "accuracy": 0.724, "accuracy_std_dev": 0.040792156108742275, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.013096376, "avg_duration_per_task_ms": 83392.68266666666, "avg_output_tokens_per_task": 3561.7066666666665, "last_run_timestamp": "2026-06-20T09:31:46.744Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.7238232103746397, "accuracy_std_dev": 0.022993657910877263, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.38125144668587896, "avg_duration_per_task_ms": 402514.99711815565, "avg_output_tokens_per_task": 9102.43227665706, "last_run_timestamp": "2026-09-26T08:24:54.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.7236368271604938, "accuracy_std_dev": 0.10808272562190256, "run_count": 26, "total_questions": 1296, "avg_cost_per_task_usd": 0.06827893055555555, "avg_duration_per_task_ms": 59208.529320987655, "avg_output_tokens_per_task": 5716.902777777777, "last_run_timestamp": "2026-08-30T00:01:10.606Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.7236365454545454, "accuracy_std_dev": 0.030664462731500284, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.022053703636363636, "avg_duration_per_task_ms": 116837.80363636363, "avg_output_tokens_per_task": 6700.861818181818, "last_run_timestamp": "2026-09-26T16:00:25.697Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02063872, "avg_duration_per_task_ms": 638594.61, "avg_output_tokens_per_task": 13867.61, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7233335, "accuracy_std_dev": 0.06333349999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02959498, "avg_duration_per_task_ms": 615820.2, "avg_output_tokens_per_task": 25536.79, "last_run_timestamp": "2026-09-26T08:17:01.598Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09717974, "avg_duration_per_task_ms": 146982.06, "avg_output_tokens_per_task": 51201.48, "last_run_timestamp": "2026-09-01T17:23:14.324Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7231801206896552, "accuracy_std_dev": 0.03695962243048597, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.039466484195402295, "avg_duration_per_task_ms": 165156.0747126437, "avg_output_tokens_per_task": 8083.07040229885, "last_run_timestamp": "2026-09-26T08:24:54.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.7226788730593608, "accuracy_std_dev": 0.07496294197311405, "run_count": 22, "total_questions": 1095, "avg_cost_per_task_usd": 0.10164467945205478, "avg_duration_per_task_ms": 100638.51415525115, "avg_output_tokens_per_task": 7403.919634703197, "last_run_timestamp": "2026-09-26T08:26:47.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.722499875, "accuracy_std_dev": 0.03558044972817764, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.29646921, "avg_duration_per_task_ms": 366904.835, "avg_output_tokens_per_task": 9370.25, "last_run_timestamp": "2026-09-26T08:26:14.256Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.1329602666666667, "avg_duration_per_task_ms": 430783.74, "avg_output_tokens_per_task": 33596.9, "last_run_timestamp": "2026-09-26T08:22:24.836Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.722222316091954, "accuracy_std_dev": 0.031278207950127646, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.1106247183908046, "avg_duration_per_task_ms": 557346.2183908046, "avg_output_tokens_per_task": 24361.238505747126, "last_run_timestamp": "2026-08-15T08:15:35.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.722222, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.019033145833333334, "avg_duration_per_task_ms": 65993.54166666667, "avg_output_tokens_per_task": 3355.9791666666665, "last_run_timestamp": "2026-09-12T08:12:22.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.7215385384615385, "accuracy_std_dev": 0.032936427919473515, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.1025988369230769, "avg_duration_per_task_ms": 105651.07076923076, "avg_output_tokens_per_task": 8722.726153846153, "last_run_timestamp": "2026-09-26T08:27:14.783Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.7214112116788322, "accuracy_std_dev": 0.038322526853941594, "run_count": 22, "total_questions": 1096, "avg_cost_per_task_usd": 0.04304255018248176, "avg_duration_per_task_ms": 307146.9881386861, "avg_output_tokens_per_task": 9393.599452554745, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7211113333333333, "accuracy_std_dev": 0.019309173922142694, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04666514, "avg_duration_per_task_ms": 136735.63333333333, "avg_output_tokens_per_task": 27417.226666666666, "last_run_timestamp": "2026-09-26T08:23:22.508Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.7211113333333333, "accuracy_std_dev": 0.011331263428624743, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012200653333333334, "avg_duration_per_task_ms": 38919.36666666667, "avg_output_tokens_per_task": 3634.8866666666668, "last_run_timestamp": "2026-09-26T08:23:23.813Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.7210882448979592, "accuracy_std_dev": 0.0076780598851532815, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.009489710204081632, "avg_duration_per_task_ms": 145499.9755102041, "avg_output_tokens_per_task": 16260.81224489796, "last_run_timestamp": "2026-09-26T08:06:43.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.7205555555555555, "accuracy_std_dev": 0.06057596801741523, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.0473878211111111, "avg_duration_per_task_ms": 163562.02, "avg_output_tokens_per_task": 7193.1466666666665, "last_run_timestamp": "2026-08-29T17:08:03.684Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.7203921176470588, "accuracy_std_dev": 0.042378295969520495, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.08408886352941176, "avg_duration_per_task_ms": 167842.00235294117, "avg_output_tokens_per_task": 6586.608235294118, "last_run_timestamp": "2026-08-29T16:46:44.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.72, "accuracy_std_dev": 0.0354337331437149, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.330923255, "avg_duration_per_task_ms": 203933.605, "avg_output_tokens_per_task": 10272.2525, "last_run_timestamp": "2026-09-26T08:10:43.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.72, "accuracy_std_dev": 0.11915537433269066, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.013665298181818183, "avg_duration_per_task_ms": 146966.81818181818, "avg_output_tokens_per_task": 8880.298181818182, "last_run_timestamp": "2026-08-29T16:56:37.261Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0037487199999999997, "avg_duration_per_task_ms": 165993.86, "avg_output_tokens_per_task": 15330.68, "last_run_timestamp": "2026-09-23T18:00:03.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15154258, "avg_duration_per_task_ms": 105755.46, "avg_output_tokens_per_task": 13133.32, "last_run_timestamp": "2026-08-04T03:12:01.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.72, "accuracy_std_dev": 0.01845912401316305, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13026846666666667, "avg_duration_per_task_ms": 109723.80666666667, "avg_output_tokens_per_task": 4025.5433333333335, "last_run_timestamp": "2026-09-01T13:00:52.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14489558, "avg_duration_per_task_ms": 248203.88, "avg_output_tokens_per_task": 4971.24, "last_run_timestamp": "2026-09-23T15:00:07.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.402901, "avg_duration_per_task_ms": 196016.4, "avg_output_tokens_per_task": 16256.52, "last_run_timestamp": "2026-09-27T00:00:03.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.028463379999999996, "avg_duration_per_task_ms": 583244.64, "avg_output_tokens_per_task": 43866.54, "last_run_timestamp": "2026-08-08T08:27:52.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.72, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.037352619999999996, "avg_duration_per_task_ms": 206767.2, "avg_output_tokens_per_task": 11957.14, "last_run_timestamp": "2026-09-26T08:10:43.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.7194656011450382, "accuracy_std_dev": 0.0453230167418833, "run_count": 21, "total_questions": 1048, "avg_cost_per_task_usd": 0.04039500858778625, "avg_duration_per_task_ms": 306786.39503816795, "avg_output_tokens_per_task": 8693.121183206107, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.7192630452261307, "accuracy_std_dev": 0.03505636837505419, "run_count": 20, "total_questions": 995, "avg_cost_per_task_usd": 0.11109498492462312, "avg_duration_per_task_ms": 145472.49246231155, "avg_output_tokens_per_task": 8249.809045226131, "last_run_timestamp": "2026-09-26T08:14:43.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.7191666250000001, "accuracy_std_dev": 0.020666060140587385, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0268572725, "avg_duration_per_task_ms": 236107.135, "avg_output_tokens_per_task": 10157.88, "last_run_timestamp": "2026-08-29T17:16:15.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.7190475714285715, "accuracy_std_dev": 0.02815552369991236, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3010150485714286, "avg_duration_per_task_ms": 421383.48, "avg_output_tokens_per_task": 9299.894285714287, "last_run_timestamp": "2026-09-26T08:07:18.968Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.7186897976878613, "accuracy_std_dev": 0.027015736117292515, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.02169187283236994, "avg_duration_per_task_ms": 314303.23988439306, "avg_output_tokens_per_task": 7575.85549132948, "last_run_timestamp": "2026-08-29T16:48:46.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.7185660561797753, "accuracy_std_dev": 0.0276044455361813, "run_count": 25, "total_questions": 1246, "avg_cost_per_task_usd": 0.0226209036918138, "avg_duration_per_task_ms": 287670.0088282504, "avg_output_tokens_per_task": 8543.418138041734, "last_run_timestamp": "2026-08-29T16:11:34.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.7184368957915831, "accuracy_std_dev": 0.08213389724443859, "run_count": 20, "total_questions": 998, "avg_cost_per_task_usd": 0.05849042284569138, "avg_duration_per_task_ms": 494076.376753507, "avg_output_tokens_per_task": 37893.49498997996, "last_run_timestamp": "2026-08-29T16:07:02.358Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7184294759615385, "accuracy_std_dev": 0.1596697904675889, "run_count": 21, "total_questions": 1040, "avg_cost_per_task_usd": 0.00781735096153846, "avg_duration_per_task_ms": 183262.19615384616, "avg_output_tokens_per_task": 7099.238461538462, "last_run_timestamp": "2026-08-29T16:33:29.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.71777775, "accuracy_std_dev": 0.039283684924798404, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.038988519166666666, "avg_duration_per_task_ms": 167526.4525, "avg_output_tokens_per_task": 5438.175, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7177420000000001, "accuracy_std_dev": 0.05717499919930039, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.2799884032258065, "avg_duration_per_task_ms": 377640.3185483871, "avg_output_tokens_per_task": 9988.213709677419, "last_run_timestamp": "2026-09-26T08:10:43.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7177330000000001, "accuracy_std_dev": 0.05885214850291977, "run_count": 12, "total_questions": 594, "avg_cost_per_task_usd": 0.12715290404040405, "avg_duration_per_task_ms": 1096410.3063973065, "avg_output_tokens_per_task": 23858.210437710437, "last_run_timestamp": "2026-09-26T08:12:28.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7176923076923077, "accuracy_std_dev": 0.02704492670577441, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.06838480846153847, "avg_duration_per_task_ms": 195077.87692307692, "avg_output_tokens_per_task": 9816.39, "last_run_timestamp": "2026-08-29T17:17:21.072Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.717499875, "accuracy_std_dev": 0.027474714386311185, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.35365771499999993, "avg_duration_per_task_ms": 238504.995, "avg_output_tokens_per_task": 10803.1675, "last_run_timestamp": "2026-09-26T08:10:56.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.7173334, "accuracy_std_dev": 0.025508314986294175, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.327251336, "avg_duration_per_task_ms": 427459.224, "avg_output_tokens_per_task": 9077.78, "last_run_timestamp": "2026-09-26T08:23:18.418Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.7173334, "accuracy_std_dev": 0.050526018820405805, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.171608576, "avg_duration_per_task_ms": 290919.704, "avg_output_tokens_per_task": 9956.296, "last_run_timestamp": "2026-09-26T08:22:17.301Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.71733335, "accuracy_std_dev": 0.14592836629294353, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.013306622999999998, "avg_duration_per_task_ms": 168498.887, "avg_output_tokens_per_task": 7340.109, "last_run_timestamp": "2026-08-29T16:25:14.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.7172150901287554, "accuracy_std_dev": 0.047887933631073305, "run_count": 14, "total_questions": 699, "avg_cost_per_task_usd": 0.027927947067238917, "avg_duration_per_task_ms": 173112.5693848355, "avg_output_tokens_per_task": 4396.263233190271, "last_run_timestamp": "2026-07-25T08:07:26.285Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.7171427857142857, "accuracy_std_dev": 0.030728103205322304, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.06684413571428571, "avg_duration_per_task_ms": 229014.34285714285, "avg_output_tokens_per_task": 7689.901428571428, "last_run_timestamp": "2026-09-26T08:05:12.739Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "ModelRun", "accuracy": 0.7165880636792453, "accuracy_std_dev": 0.056348748072109975, "run_count": 17, "total_questions": 848, "avg_cost_per_task_usd": 0.01914819929245283, "avg_duration_per_task_ms": 55455.03773584906, "avg_output_tokens_per_task": 4000.7818396226417, "last_run_timestamp": "2026-07-11T08:05:22.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "DeepInfra", "accuracy": 0.7162163243243244, "accuracy_std_dev": 0.04755751813214069, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.07101010135135136, "avg_duration_per_task_ms": 195853.2972972973, "avg_output_tokens_per_task": 10371.256756756757, "last_run_timestamp": "2026-06-30T19:48:55.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.7161458482142857, "accuracy_std_dev": 0.04047657579390825, "run_count": 18, "total_questions": 896, "avg_cost_per_task_usd": 0.03766043638392857, "avg_duration_per_task_ms": 222484.6875, "avg_output_tokens_per_task": 6775.514508928572, "last_run_timestamp": "2026-08-29T16:33:11.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.716111, "accuracy_std_dev": 0.031056479812324726, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.006504103333333333, "avg_duration_per_task_ms": 381723.9266666667, "avg_output_tokens_per_task": 12559.69, "last_run_timestamp": "2026-09-26T08:05:12.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7155646528925619, "accuracy_std_dev": 0.07922982153103719, "run_count": 5, "total_questions": 242, "avg_cost_per_task_usd": 0.11734726446280992, "avg_duration_per_task_ms": 254637.0785123967, "avg_output_tokens_per_task": 42006.561983471074, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7155556666666666, "accuracy_std_dev": 0.040855070557058414, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012743326666666667, "avg_duration_per_task_ms": 1224143.2066666668, "avg_output_tokens_per_task": 18669.653333333332, "last_run_timestamp": "2026-08-29T09:20:05.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.7152931648106904, "accuracy_std_dev": 0.033648368891044014, "run_count": 18, "total_questions": 898, "avg_cost_per_task_usd": 0.047023087973273946, "avg_duration_per_task_ms": 373630.32516703784, "avg_output_tokens_per_task": 7851.192650334076, "last_run_timestamp": "2026-08-01T17:38:16.661Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.715238, "accuracy_std_dev": 0.036811758695907394, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0628034857142857, "avg_duration_per_task_ms": 75005.29714285715, "avg_output_tokens_per_task": 9080.965714285714, "last_run_timestamp": "2026-09-26T08:24:48.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.715, "accuracy_std_dev": 0.04499999999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.16922785, "avg_duration_per_task_ms": 299708.78, "avg_output_tokens_per_task": 9901.85, "last_run_timestamp": "2026-08-29T08:38:56.982Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.7147222083333333, "accuracy_std_dev": 0.03967363744034902, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.03867999416666666, "avg_duration_per_task_ms": 344445.3625, "avg_output_tokens_per_task": 12226.0975, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.7145061296296297, "accuracy_std_dev": 0.033281830759235444, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.10411070679012345, "avg_duration_per_task_ms": 92462.91358024691, "avg_output_tokens_per_task": 10516.800925925925, "last_run_timestamp": "2026-09-26T08:22:18.077Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.7144443333333333, "accuracy_std_dev": 0.0015716626723173276, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04586062666666667, "avg_duration_per_task_ms": 212784.84, "avg_output_tokens_per_task": 30659.346666666668, "last_run_timestamp": "2026-09-26T08:22:27.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.7142854897959183, "accuracy_std_dev": 0.039861000000000035, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.11979946938775511, "avg_duration_per_task_ms": 677914.6734693878, "avg_output_tokens_per_task": 54287.663265306124, "last_run_timestamp": "2026-09-01T17:30:15.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.7141900923694778, "accuracy_std_dev": 0.03172465567661847, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.03655370080321286, "avg_duration_per_task_ms": 44500.680722891564, "avg_output_tokens_per_task": 5750.947791164659, "last_run_timestamp": "2026-08-08T08:31:07.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.7140255173041894, "accuracy_std_dev": 0.0345616737838867, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.06628586703096538, "avg_duration_per_task_ms": 272000.2349726776, "avg_output_tokens_per_task": 8738.27504553734, "last_run_timestamp": "2026-09-26T08:25:04.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.7135835603217157, "accuracy_std_dev": 0.05817844462211832, "run_count": 15, "total_questions": 746, "avg_cost_per_task_usd": 0.024379721179624666, "avg_duration_per_task_ms": 139322.754691689, "avg_output_tokens_per_task": 6947.269436997319, "last_run_timestamp": "2026-09-26T16:00:18.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.7133335, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.028499629999999998, "avg_duration_per_task_ms": 176126.96, "avg_output_tokens_per_task": 22001.83, "last_run_timestamp": "2026-09-26T08:08:19.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7133335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06446003, "avg_duration_per_task_ms": 348420.47, "avg_output_tokens_per_task": 34152.4, "last_run_timestamp": "2026-09-01T12:41:25.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.7133333913043478, "accuracy_std_dev": 0.03565804311334216, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.04301751304347827, "avg_duration_per_task_ms": 182842.47913043477, "avg_output_tokens_per_task": 6153.3460869565215, "last_run_timestamp": "2026-08-30T00:01:09.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.7133333333333334, "accuracy_std_dev": 0.04268744285956807, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04181986333333333, "avg_duration_per_task_ms": 299500.48, "avg_output_tokens_per_task": 13842.47, "last_run_timestamp": "2026-09-26T08:13:28.082Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Together", "accuracy": 0.7133333333333334, "accuracy_std_dev": 0.02211083193570266, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03602225, "avg_duration_per_task_ms": 85193.79666666666, "avg_output_tokens_per_task": 3088.5833333333335, "last_run_timestamp": "2026-05-16T13:21:53.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.7133333333333334, "accuracy_std_dev": 0.019625871060640555, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12069053999999999, "avg_duration_per_task_ms": 78204.86666666667, "avg_output_tokens_per_task": 4256.786666666667, "last_run_timestamp": "2026-09-01T12:59:40.225Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7133330000000001, "accuracy_std_dev": 0.02828427124746193, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.031477000000000005, "avg_duration_per_task_ms": 184080.18666666668, "avg_output_tokens_per_task": 16016.74, "last_run_timestamp": "2026-09-26T08:01:16.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.7133330000000001, "accuracy_std_dev": 0.040000000000000036, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07565915000000001, "avg_duration_per_task_ms": 95308.62, "avg_output_tokens_per_task": 3758.94, "last_run_timestamp": "2026-09-01T13:00:41.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.043553119999999994, "avg_duration_per_task_ms": 152550.44, "avg_output_tokens_per_task": 34185.24, "last_run_timestamp": "2026-09-26T08:22:24.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14862698, "avg_duration_per_task_ms": 506333.72, "avg_output_tokens_per_task": 12832.14, "last_run_timestamp": "2026-09-26T16:00:25.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.57503108, "avg_duration_per_task_ms": 657829.88, "avg_output_tokens_per_task": 51958.96, "last_run_timestamp": "2026-09-01T13:00:31.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0723164, "avg_duration_per_task_ms": 346522.38, "avg_output_tokens_per_task": 59152.72, "last_run_timestamp": "2026-09-12T16:00:24.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.7131498513761468, "accuracy_std_dev": 0.15547389772642844, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.07655383853211008, "avg_duration_per_task_ms": 42631.811009174315, "avg_output_tokens_per_task": 4027.154128440367, "last_run_timestamp": "2026-07-18T08:06:26.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.7130758729674798, "accuracy_std_dev": 0.03492932464076567, "run_count": 20, "total_questions": 984, "avg_cost_per_task_usd": 0.09219651422764227, "avg_duration_per_task_ms": 75102.03760162601, "avg_output_tokens_per_task": 7773.928861788618, "last_run_timestamp": "2026-09-26T08:12:44.279Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.713058298969072, "accuracy_std_dev": 0.025297499999999973, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.027766309278350512, "avg_duration_per_task_ms": 54271.969072164946, "avg_output_tokens_per_task": 11929.432989690722, "last_run_timestamp": "2026-09-26T08:12:00.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.7128571904761904, "accuracy_std_dev": 0.03649001975833554, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.043122668571428574, "avg_duration_per_task_ms": 201968.71333333335, "avg_output_tokens_per_task": 4435.060952380953, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7127330527950311, "accuracy_std_dev": 0.06987907508253477, "run_count": 13, "total_questions": 644, "avg_cost_per_task_usd": 0.04331319254658385, "avg_duration_per_task_ms": 245149.61645962732, "avg_output_tokens_per_task": 24723.41149068323, "last_run_timestamp": "2026-09-26T16:00:24.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.7123519847715736, "accuracy_std_dev": 0.02073271203919062, "run_count": 4, "total_questions": 197, "avg_cost_per_task_usd": 0.08227161928934011, "avg_duration_per_task_ms": 178521.6345177665, "avg_output_tokens_per_task": 15413.27918781726, "last_run_timestamp": "2026-07-18T08:00:15.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.712, "accuracy_std_dev": 0.03249611258904671, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.08037907, "avg_duration_per_task_ms": 74455.182, "avg_output_tokens_per_task": 8504.276, "last_run_timestamp": "2026-09-26T08:11:40.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.712, "accuracy_std_dev": 0.023626914974240687, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.34044856000000007, "avg_duration_per_task_ms": 188002.144, "avg_output_tokens_per_task": 11667.488, "last_run_timestamp": "2026-09-26T08:22:45.792Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.7119443333333334, "accuracy_std_dev": 0.045091638517825244, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.0304885225, "avg_duration_per_task_ms": 128203.68083333333, "avg_output_tokens_per_task": 5257.9591666666665, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7117119324324325, "accuracy_std_dev": 0.01247848271581482, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.6902402702702702, "avg_duration_per_task_ms": 688360.5945945946, "avg_output_tokens_per_task": 56108.03378378379, "last_run_timestamp": "2026-09-01T13:02:58.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.7116669999999999, "accuracy_std_dev": 0.008660254037844347, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.013152875000000001, "avg_duration_per_task_ms": 1125576.36, "avg_output_tokens_per_task": 15885.82, "last_run_timestamp": "2026-09-26T08:11:57.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.7115385708502023, "accuracy_std_dev": 0.03244045710975723, "run_count": 10, "total_questions": 494, "avg_cost_per_task_usd": 0.021128163967611337, "avg_duration_per_task_ms": 99825.94129554655, "avg_output_tokens_per_task": 6537.080971659919, "last_run_timestamp": "2026-09-26T16:00:20.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.7115308465171193, "accuracy_std_dev": 0.1512936932248644, "run_count": 17, "total_questions": 847, "avg_cost_per_task_usd": 0.0987406765053129, "avg_duration_per_task_ms": 230873.95631641085, "avg_output_tokens_per_task": 6065.546635182999, "last_run_timestamp": "2026-08-29T16:35:10.931Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Fireworks", "accuracy": 0.7114285714285714, "accuracy_std_dev": 0.04323641700120444, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.013884184285714283, "avg_duration_per_task_ms": 81133.41285714286, "avg_output_tokens_per_task": 3767.2885714285712, "last_run_timestamp": "2026-06-20T08:38:34.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.711111, "accuracy_std_dev": 0.016629528997138404, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03228578, "avg_duration_per_task_ms": 111202.65333333334, "avg_output_tokens_per_task": 22255.406666666666, "last_run_timestamp": "2026-09-26T08:17:01.712Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7109804117647058, "accuracy_std_dev": 0.05357489628334922, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.07005771294117646, "avg_duration_per_task_ms": 197809.54470588235, "avg_output_tokens_per_task": 7675.065882352941, "last_run_timestamp": "2026-08-29T16:43:32.997Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7109091363636364, "accuracy_std_dev": 0.03827314426745951, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.04138232000000001, "avg_duration_per_task_ms": 208650.05909090908, "avg_output_tokens_per_task": 6790.424545454545, "last_run_timestamp": "2026-08-29T17:03:20.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.710762307174888, "accuracy_std_dev": 0.08669531485242507, "run_count": 18, "total_questions": 892, "avg_cost_per_task_usd": 0.05282534865470852, "avg_duration_per_task_ms": 474386.9360986547, "avg_output_tokens_per_task": 38674.68161434978, "last_run_timestamp": "2026-08-29T17:09:45.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.7107022775919732, "accuracy_std_dev": 0.14302186991804494, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.10911909030100336, "avg_duration_per_task_ms": 144349.8110367893, "avg_output_tokens_per_task": 8000.060200668896, "last_run_timestamp": "2026-09-26T08:22:30.723Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.7102565384615385, "accuracy_std_dev": 0.0380387521707805, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.04511441076923077, "avg_duration_per_task_ms": 639644.943076923, "avg_output_tokens_per_task": 6877.484615384616, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.71, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.009475860000000001, "avg_duration_per_task_ms": 101117.88, "avg_output_tokens_per_task": 5437.48, "last_run_timestamp": "2026-09-01T17:32:44.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DekaLLM", "accuracy": 0.71, "accuracy_std_dev": 0.043588989435406726, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02106588, "avg_duration_per_task_ms": 66495.715, "avg_output_tokens_per_task": 1737.11, "last_run_timestamp": "2026-05-02T22:49:37.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7094593108108108, "accuracy_std_dev": 0.025596061967072654, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.29917547972972974, "avg_duration_per_task_ms": 177661.23648648648, "avg_output_tokens_per_task": 4943.472972972973, "last_run_timestamp": "2026-09-01T13:00:41.783Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.709333, "accuracy_std_dev": 0.034409301068170535, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.23672154, "avg_duration_per_task_ms": 252134.184, "avg_output_tokens_per_task": 20169.304, "last_run_timestamp": "2026-09-26T08:14:47.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7091838979591837, "accuracy_std_dev": 0.022986000000000006, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.021414438775510206, "avg_duration_per_task_ms": 174657.33673469388, "avg_output_tokens_per_task": 20207.65306122449, "last_run_timestamp": "2026-09-26T08:07:31.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.7091666875, "accuracy_std_dev": 0.07271997399676271, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.047436745, "avg_duration_per_task_ms": 166812.7925, "avg_output_tokens_per_task": 6450.30625, "last_run_timestamp": "2026-08-29T16:45:57.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.7091665, "accuracy_std_dev": 0.012774768970513724, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01202336, "avg_duration_per_task_ms": 319678.925, "avg_output_tokens_per_task": 17962.82, "last_run_timestamp": "2026-09-26T08:14:04.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.7090909545454546, "accuracy_std_dev": 0.07535893638416402, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.060541024545454535, "avg_duration_per_task_ms": 57240.09636363637, "avg_output_tokens_per_task": 5399.512727272728, "last_run_timestamp": "2026-08-29T16:33:39.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.708889, "accuracy_std_dev": 0.019115994210782406, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01979013333333333, "avg_duration_per_task_ms": 164929.67333333334, "avg_output_tokens_per_task": 14302.1, "last_run_timestamp": "2026-09-26T08:17:02.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.7088888888888889, "accuracy_std_dev": 0.03590109527559924, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.014436273333333333, "avg_duration_per_task_ms": 152553.27555555556, "avg_output_tokens_per_task": 4675.15, "last_run_timestamp": "2026-08-01T17:39:06.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7088886666666667, "accuracy_std_dev": 0.0339208519579853, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08842117999999999, "avg_duration_per_task_ms": 151404.06666666668, "avg_output_tokens_per_task": 46789.46333333333, "last_run_timestamp": "2026-09-01T17:31:47.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7088886666666667, "accuracy_std_dev": 0.025724129999325985, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.062267260000000005, "avg_duration_per_task_ms": 262899.92, "avg_output_tokens_per_task": 8890.053333333333, "last_run_timestamp": "2026-09-26T08:07:15.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7087542545454545, "accuracy_std_dev": 0.05428965504228222, "run_count": 10, "total_questions": 495, "avg_cost_per_task_usd": 0.050922222222222216, "avg_duration_per_task_ms": 787128.494949495, "avg_output_tokens_per_task": 54707.670707070705, "last_run_timestamp": "2026-09-26T16:00:25.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.7081481666666667, "accuracy_std_dev": 0.029106428330162543, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.030871916666666666, "avg_duration_per_task_ms": 204088.72444444444, "avg_output_tokens_per_task": 6242.532222222222, "last_run_timestamp": "2026-08-29T16:33:11.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.708, "accuracy_std_dev": 0.024367428169587355, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.117385436, "avg_duration_per_task_ms": 671239.06, "avg_output_tokens_per_task": 25704.02, "last_run_timestamp": "2026-09-26T08:22:25.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7078693118040089, "accuracy_std_dev": 0.037495695230779585, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.13154344097995546, "avg_duration_per_task_ms": 74759.52783964365, "avg_output_tokens_per_task": 10228.11135857461, "last_run_timestamp": "2026-08-29T17:11:32.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7073577859531772, "accuracy_std_dev": 0.05214988620366833, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.0982930769230769, "avg_duration_per_task_ms": 117571.96321070234, "avg_output_tokens_per_task": 8895.745819397993, "last_run_timestamp": "2026-09-26T08:22:36.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.7073044861111111, "accuracy_std_dev": 0.04160167414852507, "run_count": 26, "total_questions": 1296, "avg_cost_per_task_usd": 0.018744344907407406, "avg_duration_per_task_ms": 142416.33564814815, "avg_output_tokens_per_task": 6045.990740740741, "last_run_timestamp": "2026-08-29T16:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.706867824120603, "accuracy_std_dev": 0.024112945099248537, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.12147276381909548, "avg_duration_per_task_ms": 366567.0226130653, "avg_output_tokens_per_task": 8862.291457286432, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openrouter/auto-beta", "model_slug": "openrouter/auto-beta", "model_name": "Auto Router (Beta)", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2407015, "avg_duration_per_task_ms": 149076.32, "avg_output_tokens_per_task": 17938.72, "last_run_timestamp": "2026-07-22T00:25:39.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.3541182, "avg_duration_per_task_ms": 1646927.84, "avg_output_tokens_per_task": 24534.74, "last_run_timestamp": "2026-09-10T00:00:02.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02431272, "avg_duration_per_task_ms": 304327.62, "avg_output_tokens_per_task": 14297.9, "last_run_timestamp": "2026-09-12T08:11:25.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08614948, "avg_duration_per_task_ms": 52133.28, "avg_output_tokens_per_task": 7649.3, "last_run_timestamp": "2026-07-18T08:04:46.262Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00934721, "avg_duration_per_task_ms": 103901.71, "avg_output_tokens_per_task": 5395.91, "last_run_timestamp": "2026-09-01T17:30:15.713Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1838298, "avg_duration_per_task_ms": 329811.9, "avg_output_tokens_per_task": 9540.84, "last_run_timestamp": "2026-08-15T00:16:18.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06360505, "avg_duration_per_task_ms": 327814.04, "avg_output_tokens_per_task": 33622.71, "last_run_timestamp": "2026-09-01T13:01:01.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.70666675, "accuracy_std_dev": 0.03496013602930485, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.053884480000000005, "avg_duration_per_task_ms": 561783.395, "avg_output_tokens_per_task": 5383.42, "last_run_timestamp": "2026-09-26T08:28:05.282Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.042424746666666666, "avg_duration_per_task_ms": 131868.91333333333, "avg_output_tokens_per_task": 15613.653333333334, "last_run_timestamp": "2026-09-26T08:23:01.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.033993463423951896, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017786823333333337, "avg_duration_per_task_ms": 128082.29, "avg_output_tokens_per_task": 8061.84, "last_run_timestamp": "2026-05-23T11:46:07.128Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.706237370221328, "accuracy_std_dev": 0.032130056570911926, "run_count": 10, "total_questions": 497, "avg_cost_per_task_usd": 0.37140224144869216, "avg_duration_per_task_ms": 217046.72837022133, "avg_output_tokens_per_task": 8219.961770623742, "last_run_timestamp": "2026-09-26T08:16:17.130Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.706111138888889, "accuracy_std_dev": 0.032974923600410835, "run_count": 36, "total_questions": 1800, "avg_cost_per_task_usd": 0.025842573888888876, "avg_duration_per_task_ms": 147179.83222222223, "avg_output_tokens_per_task": 6878.078888888889, "last_run_timestamp": "2026-08-29T16:54:26.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7057822423469389, "accuracy_std_dev": 0.08819684229599366, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.025861540816326536, "avg_duration_per_task_ms": 1299840.9795918367, "avg_output_tokens_per_task": 37856.7193877551, "last_run_timestamp": "2026-09-26T16:00:24.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7055556666666667, "accuracy_std_dev": 0.03430073605462263, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.038495376666666664, "avg_duration_per_task_ms": 114809.37333333334, "avg_output_tokens_per_task": 5174.493333333333, "last_run_timestamp": "2026-09-01T17:31:46.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7052709777282851, "accuracy_std_dev": 0.1455296969729975, "run_count": 18, "total_questions": 898, "avg_cost_per_task_usd": 0.08249193763919824, "avg_duration_per_task_ms": 252331.7171492205, "avg_output_tokens_per_task": 5844.012249443207, "last_run_timestamp": "2026-08-29T16:44:15.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.7046486190476191, "accuracy_std_dev": 0.037923417525396554, "run_count": 6, "total_questions": 294, "avg_cost_per_task_usd": 0.11717974149659866, "avg_duration_per_task_ms": 474565.40476190473, "avg_output_tokens_per_task": 22901.343537414967, "last_run_timestamp": "2026-08-22T08:19:48.101Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Parasail", "accuracy": 0.704, "accuracy_std_dev": 0.047159304490206375, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.022172070000000002, "avg_duration_per_task_ms": 76616.802, "avg_output_tokens_per_task": 3006.926, "last_run_timestamp": "2026-06-06T08:18:22.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7036363636363636, "accuracy_std_dev": 0.0226280216677252, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.008178456363636363, "avg_duration_per_task_ms": 314792.5272727273, "avg_output_tokens_per_task": 14051.845454545455, "last_run_timestamp": "2026-09-26T08:22:46.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7033121966173361, "accuracy_std_dev": 0.09340545024178089, "run_count": 19, "total_questions": 946, "avg_cost_per_task_usd": 0.11567343551797042, "avg_duration_per_task_ms": 101375.4778012685, "avg_output_tokens_per_task": 8107.707188160676, "last_run_timestamp": "2026-09-26T08:26:14.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.70277775, "accuracy_std_dev": 0.05294707729284183, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.039451974166666674, "avg_duration_per_task_ms": 188705.17083333334, "avg_output_tokens_per_task": 5188.7, "last_run_timestamp": "2026-08-29T16:33:36.567Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.4077153666666667, "avg_duration_per_task_ms": 153641.3, "avg_output_tokens_per_task": 23369.493333333332, "last_run_timestamp": "2026-09-01T17:32:03.083Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.7015590066815145, "accuracy_std_dev": 0.12728152373817306, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.049956171492204896, "avg_duration_per_task_ms": 149113.59020044544, "avg_output_tokens_per_task": 5756.837416481069, "last_run_timestamp": "2026-08-15T08:13:33.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.7004762142857143, "accuracy_std_dev": 0.07309331857307827, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.024353129999999997, "avg_duration_per_task_ms": 88796.36857142857, "avg_output_tokens_per_task": 6881.517142857143, "last_run_timestamp": "2026-09-26T16:00:24.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Nebius", "accuracy": 0.7, "accuracy_std_dev": 0.03999999999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06787243, "avg_duration_per_task_ms": 42898.22, "avg_output_tokens_per_task": 4562.86, "last_run_timestamp": "2026-05-23T10:53:37.096Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.31292506, "avg_duration_per_task_ms": 261751.42, "avg_output_tokens_per_task": 14948.8, "last_run_timestamp": "2026-08-01T08:06:08.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.9826790200000001, "avg_duration_per_task_ms": 401624.28, "avg_output_tokens_per_task": 9843.34, "last_run_timestamp": "2026-08-26T00:00:04.572Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.7, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00948338, "avg_duration_per_task_ms": 87248.24, "avg_output_tokens_per_task": 5625.9, "last_run_timestamp": "2026-09-26T08:10:43.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0112321, "avg_duration_per_task_ms": 596563.34, "avg_output_tokens_per_task": 4097.46, "last_run_timestamp": "2026-09-12T09:21:58.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.6995833125, "accuracy_std_dev": 0.02938897155975083, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.033650589999999994, "avg_duration_per_task_ms": 154427.47, "avg_output_tokens_per_task": 6103.86125, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.6994872307692308, "accuracy_std_dev": 0.031968156822673593, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.03178479384615384, "avg_duration_per_task_ms": 200835.56307692308, "avg_output_tokens_per_task": 5041.824615384616, "last_run_timestamp": "2026-08-01T17:38:23.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.6989348609022557, "accuracy_std_dev": 0.11823452094841348, "run_count": 11, "total_questions": 532, "avg_cost_per_task_usd": 0.05432932518796992, "avg_duration_per_task_ms": 113791.10338345864, "avg_output_tokens_per_task": 7205.951127819549, "last_run_timestamp": "2026-09-26T08:08:33.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.69833325, "accuracy_std_dev": 0.009859949857250788, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.14240881, "avg_duration_per_task_ms": 514302.935, "avg_output_tokens_per_task": 58927.42, "last_run_timestamp": "2026-09-01T17:31:48.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.6966669999999999, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12995234, "avg_duration_per_task_ms": 694377.26, "avg_output_tokens_per_task": 58360.09, "last_run_timestamp": "2026-09-01T17:23:14.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.69666675, "accuracy_std_dev": 0.019720195515955205, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.27175437, "avg_duration_per_task_ms": 364436.225, "avg_output_tokens_per_task": 35416.46, "last_run_timestamp": "2026-09-01T13:04:22.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.6958332500000001, "accuracy_std_dev": 0.010374963189693743, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.042198395, "avg_duration_per_task_ms": 172150.8225, "avg_output_tokens_per_task": 5272.2875, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6955437880386983, "accuracy_std_dev": 0.0690966463341572, "run_count": 23, "total_questions": 1137, "avg_cost_per_task_usd": 0.09496287071240103, "avg_duration_per_task_ms": 258193.17766051012, "avg_output_tokens_per_task": 6953.824978012313, "last_run_timestamp": "2026-08-29T16:44:17.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Parasail", "accuracy": 0.695, "accuracy_std_dev": 0.038405728739343024, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.020702055, "avg_duration_per_task_ms": 135918.28, "avg_output_tokens_per_task": 3857.625, "last_run_timestamp": "2026-05-16T23:02:52.223Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.6949153322033899, "accuracy_std_dev": 0.055158692701843356, "run_count": 12, "total_questions": 590, "avg_cost_per_task_usd": 0.02756941186440678, "avg_duration_per_task_ms": 850370.9288135594, "avg_output_tokens_per_task": 36904.2406779661, "last_run_timestamp": "2026-09-26T16:00:18.485Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.6948148333333334, "accuracy_std_dev": 0.02859300565843564, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.024409364444444435, "avg_duration_per_task_ms": 211749.8111111111, "avg_output_tokens_per_task": 5141.246666666667, "last_run_timestamp": "2026-08-01T17:38:21.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.6947618571428572, "accuracy_std_dev": 0.029726664088604592, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.03172098571428572, "avg_duration_per_task_ms": 126873.46571428572, "avg_output_tokens_per_task": 4470.107142857143, "last_run_timestamp": "2026-07-18T08:04:08.785Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.6946935773542601, "accuracy_std_dev": 0.21264485878543257, "run_count": 18, "total_questions": 892, "avg_cost_per_task_usd": 0.10174442825112107, "avg_duration_per_task_ms": 197126.54596412557, "avg_output_tokens_per_task": 10724.269058295964, "last_run_timestamp": "2026-07-18T16:00:09.881Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.694166625, "accuracy_std_dev": 0.06141286114475023, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.3036356975, "avg_duration_per_task_ms": 394520.48, "avg_output_tokens_per_task": 9370.85, "last_run_timestamp": "2026-09-26T08:08:19.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Chutes", "accuracy": 0.693333375, "accuracy_std_dev": 0.1814600292688017, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.042266376249999994, "avg_duration_per_task_ms": 100861.29625, "avg_output_tokens_per_task": 3413.32875, "last_run_timestamp": "2026-07-04T16:00:23.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6933331428571429, "accuracy_std_dev": 0.02519760633715926, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.11111235000000001, "avg_duration_per_task_ms": 77728.44714285714, "avg_output_tokens_per_task": 9319.027142857143, "last_run_timestamp": "2026-09-26T08:24:41.403Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015592, "avg_duration_per_task_ms": 1084747.56, "avg_output_tokens_per_task": 17745.48, "last_run_timestamp": "2026-07-31T03:44:20.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02187568, "avg_duration_per_task_ms": 236451.76, "avg_output_tokens_per_task": 11760.74, "last_run_timestamp": "2026-07-18T08:04:03.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.6913580481481482, "accuracy_std_dev": 0.1030309433019321, "run_count": 11, "total_questions": 540, "avg_cost_per_task_usd": 0.1584400537037037, "avg_duration_per_task_ms": 922948.7851851851, "avg_output_tokens_per_task": 40801.18148148148, "last_run_timestamp": "2026-09-26T08:11:26.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.6905443180515759, "accuracy_std_dev": 0.13570623589352024, "run_count": 14, "total_questions": 698, "avg_cost_per_task_usd": 0.055779584527220635, "avg_duration_per_task_ms": 503090.9255014327, "avg_output_tokens_per_task": 6839.69770773639, "last_run_timestamp": "2026-09-26T08:22:17.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.6904283835341366, "accuracy_std_dev": 0.027558639980730167, "run_count": 20, "total_questions": 996, "avg_cost_per_task_usd": 0.012428607429718873, "avg_duration_per_task_ms": 150833.32128514056, "avg_output_tokens_per_task": 5172.266064257028, "last_run_timestamp": "2026-08-01T17:38:51.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.6901754210526316, "accuracy_std_dev": 0.039015354444811766, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.014771705263157893, "avg_duration_per_task_ms": 199050.82631578948, "avg_output_tokens_per_task": 12000.06, "last_run_timestamp": "2026-08-29T17:51:56.583Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6900003333333334, "accuracy_std_dev": 0.012472191289246483, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04195019333333334, "avg_duration_per_task_ms": 246231.66666666666, "avg_output_tokens_per_task": 31170.286666666667, "last_run_timestamp": "2026-09-24T09:00:05.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.69, "accuracy_std_dev": 0.032088096328701056, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.18925239666666666, "avg_duration_per_task_ms": 1912773.68, "avg_output_tokens_per_task": 13075.366666666667, "last_run_timestamp": "2026-08-29T16:50:22.450Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "AtlasCloud", "accuracy": 0.69, "accuracy_std_dev": 0.038729833462074155, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.04847448249999999, "avg_duration_per_task_ms": 102036.4875, "avg_output_tokens_per_task": 3335.44, "last_run_timestamp": "2026-06-20T12:59:17.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "SiliconFlow", "accuracy": 0.69, "accuracy_std_dev": 0.04999999999999999, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04272526, "avg_duration_per_task_ms": 284841.43, "avg_output_tokens_per_task": 3214.01, "last_run_timestamp": "2026-05-02T08:48:58.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.6899998333333334, "accuracy_std_dev": 0.03415651881860261, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09539628666666666, "avg_duration_per_task_ms": 502649.19, "avg_output_tokens_per_task": 22095.93666666667, "last_run_timestamp": "2026-08-15T08:15:24.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.6893332666666667, "accuracy_std_dev": 0.03976225718352628, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.0426068, "avg_duration_per_task_ms": 444126.1826666667, "avg_output_tokens_per_task": 13135.968, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.689300549382716, "accuracy_std_dev": 0.06636481757428885, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.10339247685185184, "avg_duration_per_task_ms": 75157.70987654322, "avg_output_tokens_per_task": 8117.143518518518, "last_run_timestamp": "2026-09-26T08:22:29.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.689041575052854, "accuracy_std_dev": 0.03094305825454303, "run_count": 19, "total_questions": 946, "avg_cost_per_task_usd": 0.021175065539112053, "avg_duration_per_task_ms": 438783.39852008456, "avg_output_tokens_per_task": 11967.172304439746, "last_run_timestamp": "2026-08-29T17:42:21.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.688717923076923, "accuracy_std_dev": 0.1628459988939345, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.052740692307692315, "avg_duration_per_task_ms": 145669.94461538462, "avg_output_tokens_per_task": 17932.95076923077, "last_run_timestamp": "2026-09-26T16:00:25.686Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.6880856967871486, "accuracy_std_dev": 0.11576392654211201, "run_count": 20, "total_questions": 996, "avg_cost_per_task_usd": 0.047004783132530134, "avg_duration_per_task_ms": 101938.27108433735, "avg_output_tokens_per_task": 4039.1506024096384, "last_run_timestamp": "2026-08-01T17:38:27.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.6878611676300578, "accuracy_std_dev": 0.042049584814859244, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.013951661849710981, "avg_duration_per_task_ms": 210919.60693641618, "avg_output_tokens_per_task": 8494.329479768787, "last_run_timestamp": "2026-08-01T17:38:31.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.6875626118355065, "accuracy_std_dev": 0.0373234995892668, "run_count": 20, "total_questions": 997, "avg_cost_per_task_usd": 0.03210182146439318, "avg_duration_per_task_ms": 179885.19558676027, "avg_output_tokens_per_task": 7259.881644934804, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1282767, "avg_duration_per_task_ms": 113186.12, "avg_output_tokens_per_task": 4988.34, "last_run_timestamp": "2026-09-23T15:00:14.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.6866666666666666, "accuracy_std_dev": 0.005443174461857887, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.009258993333333333, "avg_duration_per_task_ms": 105180.14666666667, "avg_output_tokens_per_task": 8673.2, "last_run_timestamp": "2026-09-23T06:00:08.557Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6866663333333333, "accuracy_std_dev": 0.024944382578492918, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013524920000000001, "avg_duration_per_task_ms": 73418.06, "avg_output_tokens_per_task": 5390.6866666666665, "last_run_timestamp": "2026-09-01T17:31:44.289Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.6857144285714285, "accuracy_std_dev": 0.042077516201333307, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.030182771428571425, "avg_duration_per_task_ms": 323904.1228571429, "avg_output_tokens_per_task": 10162.908571428572, "last_run_timestamp": "2026-08-29T16:50:26.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Io Net", "accuracy": 0.684, "accuracy_std_dev": 0.02939387691339811, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011617532, "avg_duration_per_task_ms": 184984.468, "avg_output_tokens_per_task": 3721.176, "last_run_timestamp": "2026-06-13T09:50:28.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.683673530612245, "accuracy_std_dev": 0.03777750000000002, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.6821915918367347, "avg_duration_per_task_ms": 803506.224489796, "avg_output_tokens_per_task": 61161.97959183674, "last_run_timestamp": "2026-09-01T12:56:59.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6826308620689655, "accuracy_std_dev": 0.039302124463935585, "run_count": 21, "total_questions": 1044, "avg_cost_per_task_usd": 0.01650072318007663, "avg_duration_per_task_ms": 286180.8256704981, "avg_output_tokens_per_task": 13045.386973180077, "last_run_timestamp": "2026-08-29T16:12:10.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.6820986111111111, "accuracy_std_dev": 0.16187486847591467, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.06144617438271606, "avg_duration_per_task_ms": 160426.91666666666, "avg_output_tokens_per_task": 7038.361111111111, "last_run_timestamp": "2026-09-26T08:23:24.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "Xiaomi", "accuracy": 0.6816666666666666, "accuracy_std_dev": 0.037822685726367385, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.005967318333333332, "avg_duration_per_task_ms": 82883.09833333333, "avg_output_tokens_per_task": 4062.4183333333335, "last_run_timestamp": "2026-06-13T08:42:56.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.6816424057971014, "accuracy_std_dev": 0.18410546186501534, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.10899141739130434, "avg_duration_per_task_ms": 177787.73623188405, "avg_output_tokens_per_task": 6552.942028985507, "last_run_timestamp": "2026-08-15T08:10:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.6813061126126125, "accuracy_std_dev": 0.17362449557921197, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.04568808558558559, "avg_duration_per_task_ms": 182111.46846846846, "avg_output_tokens_per_task": 5794.326576576576, "last_run_timestamp": "2026-08-15T08:17:17.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.6803882349397591, "accuracy_std_dev": 0.08651381511485608, "run_count": 20, "total_questions": 996, "avg_cost_per_task_usd": 0.04140781124497992, "avg_duration_per_task_ms": 212525.218875502, "avg_output_tokens_per_task": 6209.691767068273, "last_run_timestamp": "2026-08-01T17:38:10.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.6802083125, "accuracy_std_dev": 0.12506379958681027, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.11843063625000003, "avg_duration_per_task_ms": 132394.155, "avg_output_tokens_per_task": 7828.2725, "last_run_timestamp": "2026-09-26T08:16:17.531Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33301777, "avg_duration_per_task_ms": 171524.99, "avg_output_tokens_per_task": 8948.71, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Inceptron", "accuracy": 0.68, "accuracy_std_dev": 0.03999999999999998, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021491569999999998, "avg_duration_per_task_ms": 71328.98, "avg_output_tokens_per_task": 3122.245, "last_run_timestamp": "2026-05-02T08:19:29.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Decart", "accuracy": 0.68, "accuracy_std_dev": 0.03999999999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.028420829999999998, "avg_duration_per_task_ms": 35643.83, "avg_output_tokens_per_task": 3017.33, "last_run_timestamp": "2026-06-20T09:16:42.673Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.65240112, "avg_duration_per_task_ms": 90576.24, "avg_output_tokens_per_task": 3843.06, "last_run_timestamp": "2026-09-26T09:24:36.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.68, "accuracy_std_dev": 0, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.088178175, "avg_duration_per_task_ms": 74028.285, "avg_output_tokens_per_task": 3643.675, "last_run_timestamp": "2026-09-19T08:08:39.705Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06724904000000001, "avg_duration_per_task_ms": 102008.58, "avg_output_tokens_per_task": 2510.66, "last_run_timestamp": "2026-08-22T08:23:03.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Z.AI", "accuracy": 0.68, "accuracy_std_dev": 0.04472135954999579, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.006251205000000001, "avg_duration_per_task_ms": 157772.555, "avg_output_tokens_per_task": 5287.735, "last_run_timestamp": "2026-05-09T21:59:21.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.6788891666666665, "accuracy_std_dev": 0.08738317986587706, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.057002019999999994, "avg_duration_per_task_ms": 515207.3233333333, "avg_output_tokens_per_task": 18064.933333333334, "last_run_timestamp": "2026-08-29T17:31:24.151Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.6783625714285715, "accuracy_std_dev": 0.18503792379476175, "run_count": 24, "total_questions": 1197, "avg_cost_per_task_usd": 0.06683292147034252, "avg_duration_per_task_ms": 46030.891395154555, "avg_output_tokens_per_task": 5800.272347535505, "last_run_timestamp": "2026-08-15T14:12:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "auto-routing", "accuracy": 0.6783333333333333, "accuracy_std_dev": 0.06706133179576904, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.0056993750000000004, "avg_duration_per_task_ms": 43275.725, "avg_output_tokens_per_task": 3567.115, "last_run_timestamp": "2026-06-13T09:16:25.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.6781481111111112, "accuracy_std_dev": 0.03020960748005119, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.050625214444444455, "avg_duration_per_task_ms": 461844.62888888887, "avg_output_tokens_per_task": 6960.154444444444, "last_run_timestamp": "2026-07-18T16:00:08.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.6779184535864979, "accuracy_std_dev": 0.04493753781451566, "run_count": 19, "total_questions": 948, "avg_cost_per_task_usd": 0.05880673312236288, "avg_duration_per_task_ms": 487546.12974683545, "avg_output_tokens_per_task": 9480.43670886076, "last_run_timestamp": "2026-08-01T17:38:07.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.6774691543209876, "accuracy_std_dev": 0.13202811911229878, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.04778613271604938, "avg_duration_per_task_ms": 151992.8487654321, "avg_output_tokens_per_task": 7276.706790123457, "last_run_timestamp": "2026-09-26T08:30:17.801Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.6770175263157895, "accuracy_std_dev": 0.07095745902363905, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.02569281263157895, "avg_duration_per_task_ms": 247756.84421052632, "avg_output_tokens_per_task": 6891.394736842105, "last_run_timestamp": "2026-08-22T16:09:51.985Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.6768421052631579, "accuracy_std_dev": 0.09630283912571388, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.031000393684210526, "avg_duration_per_task_ms": 226486.0452631579, "avg_output_tokens_per_task": 8460.411578947369, "last_run_timestamp": "2026-08-29T17:02:01.052Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.675675972972973, "accuracy_std_dev": 0.02284471405428887, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.04042408108108108, "avg_duration_per_task_ms": 518660.3783783784, "avg_output_tokens_per_task": 20990.527027027027, "last_run_timestamp": "2026-09-26T08:11:26.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.6751515454545455, "accuracy_std_dev": 0.21232880724838227, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.044593925454545456, "avg_duration_per_task_ms": 85905.94363636363, "avg_output_tokens_per_task": 3735.2381818181816, "last_run_timestamp": "2026-06-30T19:07:40.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.6742424671717172, "accuracy_std_dev": 0.0476536543875113, "run_count": 16, "total_questions": 792, "avg_cost_per_task_usd": 0.017849546717171715, "avg_duration_per_task_ms": 164206.20707070708, "avg_output_tokens_per_task": 12386.181818181818, "last_run_timestamp": "2026-08-29T17:44:03.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.6739257299651568, "accuracy_std_dev": 0.124700233270467, "run_count": 23, "total_questions": 1148, "avg_cost_per_task_usd": 0.04174416724738675, "avg_duration_per_task_ms": 113875.53397212544, "avg_output_tokens_per_task": 5766.850174216028, "last_run_timestamp": "2026-08-29T17:03:33.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.6733836574307305, "accuracy_std_dev": 0.036202263205970635, "run_count": 16, "total_questions": 794, "avg_cost_per_task_usd": 0.011785144836272042, "avg_duration_per_task_ms": 140094.4420654912, "avg_output_tokens_per_task": 7374.278337531486, "last_run_timestamp": "2026-08-30T00:01:09.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14733292, "avg_duration_per_task_ms": 130030.77, "avg_output_tokens_per_task": 5038.48, "last_run_timestamp": "2026-09-23T15:13:09.897Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1116277, "avg_duration_per_task_ms": 79582.44, "avg_output_tokens_per_task": 4280.42, "last_run_timestamp": "2026-08-01T08:05:57.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.039595319999999996, "avg_duration_per_task_ms": 304692.26, "avg_output_tokens_per_task": 31675.6, "last_run_timestamp": "2026-09-24T09:00:04.937Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.6731745714285714, "accuracy_std_dev": 0.04441834647612931, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.07005776761904764, "avg_duration_per_task_ms": 72619.98952380952, "avg_output_tokens_per_task": 4610.935238095238, "last_run_timestamp": "2026-08-01T17:38:46.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.6700102376543211, "accuracy_std_dev": 0.10496553242878909, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.027610222222222223, "avg_duration_per_task_ms": 162024.1913580247, "avg_output_tokens_per_task": 7429.558641975309, "last_run_timestamp": "2026-08-29T16:15:12.662Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.67, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03314721, "avg_duration_per_task_ms": 151834.99, "avg_output_tokens_per_task": 17524.36, "last_run_timestamp": "2026-09-01T17:29:39.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.67, "accuracy_std_dev": 0.031446692107437925, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07852493499999999, "avg_duration_per_task_ms": 68196.455, "avg_output_tokens_per_task": 3640.76, "last_run_timestamp": "2026-09-19T08:12:48.272Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.67, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014700359999999999, "avg_duration_per_task_ms": 1773933.08, "avg_output_tokens_per_task": 23989.78, "last_run_timestamp": "2026-07-18T08:04:08.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.6683333333333333, "accuracy_std_dev": 0.06147128206370479, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07486647166666667, "avg_duration_per_task_ms": 623524.1416666667, "avg_output_tokens_per_task": 16533.455, "last_run_timestamp": "2026-09-26T08:15:33.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6681159130434783, "accuracy_std_dev": 0.032598105721055856, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.12360180173913042, "avg_duration_per_task_ms": 117551.10695652173, "avg_output_tokens_per_task": 12432.393913043477, "last_run_timestamp": "2026-09-26T08:11:26.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Baidu", "accuracy": 0.668, "accuracy_std_dev": 0.056, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.009852277999999999, "avg_duration_per_task_ms": 49533.244, "avg_output_tokens_per_task": 3541.168, "last_run_timestamp": "2026-06-06T17:11:08.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.6677193421052631, "accuracy_std_dev": 0.061111115868114546, "run_count": 38, "total_questions": 1900, "avg_cost_per_task_usd": 0.024342005789473677, "avg_duration_per_task_ms": 113629.73684210527, "avg_output_tokens_per_task": 7236.974736842105, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.6675946726057906, "accuracy_std_dev": 0.062223605171164056, "run_count": 18, "total_questions": 898, "avg_cost_per_task_usd": 0.02003099443207127, "avg_duration_per_task_ms": 278955.9331848552, "avg_output_tokens_per_task": 10682.610244988864, "last_run_timestamp": "2026-08-29T16:44:07.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.6674998750000001, "accuracy_std_dev": 0.034710984293295034, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11729118250000001, "avg_duration_per_task_ms": 121864.08, "avg_output_tokens_per_task": 12614.4, "last_run_timestamp": "2026-09-26T08:10:43.582Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.667439403400309, "accuracy_std_dev": 0.1898774300860501, "run_count": 13, "total_questions": 647, "avg_cost_per_task_usd": 0.0928107310664606, "avg_duration_per_task_ms": 257455.93199381762, "avg_output_tokens_per_task": 7895.010819165379, "last_run_timestamp": "2026-09-26T08:24:45.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.6672268756302522, "accuracy_std_dev": 0.19692376950802565, "run_count": 12, "total_questions": 595, "avg_cost_per_task_usd": 0.04049237815126051, "avg_duration_per_task_ms": 57680.32773109244, "avg_output_tokens_per_task": 4304.621848739495, "last_run_timestamp": "2026-08-04T03:12:18.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.6666666774193548, "accuracy_std_dev": 0.059532208685657846, "run_count": 31, "total_questions": 1550, "avg_cost_per_task_usd": 0.019698243870967743, "avg_duration_per_task_ms": 105867.46322580645, "avg_output_tokens_per_task": 6375.503870967742, "last_run_timestamp": "2026-08-01T17:38:12.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.6666665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11778575999999999, "avg_duration_per_task_ms": 148387.42, "avg_output_tokens_per_task": 58241.15, "last_run_timestamp": "2026-09-01T17:32:23.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.6655251643835617, "accuracy_std_dev": 0.050919510401766024, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.0655772808219178, "avg_duration_per_task_ms": 50910.90410958904, "avg_output_tokens_per_task": 3110.958904109589, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.6648484545454546, "accuracy_std_dev": 0.03613567256846351, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.015155524545454546, "avg_duration_per_task_ms": 85746.93454545454, "avg_output_tokens_per_task": 5712.461818181818, "last_run_timestamp": "2026-08-01T17:38:10.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.6633333333333333, "accuracy_std_dev": 0.016555205069369833, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01703684, "avg_duration_per_task_ms": 144424.58666666667, "avg_output_tokens_per_task": 21218.593333333334, "last_run_timestamp": "2026-09-26T08:04:53.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.6631578421052632, "accuracy_std_dev": 0.03436091497198395, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.016520328421052635, "avg_duration_per_task_ms": 162137.86210526316, "avg_output_tokens_per_task": 6763.676842105263, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6622221666666667, "accuracy_std_dev": 0.0198761101108564, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.31710583, "avg_duration_per_task_ms": 192864.10333333333, "avg_output_tokens_per_task": 8814.376666666667, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.6611110833333333, "accuracy_std_dev": 0.01760328077120064, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03028098166666666, "avg_duration_per_task_ms": 310414.11, "avg_output_tokens_per_task": 9537.056666666667, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00819156, "avg_duration_per_task_ms": 91056.4, "avg_output_tokens_per_task": 8317.36, "last_run_timestamp": "2026-09-23T06:00:08.542Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.658750125, "accuracy_std_dev": 0.021339771978851493, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.113134535, "avg_duration_per_task_ms": 123266.8825, "avg_output_tokens_per_task": 12095.085, "last_run_timestamp": "2026-09-26T08:08:19.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Phala", "accuracy": 0.6575, "accuracy_std_dev": 0.028173569173961615, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.00993166625, "avg_duration_per_task_ms": 80526.21125, "avg_output_tokens_per_task": 4264.58, "last_run_timestamp": "2026-06-20T21:23:09.799Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.6573334, "accuracy_std_dev": 0.2126070154586626, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.04436226000000001, "avg_duration_per_task_ms": 85213.446, "avg_output_tokens_per_task": 3863.376, "last_run_timestamp": "2026-08-04T03:12:10.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.6571529281183933, "accuracy_std_dev": 0.05360818932369472, "run_count": 19, "total_questions": 946, "avg_cost_per_task_usd": 0.02660617758985201, "avg_duration_per_task_ms": 237280.09619450316, "avg_output_tokens_per_task": 13635.822410147992, "last_run_timestamp": "2026-08-29T16:50:48.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6565315, "accuracy_std_dev": 0.011837568509716087, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.014823912162162164, "avg_duration_per_task_ms": 271525.86486486485, "avg_output_tokens_per_task": 19985.412162162163, "last_run_timestamp": "2026-09-19T08:01:36.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6555556666666666, "accuracy_std_dev": 0.03189478845969805, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023210153333333334, "avg_duration_per_task_ms": 207735.58, "avg_output_tokens_per_task": 33693.153333333335, "last_run_timestamp": "2026-09-26T08:25:43.443Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.655102163265306, "accuracy_std_dev": 0.015608943341559031, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.012813330612244898, "avg_duration_per_task_ms": 278688.9183673469, "avg_output_tokens_per_task": 16201.359183673469, "last_run_timestamp": "2026-08-29T16:33:39.228Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.6549999166666667, "accuracy_std_dev": 0.02926570645242861, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.12179273749999998, "avg_duration_per_task_ms": 120187.11666666667, "avg_output_tokens_per_task": 12516.864166666666, "last_run_timestamp": "2026-09-26T08:13:57.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Inceptron", "accuracy": 0.6546666, "accuracy_std_dev": 0.14424683227800877, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.041911724, "avg_duration_per_task_ms": 194571.3, "avg_output_tokens_per_task": 4511.004, "last_run_timestamp": "2026-06-30T19:22:10.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.6536110833333333, "accuracy_std_dev": 0.045804700989560625, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.016743790000000005, "avg_duration_per_task_ms": 95651.83166666667, "avg_output_tokens_per_task": 6607.111666666667, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.6534847727272728, "accuracy_std_dev": 0.034822512390343396, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.014245854545454545, "avg_duration_per_task_ms": 157291.03181818183, "avg_output_tokens_per_task": 6496.27, "last_run_timestamp": "2026-08-15T14:15:19.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.6533335, "accuracy_std_dev": 0.053333500000000034, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.22863909999999998, "avg_duration_per_task_ms": 717866.38, "avg_output_tokens_per_task": 11422.01, "last_run_timestamp": "2026-09-26T08:13:25.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00923216, "avg_duration_per_task_ms": 145753.66, "avg_output_tokens_per_task": 8192.38, "last_run_timestamp": "2026-09-23T06:00:08.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.65226668, "accuracy_std_dev": 0.04159829731584696, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.0068952256000000016, "avg_duration_per_task_ms": 200888.324, "avg_output_tokens_per_task": 7369.9624, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.65222225, "accuracy_std_dev": 0.05094171990802333, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.019855066666666674, "avg_duration_per_task_ms": 138088.43583333332, "avg_output_tokens_per_task": 6770.496666666667, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.6518518333333333, "accuracy_std_dev": 0.03388838939199286, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.026386046666666666, "avg_duration_per_task_ms": 174668.08444444445, "avg_output_tokens_per_task": 7625.071111111111, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.65166675, "accuracy_std_dev": 0.04506177549972369, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04474220500000001, "avg_duration_per_task_ms": 212693.005, "avg_output_tokens_per_task": 3236.785, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.651111, "accuracy_std_dev": 0.09803010371649448, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.010906386666666667, "avg_duration_per_task_ms": 303175.14, "avg_output_tokens_per_task": 4777.48, "last_run_timestamp": "2026-09-26T08:13:57.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.6485714285714286, "accuracy_std_dev": 0.061434100526748275, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.01923216428571428, "avg_duration_per_task_ms": 162362.36428571428, "avg_output_tokens_per_task": 6530.884285714285, "last_run_timestamp": "2026-08-29T16:37:21.285Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.6481482020202021, "accuracy_std_dev": 0.0054059166891471805, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.009114631313131312, "avg_duration_per_task_ms": 604721.6666666666, "avg_output_tokens_per_task": 19746.29292929293, "last_run_timestamp": "2026-08-22T16:10:30.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6476509932885907, "accuracy_std_dev": 0.013206828848070314, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.018146744966442953, "avg_duration_per_task_ms": 194939.0268456376, "avg_output_tokens_per_task": 22002.395973154362, "last_run_timestamp": "2026-09-26T08:13:57.279Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.6466667083333333, "accuracy_std_dev": 0.04311917808284303, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.016928467500000002, "avg_duration_per_task_ms": 184800.4625, "avg_output_tokens_per_task": 7229.21, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.6466666818181818, "accuracy_std_dev": 0.11090082019181345, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.03871445727272728, "avg_duration_per_task_ms": 229660.09454545454, "avg_output_tokens_per_task": 6490.092727272728, "last_run_timestamp": "2026-08-29T16:11:23.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SambaNova", "accuracy": 0.6466666666666666, "accuracy_std_dev": 0.053748384988656986, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.027454210000000003, "avg_duration_per_task_ms": 53778.18666666667, "avg_output_tokens_per_task": 3634.7233333333334, "last_run_timestamp": "2026-05-16T16:36:37.985Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6460429376257546, "accuracy_std_dev": 0.1396968497913965, "run_count": 20, "total_questions": 994, "avg_cost_per_task_usd": 0.013013100603621729, "avg_duration_per_task_ms": 399798.5160965795, "avg_output_tokens_per_task": 11031.460764587526, "last_run_timestamp": "2026-08-29T16:48:06.159Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.645833375, "accuracy_std_dev": 0.02526261098905604, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.11683335, "avg_duration_per_task_ms": 147559.335, "avg_output_tokens_per_task": 12607.1725, "last_run_timestamp": "2026-09-26T08:22:25.227Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.6456595058430717, "accuracy_std_dev": 0.16063234813378102, "run_count": 24, "total_questions": 1198, "avg_cost_per_task_usd": 0.04617719449081804, "avg_duration_per_task_ms": 194681.6978297162, "avg_output_tokens_per_task": 5492.883138564273, "last_run_timestamp": "2026-08-29T17:36:31.188Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.6440944614173228, "accuracy_std_dev": 0.07748571441835636, "run_count": 13, "total_questions": 635, "avg_cost_per_task_usd": 0.020678966929133857, "avg_duration_per_task_ms": 227010.8503937008, "avg_output_tokens_per_task": 22675.00472440945, "last_run_timestamp": "2026-09-26T16:00:24.609Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.6433335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.034682500000000005, "avg_duration_per_task_ms": 159195.12, "avg_output_tokens_per_task": 18151.16, "last_run_timestamp": "2026-09-01T17:31:43.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.6433333333333333, "accuracy_std_dev": 0.14455548467015086, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10531282, "avg_duration_per_task_ms": 1453976.54, "avg_output_tokens_per_task": 18878.353333333333, "last_run_timestamp": "2026-08-08T08:21:31.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "WandB", "accuracy": 0.6433333125, "accuracy_std_dev": 0.0463680879845163, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.03536953125, "avg_duration_per_task_ms": 94585.0525, "avg_output_tokens_per_task": 4734.50875, "last_run_timestamp": "2026-07-04T16:00:23.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.6431451290322581, "accuracy_std_dev": 0.07480941255490514, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.028669461693548386, "avg_duration_per_task_ms": 341904.4637096774, "avg_output_tokens_per_task": 10131.977822580646, "last_run_timestamp": "2026-08-08T16:03:43.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.6427272272727272, "accuracy_std_dev": 0.0360504277719316, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.014966787272727274, "avg_duration_per_task_ms": 83301.79454545454, "avg_output_tokens_per_task": 5789.916363636364, "last_run_timestamp": "2026-08-01T17:38:10.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.64166675, "accuracy_std_dev": 0.00985994985725082, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0329834, "avg_duration_per_task_ms": 238262.615, "avg_output_tokens_per_task": 16910.1, "last_run_timestamp": "2026-09-01T17:31:47.440Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.64, "accuracy_std_dev": 0.025647213408087847, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.218415728, "avg_duration_per_task_ms": 1219489.072, "avg_output_tokens_per_task": 31443.888, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AkashML", "accuracy": 0.6395555333333334, "accuracy_std_dev": 0.08663140524841761, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.018671574666666666, "avg_duration_per_task_ms": 106669.63333333333, "avg_output_tokens_per_task": 3706.6453333333334, "last_run_timestamp": "2026-07-01T02:04:30.298Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DeepInfra", "accuracy": 0.6392073065902578, "accuracy_std_dev": 0.050170485537427534, "run_count": 14, "total_questions": 698, "avg_cost_per_task_usd": 0.011382749283667623, "avg_duration_per_task_ms": 156016.08022922635, "avg_output_tokens_per_task": 5066.042979942694, "last_run_timestamp": "2026-07-11T15:34:15.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.63875, "accuracy_std_dev": 0.031021421810178213, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.016371615000000003, "avg_duration_per_task_ms": 231785.61, "avg_output_tokens_per_task": 12424.35125, "last_run_timestamp": "2026-07-11T15:34:14.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.6383647665094339, "accuracy_std_dev": 0.026715950245675562, "run_count": 17, "total_questions": 848, "avg_cost_per_task_usd": 0.011237323113207546, "avg_duration_per_task_ms": 166991.90094339623, "avg_output_tokens_per_task": 10001.11320754717, "last_run_timestamp": "2026-07-18T16:00:10.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.6382114250871079, "accuracy_std_dev": 0.03539796279192529, "run_count": 23, "total_questions": 1148, "avg_cost_per_task_usd": 0.013505973867595823, "avg_duration_per_task_ms": 198911.70383275262, "avg_output_tokens_per_task": 13526.754355400697, "last_run_timestamp": "2026-08-29T16:25:26.695Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.637681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.04568002173913044, "avg_duration_per_task_ms": 70889.17391304347, "avg_output_tokens_per_task": 8168.934782608696, "last_run_timestamp": "2026-08-01T17:41:39.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.6372548823529411, "accuracy_std_dev": 0.058613911349040856, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.028036784705882355, "avg_duration_per_task_ms": 144149.16352941177, "avg_output_tokens_per_task": 4911.451764705882, "last_run_timestamp": "2026-07-18T16:00:10.793Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.6365, "accuracy_std_dev": 0.07836859560946591, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.06096936600000001, "avg_duration_per_task_ms": 147673.036, "avg_output_tokens_per_task": 6219.183, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.6360152471264368, "accuracy_std_dev": 0.13554242758288354, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.04763444827586207, "avg_duration_per_task_ms": 2222096.040229885, "avg_output_tokens_per_task": 71368.38505747127, "last_run_timestamp": "2026-09-26T08:08:19.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.6352777083333333, "accuracy_std_dev": 0.04398987634338833, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.019619210000000005, "avg_duration_per_task_ms": 153720.92, "avg_output_tokens_per_task": 6503.970833333334, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.6337365483870968, "accuracy_std_dev": 0.02579029640950258, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.016857699596774196, "avg_duration_per_task_ms": 45111.594758064515, "avg_output_tokens_per_task": 9635.451612903225, "last_run_timestamp": "2026-08-29T16:33:38.021Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6333332500000001, "accuracy_std_dev": 0.03299835012826401, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.102073965, "avg_duration_per_task_ms": 160807.645, "avg_output_tokens_per_task": 12456.285, "last_run_timestamp": "2026-09-01T17:31:46.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6331093959731544, "accuracy_std_dev": 0.01453205911080741, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.012315053691275168, "avg_duration_per_task_ms": 1064929.322147651, "avg_output_tokens_per_task": 21067.724832214764, "last_run_timestamp": "2026-08-29T17:18:10.895Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6319998, "accuracy_std_dev": 0.0363685871784979, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.09467958400000001, "avg_duration_per_task_ms": 55836.916, "avg_output_tokens_per_task": 3550.436, "last_run_timestamp": "2026-08-29T16:15:31.797Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.6303740787269682, "accuracy_std_dev": 0.08664725240494583, "run_count": 24, "total_questions": 1194, "avg_cost_per_task_usd": 0.05638804941373535, "avg_duration_per_task_ms": 323905.2269681742, "avg_output_tokens_per_task": 9046.836683417085, "last_run_timestamp": "2026-08-30T00:01:02.157Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.63, "accuracy_std_dev": 0.023333000000000048, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07672618, "avg_duration_per_task_ms": 231628.06, "avg_output_tokens_per_task": 10580.58, "last_run_timestamp": "2026-08-01T17:40:48.165Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.63, "accuracy_std_dev": 0.050000000000000044, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09203941, "avg_duration_per_task_ms": 133131.18, "avg_output_tokens_per_task": 12511.49, "last_run_timestamp": "2026-09-01T17:23:15.998Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.6280864185185184, "accuracy_std_dev": 0.08641467998795974, "run_count": 11, "total_questions": 540, "avg_cost_per_task_usd": 0.01978988888888889, "avg_duration_per_task_ms": 730210.6111111111, "avg_output_tokens_per_task": 25514.716666666667, "last_run_timestamp": "2026-09-26T16:00:20.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.6280215229007634, "accuracy_std_dev": 0.052631933879496495, "run_count": 21, "total_questions": 1048, "avg_cost_per_task_usd": 0.03512998282442747, "avg_duration_per_task_ms": 195080.4961832061, "avg_output_tokens_per_task": 6126.911259541985, "last_run_timestamp": "2026-08-22T16:05:40.004Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.6278446465863453, "accuracy_std_dev": 0.02683498380174657, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.03220683132530121, "avg_duration_per_task_ms": 87364.718875502, "avg_output_tokens_per_task": 22117.32730923695, "last_run_timestamp": "2026-08-29T17:17:57.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6258440128913444, "accuracy_std_dev": 0.22029611561511347, "run_count": 11, "total_questions": 543, "avg_cost_per_task_usd": 0.020638675874769797, "avg_duration_per_task_ms": 355913.0128913444, "avg_output_tokens_per_task": 11160.08655616943, "last_run_timestamp": "2026-09-26T08:14:40.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.6234782173913044, "accuracy_std_dev": 0.04417228647797023, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.008193006086956521, "avg_duration_per_task_ms": 187011.97043478262, "avg_output_tokens_per_task": 11024.112173913043, "last_run_timestamp": "2026-08-29T16:33:38.124Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "AtlasCloud", "accuracy": 0.6233333125, "accuracy_std_dev": 0.051153368250192906, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.05906399125, "avg_duration_per_task_ms": 293035.57125, "avg_output_tokens_per_task": 14125.07375, "last_run_timestamp": "2026-07-01T03:04:05.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.6222934707297514, "accuracy_std_dev": 0.051089021576005936, "run_count": 25, "total_questions": 1247, "avg_cost_per_task_usd": 0.007040149959903768, "avg_duration_per_task_ms": 147687.18203688852, "avg_output_tokens_per_task": 7570.056134723336, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.6220456222493888, "accuracy_std_dev": 0.040780983283176246, "run_count": 33, "total_questions": 1636, "avg_cost_per_task_usd": 0.014479357579462098, "avg_duration_per_task_ms": 232578.68643031784, "avg_output_tokens_per_task": 14319.570293398532, "last_run_timestamp": "2026-08-29T16:49:23.763Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6214190008032129, "accuracy_std_dev": 0.13040949799473348, "run_count": 25, "total_questions": 1245, "avg_cost_per_task_usd": 0.012814858634538153, "avg_duration_per_task_ms": 167474.9903614458, "avg_output_tokens_per_task": 5278.168674698795, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6207692692307692, "accuracy_std_dev": 0.15044459474544053, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.029133950769230768, "avg_duration_per_task_ms": 194019.7423076923, "avg_output_tokens_per_task": 5212.69076923077, "last_run_timestamp": "2026-08-30T00:01:10.585Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.6204417565485362, "accuracy_std_dev": 0.06516705835591563, "run_count": 26, "total_questions": 1298, "avg_cost_per_task_usd": 0.00849505469953775, "avg_duration_per_task_ms": 165342.74576271186, "avg_output_tokens_per_task": 8391.972265023112, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.620246288590604, "accuracy_std_dev": 0.024043751131473293, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.19689856375838927, "avg_duration_per_task_ms": 175265.28187919463, "avg_output_tokens_per_task": 13638.724832214764, "last_run_timestamp": "2026-09-01T13:00:41.540Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Ambient", "accuracy": 0.62, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01353782, "avg_duration_per_task_ms": 89487.42, "avg_output_tokens_per_task": 4959.24, "last_run_timestamp": "2026-06-13T15:23:43.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.6176750230547552, "accuracy_std_dev": 0.22454644005875757, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.04794541498559077, "avg_duration_per_task_ms": 857866.4149855907, "avg_output_tokens_per_task": 11712.968299711816, "last_run_timestamp": "2026-08-29T17:09:01.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6159998000000001, "accuracy_std_dev": 0.03440923906976152, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045486667999999994, "avg_duration_per_task_ms": 195084.972, "avg_output_tokens_per_task": 3358.316, "last_run_timestamp": "2026-09-19T08:08:42.956Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.6148485, "accuracy_std_dev": 0.04287317952429849, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.05723192454545456, "avg_duration_per_task_ms": 395544.69636363635, "avg_output_tokens_per_task": 20268.843636363636, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.613394855704698, "accuracy_std_dev": 0.05588312171671833, "run_count": 24, "total_questions": 1192, "avg_cost_per_task_usd": 0.02671478104026846, "avg_duration_per_task_ms": 57602.19043624161, "avg_output_tokens_per_task": 15352.657718120805, "last_run_timestamp": "2026-08-29T17:17:47.080Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.6133335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.22014136, "avg_duration_per_task_ms": 192160.83, "avg_output_tokens_per_task": 15089.79, "last_run_timestamp": "2026-09-01T12:57:05.201Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.6120478419388831, "accuracy_std_dev": 0.0713164995507085, "run_count": 19, "total_questions": 949, "avg_cost_per_task_usd": 0.019381050579557434, "avg_duration_per_task_ms": 108067.79030558483, "avg_output_tokens_per_task": 9183.234984193889, "last_run_timestamp": "2026-08-01T17:40:52.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.612021883424408, "accuracy_std_dev": 0.055254347051473786, "run_count": 22, "total_questions": 1098, "avg_cost_per_task_usd": 0.09755443624772316, "avg_duration_per_task_ms": 391161.3069216758, "avg_output_tokens_per_task": 18802.257741347905, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.6113782756410256, "accuracy_std_dev": 0.036669727145404285, "run_count": 25, "total_questions": 1248, "avg_cost_per_task_usd": 0.06140813701923077, "avg_duration_per_task_ms": 418606.3149038461, "avg_output_tokens_per_task": 21705.047275641027, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.608, "accuracy_std_dev": 0.25443270230062803, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.005877256, "avg_duration_per_task_ms": 51102.604, "avg_output_tokens_per_task": 3478.088, "last_run_timestamp": "2026-06-30T19:12:02.101Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6077776666666667, "accuracy_std_dev": 0.08590813209793095, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01715707, "avg_duration_per_task_ms": 128260.91, "avg_output_tokens_per_task": 13292.39, "last_run_timestamp": "2026-08-29T16:48:14.218Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.6066666666666667, "accuracy_std_dev": 0.15241273398753646, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017056396666666668, "avg_duration_per_task_ms": 233868.72666666665, "avg_output_tokens_per_task": 8308.71, "last_run_timestamp": "2026-08-29T16:33:39.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.604, "accuracy_std_dev": 0.19783738552457672, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.131018624, "avg_duration_per_task_ms": 356662.58, "avg_output_tokens_per_task": 7644.116, "last_run_timestamp": "2026-09-26T08:24:48.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.603913, "accuracy_std_dev": 0.03467796953586577, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.039784393043478264, "avg_duration_per_task_ms": 20972.738260869566, "avg_output_tokens_per_task": 12506.93304347826, "last_run_timestamp": "2026-08-29T16:55:05.300Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6033335000000001, "accuracy_std_dev": 0.028867224784346696, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04717542499999999, "avg_duration_per_task_ms": 204909.125, "avg_output_tokens_per_task": 3753.575, "last_run_timestamp": "2026-09-19T08:01:34.089Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.6004564109589041, "accuracy_std_dev": 0.09338877055873593, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.485756647260274, "avg_duration_per_task_ms": 2654805.914383562, "avg_output_tokens_per_task": 83086.6404109589, "last_run_timestamp": "2026-09-26T08:17:02.590Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "AtlasCloud", "accuracy": 0.5991111333333333, "accuracy_std_dev": 0.04144548035651441, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.02262494266666667, "avg_duration_per_task_ms": 44422.645333333334, "avg_output_tokens_per_task": 2008.0613333333333, "last_run_timestamp": "2026-07-01T01:59:48.107Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.5955284268292683, "accuracy_std_dev": 0.10689581341902113, "run_count": 23, "total_questions": 1148, "avg_cost_per_task_usd": 0.07586419512195122, "avg_duration_per_task_ms": 273117.974738676, "avg_output_tokens_per_task": 6308.23606271777, "last_run_timestamp": "2026-08-29T16:33:38.223Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.595, "accuracy_std_dev": 0.08500000000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.15232006, "avg_duration_per_task_ms": 976036.98, "avg_output_tokens_per_task": 119494.16, "last_run_timestamp": "2026-09-26T08:22:20.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.5933335000000001, "accuracy_std_dev": 0.05333349999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08864457999999999, "avg_duration_per_task_ms": 125444.95, "avg_output_tokens_per_task": 11722.21, "last_run_timestamp": "2026-09-01T17:31:29.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "Novita", "accuracy": 0.5933333333333334, "accuracy_std_dev": 0.02748737083745107, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00537365, "avg_duration_per_task_ms": 42524.18666666667, "avg_output_tokens_per_task": 1858.8366666666666, "last_run_timestamp": "2026-05-23T08:43:48.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.593333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04552234, "avg_duration_per_task_ms": 32685.39, "avg_output_tokens_per_task": 3125.98, "last_run_timestamp": "2026-09-26T08:22:32.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Phala", "accuracy": 0.5911110833333333, "accuracy_std_dev": 0.06849353668711902, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03914455666666666, "avg_duration_per_task_ms": 242677.80666666667, "avg_output_tokens_per_task": 10777.653333333334, "last_run_timestamp": "2026-07-04T16:00:25.388Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.5893471237113401, "accuracy_std_dev": 0.30515532350212654, "run_count": 4, "total_questions": 194, "avg_cost_per_task_usd": 0.02748909793814433, "avg_duration_per_task_ms": 98623.75773195876, "avg_output_tokens_per_task": 4244.2525773195875, "last_run_timestamp": "2026-08-15T14:23:29.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Io Net", "accuracy": 0.588, "accuracy_std_dev": 0.027129319932501058, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.013733939999999998, "avg_duration_per_task_ms": 254046.672, "avg_output_tokens_per_task": 9671.628, "last_run_timestamp": "2026-05-09T16:36:33.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.586667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15373889999999998, "avg_duration_per_task_ms": 5757447.74, "avg_output_tokens_per_task": 125764.32, "last_run_timestamp": "2026-09-23T03:50:04.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.5866667142857143, "accuracy_std_dev": 0.06443891163999389, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.058708722857142855, "avg_duration_per_task_ms": 207994.24285714285, "avg_output_tokens_per_task": 9282.182857142858, "last_run_timestamp": "2026-08-15T14:16:11.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.5866666666666667, "accuracy_std_dev": 0.0066666666750000055, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029609283333333326, "avg_duration_per_task_ms": 75747.02, "avg_output_tokens_per_task": 8199.643333333333, "last_run_timestamp": "2026-09-01T17:32:36.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.5816638697829716, "accuracy_std_dev": 0.09157507085172394, "run_count": 24, "total_questions": 1198, "avg_cost_per_task_usd": 0.013990112687813019, "avg_duration_per_task_ms": 68737.19449081803, "avg_output_tokens_per_task": 2743.449916527546, "last_run_timestamp": "2026-08-29T16:08:08.875Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.5811856778350516, "accuracy_std_dev": 0.03980730883718937, "run_count": 8, "total_questions": 388, "avg_cost_per_task_usd": 0.07594913402061856, "avg_duration_per_task_ms": 92087.82731958762, "avg_output_tokens_per_task": 5709.461340206186, "last_run_timestamp": "2026-08-29T16:11:10.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.58, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10846708, "avg_duration_per_task_ms": 79571.2, "avg_output_tokens_per_task": 4199.82, "last_run_timestamp": "2026-08-01T08:05:35.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.5783332916666667, "accuracy_std_dev": 0.06758403856340586, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.02616774333333334, "avg_duration_per_task_ms": 66057.43916666666, "avg_output_tokens_per_task": 2955.8108333333334, "last_run_timestamp": "2026-08-29T16:43:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.5766665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.028599199999999998, "avg_duration_per_task_ms": 78841.33, "avg_output_tokens_per_task": 8172.09, "last_run_timestamp": "2026-09-01T17:31:45.195Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "AtlasCloud", "accuracy": 0.575686294117647, "accuracy_std_dev": 0.14503435534089426, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.009561005882352942, "avg_duration_per_task_ms": 71150.81176470588, "avg_output_tokens_per_task": 3692.6835294117645, "last_run_timestamp": "2026-07-01T02:00:24.123Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.5747916875, "accuracy_std_dev": 0.09687443552720627, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.033850981249999995, "avg_duration_per_task_ms": 62474.61875, "avg_output_tokens_per_task": 3318.425, "last_run_timestamp": "2026-08-29T16:44:17.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.573333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012256119999999999, "avg_duration_per_task_ms": 120534.1, "avg_output_tokens_per_task": 16750, "last_run_timestamp": "2026-08-07T00:00:02.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.5712366451612902, "accuracy_std_dev": 0.0858999966086146, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.038580427419354836, "avg_duration_per_task_ms": 2634832.3427419355, "avg_output_tokens_per_task": 80935.72983870968, "last_run_timestamp": "2026-08-29T17:24:25.648Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "AtlasCloud", "accuracy": 0.5706666666666667, "accuracy_std_dev": 0.07187643718369895, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.059964562666666665, "avg_duration_per_task_ms": 452906.82933333336, "avg_output_tokens_per_task": 11189.822666666667, "last_run_timestamp": "2026-06-20T22:23:15.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Together", "accuracy": 0.57, "accuracy_std_dev": 0.03316624790355398, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04864015, "avg_duration_per_task_ms": 28286.98, "avg_output_tokens_per_task": 1797.17, "last_run_timestamp": "2026-05-09T08:27:19.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.569727612244898, "accuracy_std_dev": 0.026944500000000038, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.0024336428571428573, "avg_duration_per_task_ms": 317617.1530612245, "avg_output_tokens_per_task": 5820.755102040816, "last_run_timestamp": "2026-08-29T16:57:41.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5685557531992688, "accuracy_std_dev": 0.05835119458074531, "run_count": 22, "total_questions": 1094, "avg_cost_per_task_usd": 0.092415978976234, "avg_duration_per_task_ms": 695361.1864716636, "avg_output_tokens_per_task": 15899.566727605119, "last_run_timestamp": "2026-08-29T16:31:42.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.5653333, "accuracy_std_dev": 0.04655220391034133, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.12865686099999998, "avg_duration_per_task_ms": 196202.004, "avg_output_tokens_per_task": 3859.015, "last_run_timestamp": "2026-08-01T17:38:24.442Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.5648772953020134, "accuracy_std_dev": 0.0424873867714486, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1486826308724832, "avg_duration_per_task_ms": 18002.88590604027, "avg_output_tokens_per_task": 2546.3288590604025, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.5633528237259816, "accuracy_std_dev": 0.139614019301722, "run_count": 24, "total_questions": 1197, "avg_cost_per_task_usd": 0.024817531328320798, "avg_duration_per_task_ms": 144439.48621553884, "avg_output_tokens_per_task": 7778.0726817042605, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5604491347150259, "accuracy_std_dev": 0.10678459157041047, "run_count": 8, "total_questions": 386, "avg_cost_per_task_usd": 0.07120025388601035, "avg_duration_per_task_ms": 510444.22279792745, "avg_output_tokens_per_task": 29300.067357512955, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.5590908181818182, "accuracy_std_dev": 0.11381447181493397, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.030268118181818182, "avg_duration_per_task_ms": 253612.38363636364, "avg_output_tokens_per_task": 13343.18909090909, "last_run_timestamp": "2026-07-18T16:00:09.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "SiliconFlow", "accuracy": 0.556, "accuracy_std_dev": 0.04963869458396342, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.032140744, "avg_duration_per_task_ms": 123464.168, "avg_output_tokens_per_task": 1432.936, "last_run_timestamp": "2026-05-09T11:38:50.156Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.5544422404809619, "accuracy_std_dev": 0.0556916211755413, "run_count": 20, "total_questions": 998, "avg_cost_per_task_usd": 0.1281926412825651, "avg_duration_per_task_ms": 173153.67835671344, "avg_output_tokens_per_task": 3644.451903807615, "last_run_timestamp": "2026-08-01T17:38:23.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.5533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03619409, "avg_duration_per_task_ms": 60452.63, "avg_output_tokens_per_task": 6631.82, "last_run_timestamp": "2026-09-01T17:31:47.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.5500352600422833, "accuracy_std_dev": 0.058938257867956145, "run_count": 19, "total_questions": 946, "avg_cost_per_task_usd": 0.0899786046511628, "avg_duration_per_task_ms": 885342.9714587738, "avg_output_tokens_per_task": 18576.52959830867, "last_run_timestamp": "2026-08-29T16:25:50.849Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.55, "accuracy_std_dev": 0.036667000000000005, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23514412, "avg_duration_per_task_ms": 166142.51, "avg_output_tokens_per_task": 13675.21, "last_run_timestamp": "2026-09-01T13:01:57.044Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.5405240859232175, "accuracy_std_dev": 0.0807969531074554, "run_count": 22, "total_questions": 1094, "avg_cost_per_task_usd": 0.08495231718464354, "avg_duration_per_task_ms": 201381.6352833638, "avg_output_tokens_per_task": 33884.773308957956, "last_run_timestamp": "2026-08-29T17:20:10.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.5342466164383561, "accuracy_std_dev": 0.014081561995113387, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.1389923630136986, "avg_duration_per_task_ms": 68151.21917808219, "avg_output_tokens_per_task": 2403.2671232876714, "last_run_timestamp": "2026-09-01T13:01:05.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Ambient", "accuracy": 0.53, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00720358, "avg_duration_per_task_ms": 62972.84, "avg_output_tokens_per_task": 2706.77, "last_run_timestamp": "2026-05-23T16:34:17.047Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.5213333333333333, "accuracy_std_dev": 0.019352465629876622, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.027455891999999996, "avg_duration_per_task_ms": 341646.812, "avg_output_tokens_per_task": 11097.765333333333, "last_run_timestamp": "2026-08-29T16:44:22.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.5200001111111111, "accuracy_std_dev": 0.02454520646772421, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.02573156, "avg_duration_per_task_ms": 449048.8288888889, "avg_output_tokens_per_task": 52732.037777777776, "last_run_timestamp": "2026-09-26T16:00:18.487Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5151512828282828, "accuracy_std_dev": 0.04224499999999998, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.026153939393939396, "avg_duration_per_task_ms": 213641.58585858587, "avg_output_tokens_per_task": 54161.38383838384, "last_run_timestamp": "2026-09-01T17:31:30.190Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.513371664756447, "accuracy_std_dev": 0.024300137045049023, "run_count": 14, "total_questions": 698, "avg_cost_per_task_usd": 0.05487210888252148, "avg_duration_per_task_ms": 254698.24212034384, "avg_output_tokens_per_task": 3318.022922636103, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.5126262474747475, "accuracy_std_dev": 0.02515381026771839, "run_count": 16, "total_questions": 792, "avg_cost_per_task_usd": 0.023066161616161616, "avg_duration_per_task_ms": 1118619.621212121, "avg_output_tokens_per_task": 98415.15909090909, "last_run_timestamp": "2026-08-29T16:33:29.988Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.5093939090909091, "accuracy_std_dev": 0.26566983901244945, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.022748017272727274, "avg_duration_per_task_ms": 109776.03, "avg_output_tokens_per_task": 2644.2045454545455, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "NextBit", "accuracy": 0.5083333333333333, "accuracy_std_dev": 0.02229848026709944, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.007687528333333332, "avg_duration_per_task_ms": 35548.63166666667, "avg_output_tokens_per_task": 2391.6783333333333, "last_run_timestamp": "2026-06-20T16:41:40.557Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.5083333125, "accuracy_std_dev": 0.0195078848024291, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.031913726249999996, "avg_duration_per_task_ms": 788235.6275, "avg_output_tokens_per_task": 119361.61125, "last_run_timestamp": "2026-07-18T16:00:10.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.5081871543859648, "accuracy_std_dev": 0.018863102843005027, "run_count": 23, "total_questions": 1140, "avg_cost_per_task_usd": 0.037433712280701754, "avg_duration_per_task_ms": 648881.6219298246, "avg_output_tokens_per_task": 187737.9605263158, "last_run_timestamp": "2026-08-29T16:44:16.523Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.5077973684210526, "accuracy_std_dev": 0.01607029428505164, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.021449652046783625, "avg_duration_per_task_ms": 1262349.283625731, "avg_output_tokens_per_task": 119951.4649122807, "last_run_timestamp": "2026-08-29T17:16:15.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.5033669747474748, "accuracy_std_dev": 0.027292446564651823, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.025276146464646466, "avg_duration_per_task_ms": 577739.3712121212, "avg_output_tokens_per_task": 53598.469696969696, "last_run_timestamp": "2026-09-26T16:00:25.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.5022222, "accuracy_std_dev": 0.048422017234862634, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.016079876, "avg_duration_per_task_ms": 34106.436, "avg_output_tokens_per_task": 1702.6026666666667, "last_run_timestamp": "2026-06-30T19:02:38.080Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Ambient", "accuracy": 0.5, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00390972, "avg_duration_per_task_ms": 84786.05, "avg_output_tokens_per_task": 2818.58, "last_run_timestamp": "2026-06-13T17:26:55.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.49866136646586345, "accuracy_std_dev": 0.20252963395096038, "run_count": 20, "total_questions": 996, "avg_cost_per_task_usd": 0.010686857429718876, "avg_duration_per_task_ms": 38588.640562249, "avg_output_tokens_per_task": 1877.5230923694778, "last_run_timestamp": "2026-08-08T16:03:49.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.49830913043478264, "accuracy_std_dev": 0.23149644601826339, "run_count": 14, "total_questions": 690, "avg_cost_per_task_usd": 0.005944484057971015, "avg_duration_per_task_ms": 247630.89855072464, "avg_output_tokens_per_task": 4340.623188405797, "last_run_timestamp": "2026-08-29T16:33:36.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.4979166875, "accuracy_std_dev": 0.05113807646548552, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.112073925, "avg_duration_per_task_ms": 113062.30875, "avg_output_tokens_per_task": 1623.53625, "last_run_timestamp": "2026-07-18T08:04:08.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Novita", "accuracy": 0.49777777777777776, "accuracy_std_dev": 0.023934065809486697, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.015927537777777778, "avg_duration_per_task_ms": 341008.0911111111, "avg_output_tokens_per_task": 17430.457777777778, "last_run_timestamp": "2026-05-23T13:37:43.176Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.49666650000000007, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.13482986, "avg_duration_per_task_ms": 58783.54, "avg_output_tokens_per_task": 2896.5, "last_run_timestamp": "2026-09-01T13:00:51.538Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.49666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23109727999999996, "avg_duration_per_task_ms": 80168.37, "avg_output_tokens_per_task": 2746.03, "last_run_timestamp": "2026-08-01T17:41:27.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.49479164285714283, "accuracy_std_dev": 0.21842675350179366, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.0037404129464285716, "avg_duration_per_task_ms": 123800.87723214286, "avg_output_tokens_per_task": 3922.089285714286, "last_run_timestamp": "2026-07-11T15:34:14.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.49476185714285714, "accuracy_std_dev": 0.035539193018527655, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02792878571428572, "avg_duration_per_task_ms": 578258.1285714286, "avg_output_tokens_per_task": 53943.23142857143, "last_run_timestamp": "2026-09-26T16:00:20.149Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.493333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14235776, "avg_duration_per_task_ms": 443941.74, "avg_output_tokens_per_task": 8833.66, "last_run_timestamp": "2026-08-06T00:00:02.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.49275365217391304, "accuracy_std_dev": 0.05838113659290146, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.042801485217391314, "avg_duration_per_task_ms": 191615.06608695653, "avg_output_tokens_per_task": 2487.1382608695653, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.48666650000000006, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12261337999999998, "avg_duration_per_task_ms": 50813.54, "avg_output_tokens_per_task": 2655.2, "last_run_timestamp": "2026-09-01T12:53:49.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.4864496585365854, "accuracy_std_dev": 0.04830717864914074, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.030671650406504065, "avg_duration_per_task_ms": 1720485.825203252, "avg_output_tokens_per_task": 131204.84552845528, "last_run_timestamp": "2026-08-29T17:10:29.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.4855528467336683, "accuracy_std_dev": 0.22212037609522364, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.011942556532663315, "avg_duration_per_task_ms": 259270.25502512563, "avg_output_tokens_per_task": 6411.776381909548, "last_run_timestamp": "2026-08-29T17:02:26.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Cerebras", "accuracy": 0.485, "accuracy_std_dev": 0.044440972086577935, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05964879999999999, "avg_duration_per_task_ms": 16505.4875, "avg_output_tokens_per_task": 1559.855, "last_run_timestamp": "2026-05-23T08:11:43.884Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.48333325000000005, "accuracy_std_dev": 0.025166081671319035, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021045685, "avg_duration_per_task_ms": 161568.115, "avg_output_tokens_per_task": 3811.995, "last_run_timestamp": "2026-08-29T16:46:44.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.48243642152466365, "accuracy_std_dev": 0.023113682722310937, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.028583977578475337, "avg_duration_per_task_ms": 385123.331838565, "avg_output_tokens_per_task": 66962.30269058296, "last_run_timestamp": "2026-09-26T16:00:20.148Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.48060604545454544, "accuracy_std_dev": 0.05092710756247356, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.022301401818181816, "avg_duration_per_task_ms": 29490.40818181818, "avg_output_tokens_per_task": 2280.981818181818, "last_run_timestamp": "2026-08-29T16:42:36.232Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Ambient", "accuracy": 0.48, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00732523, "avg_duration_per_task_ms": 33542.46, "avg_output_tokens_per_task": 1578.75, "last_run_timestamp": "2026-06-13T08:36:56.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.47916700000000007, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.04362447916666667, "avg_duration_per_task_ms": 66953.94791666667, "avg_output_tokens_per_task": 2828.96875, "last_run_timestamp": "2026-08-15T14:08:14.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.47856448265895946, "accuracy_std_dev": 0.04258833481538036, "run_count": 14, "total_questions": 692, "avg_cost_per_task_usd": 0.09286534826589596, "avg_duration_per_task_ms": 54402.82225433526, "avg_output_tokens_per_task": 3776.9132947976877, "last_run_timestamp": "2026-08-15T14:15:58.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4775, "accuracy_std_dev": 0.03857702972301004, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.0438162675, "avg_duration_per_task_ms": 100713.26625, "avg_output_tokens_per_task": 2946.27125, "last_run_timestamp": "2026-08-29T16:33:38.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.47626414551083596, "accuracy_std_dev": 0.030462766110460025, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.030607054179566565, "avg_duration_per_task_ms": 1315956.8513931888, "avg_output_tokens_per_task": 23348.664086687306, "last_run_timestamp": "2026-08-15T14:12:30.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.47619069387755103, "accuracy_std_dev": 0.04472200000000001, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.042756918367346936, "avg_duration_per_task_ms": 697970.0612244898, "avg_output_tokens_per_task": 99743.38775510204, "last_run_timestamp": "2026-09-01T17:27:02.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.4757525267558528, "accuracy_std_dev": 0.0504992076723975, "run_count": 24, "total_questions": 1196, "avg_cost_per_task_usd": 0.015268672240802679, "avg_duration_per_task_ms": 37530.696488294314, "avg_output_tokens_per_task": 2374.680602006689, "last_run_timestamp": "2026-08-29T16:12:10.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.47535832526881727, "accuracy_std_dev": 0.009377392067212614, "run_count": 15, "total_questions": 744, "avg_cost_per_task_usd": 0.11317686962365592, "avg_duration_per_task_ms": 1748531.8629032257, "avg_output_tokens_per_task": 29728.6061827957, "last_run_timestamp": "2026-08-29T16:20:58.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.4744445, "accuracy_std_dev": 0.03557299398771489, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.023446433333333332, "avg_duration_per_task_ms": 1018917.68, "avg_output_tokens_per_task": 62981.19, "last_run_timestamp": "2026-07-18T16:00:08.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.023663, "avg_duration_per_task_ms": 175642.6, "avg_output_tokens_per_task": 45106.7, "last_run_timestamp": "2026-08-01T17:39:14.289Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "OpenAI", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09545477999999999, "avg_duration_per_task_ms": 59370.42, "avg_output_tokens_per_task": 2482.38, "last_run_timestamp": "2026-08-01T08:06:06.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.4725397142857143, "accuracy_std_dev": 0.037145532599612084, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.01800119238095238, "avg_duration_per_task_ms": 57921.885714285716, "avg_output_tokens_per_task": 1903.9952380952382, "last_run_timestamp": "2026-08-29T17:27:31.024Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.4710981791907515, "accuracy_std_dev": 0.2776117089494547, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.01732042774566474, "avg_duration_per_task_ms": 101207.72832369943, "avg_output_tokens_per_task": 18057.771676300577, "last_run_timestamp": "2026-08-29T17:06:47.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4704347391304348, "accuracy_std_dev": 0.03718740744464026, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.015934632173913044, "avg_duration_per_task_ms": 67733.77913043478, "avg_output_tokens_per_task": 2419.831304347826, "last_run_timestamp": "2026-08-29T16:22:50.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.4674228813263525, "accuracy_std_dev": 0.04047160730787914, "run_count": 23, "total_questions": 1146, "avg_cost_per_task_usd": 0.01647780541012216, "avg_duration_per_task_ms": 158446.44415357767, "avg_output_tokens_per_task": 11545.342931937173, "last_run_timestamp": "2026-08-29T16:33:30.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.4666666666666667, "accuracy_std_dev": 0.009428090415820616, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012639673333333332, "avg_duration_per_task_ms": 75585.13333333333, "avg_output_tokens_per_task": 1995.3766666666668, "last_run_timestamp": "2026-05-16T08:26:24.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.46561775, "accuracy_std_dev": 0.03549794632569826, "run_count": 23, "total_questions": 1144, "avg_cost_per_task_usd": 0.01601312237762238, "avg_duration_per_task_ms": 458927.9431818182, "avg_output_tokens_per_task": 13622.401223776224, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.46516165886287625, "accuracy_std_dev": 0.03724010007510945, "run_count": 24, "total_questions": 1196, "avg_cost_per_task_usd": 0.015563879598662204, "avg_duration_per_task_ms": 101650.67391304347, "avg_output_tokens_per_task": 2399.9096989966556, "last_run_timestamp": "2026-08-29T16:22:50.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.46444433333333335, "accuracy_std_dev": 0.008314853589945066, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.033371, "avg_duration_per_task_ms": 305705.87333333335, "avg_output_tokens_per_task": 3594.2266666666665, "last_run_timestamp": "2026-08-15T08:15:42.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.464166625, "accuracy_std_dev": 0.03843292588047877, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05392122499999999, "avg_duration_per_task_ms": 149701.2725, "avg_output_tokens_per_task": 3823.17, "last_run_timestamp": "2026-08-29T17:03:47.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.46333349999999995, "accuracy_std_dev": 0.0166665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20313888, "avg_duration_per_task_ms": 52334.34, "avg_output_tokens_per_task": 2078.1, "last_run_timestamp": "2026-09-01T12:41:31.668Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.463333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26064434, "avg_duration_per_task_ms": 31467.4, "avg_output_tokens_per_task": 1819.56, "last_run_timestamp": "2026-08-01T08:06:06.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Together", "accuracy": 0.4625, "accuracy_std_dev": 0.01198957880828179, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.17673909249999997, "avg_duration_per_task_ms": 37836.6175, "avg_output_tokens_per_task": 1280.4725, "last_run_timestamp": "2026-05-23T08:17:34.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 156315.84, "avg_output_tokens_per_task": 4524, "last_run_timestamp": "2026-09-01T12:59:40.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09873096, "avg_duration_per_task_ms": 65244.1, "avg_output_tokens_per_task": 2748.04, "last_run_timestamp": "2026-08-01T08:05:59.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.46, "accuracy_std_dev": 0.013333000000000012, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02811872, "avg_duration_per_task_ms": 579611.05, "avg_output_tokens_per_task": 20031.27, "last_run_timestamp": "2026-09-01T13:02:37.014Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.46, "accuracy_std_dev": 0.04242640687119284, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01170933, "avg_duration_per_task_ms": 112399.705, "avg_output_tokens_per_task": 1534.125, "last_run_timestamp": "2026-05-09T11:02:11.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.4599999444444445, "accuracy_std_dev": 0.0405517410689003, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.017139246666666674, "avg_duration_per_task_ms": 199527.3188888889, "avg_output_tokens_per_task": 16295.887777777778, "last_run_timestamp": "2026-07-18T16:00:08.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.4593055833333333, "accuracy_std_dev": 0.04214152751237655, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.03695082416666666, "avg_duration_per_task_ms": 45466.5775, "avg_output_tokens_per_task": 2368.125, "last_run_timestamp": "2026-08-29T17:01:46.593Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.45861108333333334, "accuracy_std_dev": 0.16222959206063398, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.02081920916666667, "avg_duration_per_task_ms": 35347.055, "avg_output_tokens_per_task": 1812.1541666666667, "last_run_timestamp": "2026-08-29T16:17:15.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.45848490909090905, "accuracy_std_dev": 0.04206453621078092, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.012749035454545457, "avg_duration_per_task_ms": 51588.314545454545, "avg_output_tokens_per_task": 2288.438181818182, "last_run_timestamp": "2026-08-29T16:43:11.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.4582608260869565, "accuracy_std_dev": 0.050742022611100224, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.05333932869565218, "avg_duration_per_task_ms": 66926.48956521739, "avg_output_tokens_per_task": 1968.3739130434783, "last_run_timestamp": "2026-08-29T16:33:30.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.45777799999999996, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00622334, "avg_duration_per_task_ms": 39170.17333333333, "avg_output_tokens_per_task": 1174.36, "last_run_timestamp": "2026-08-15T08:16:13.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.45686270588235295, "accuracy_std_dev": 0.04373821409889245, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.029164419999999996, "avg_duration_per_task_ms": 27415.028235294118, "avg_output_tokens_per_task": 1606.5, "last_run_timestamp": "2026-08-29T16:58:32.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "AtlasCloud", "accuracy": 0.45644446666666666, "accuracy_std_dev": 0.06152943015594697, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.08120367199999999, "avg_duration_per_task_ms": 51087.073333333334, "avg_output_tokens_per_task": 1477.2133333333334, "last_run_timestamp": "2026-06-30T19:13:39.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.4558333333333333, "accuracy_std_dev": 0.0388641151252955, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.019138924166666665, "avg_duration_per_task_ms": 114207.36, "avg_output_tokens_per_task": 2395.9258333333332, "last_run_timestamp": "2026-08-29T16:33:39.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.45542626162790695, "accuracy_std_dev": 0.018912652487751393, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.046210372093023255, "avg_duration_per_task_ms": 174189.27034883722, "avg_output_tokens_per_task": 1898.7906976744187, "last_run_timestamp": "2026-08-29T16:23:01.234Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "AtlasCloud", "accuracy": 0.4537778, "accuracy_std_dev": 0.03489385074422139, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.017512640000000003, "avg_duration_per_task_ms": 279411.34266666666, "avg_output_tokens_per_task": 7481.072, "last_run_timestamp": "2026-07-01T02:31:49.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.452549, "accuracy_std_dev": 0.12023692617494842, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.08110716235294117, "avg_duration_per_task_ms": 124311.29176470588, "avg_output_tokens_per_task": 7596.611764705883, "last_run_timestamp": "2026-07-04T16:00:25.000Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.45217395652173914, "accuracy_std_dev": 0.04587961773700564, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.06969070695652176, "avg_duration_per_task_ms": 68529.97913043479, "avg_output_tokens_per_task": 1958.8730434782608, "last_run_timestamp": "2026-08-29T16:25:26.800Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Together", "accuracy": 0.45190471428571427, "accuracy_std_dev": 0.05152768095670046, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.02711271714285714, "avg_duration_per_task_ms": 89812.11571428571, "avg_output_tokens_per_task": 1856.527142857143, "last_run_timestamp": "2026-07-04T16:00:25.259Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.4518840869565217, "accuracy_std_dev": 0.0396095334977008, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.1784364313043478, "avg_duration_per_task_ms": 61838.67391304348, "avg_output_tokens_per_task": 1962.6434782608696, "last_run_timestamp": "2026-08-29T16:12:10.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.451310823434992, "accuracy_std_dev": 0.07639218105535146, "run_count": 25, "total_questions": 1246, "avg_cost_per_task_usd": 0.020971863563402894, "avg_duration_per_task_ms": 51140.44863563403, "avg_output_tokens_per_task": 2040.5256821829855, "last_run_timestamp": "2026-08-29T17:17:42.808Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.448889, "accuracy_std_dev": 0.0031423825355930196, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08242190666666667, "avg_duration_per_task_ms": 81216.82666666666, "avg_output_tokens_per_task": 4095.7466666666664, "last_run_timestamp": "2026-08-15T08:11:06.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.448811634369287, "accuracy_std_dev": 0.059025095658296695, "run_count": 11, "total_questions": 547, "avg_cost_per_task_usd": 0.030702687385740404, "avg_duration_per_task_ms": 605640.2120658135, "avg_output_tokens_per_task": 6772.7239488117, "last_run_timestamp": "2026-08-30T00:01:10.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.4466665, "accuracy_std_dev": 0.006666499999999992, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020371870000000004, "avg_duration_per_task_ms": 172937.47, "avg_output_tokens_per_task": 42830.02, "last_run_timestamp": "2026-08-01T17:38:24.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.4466665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20957163, "avg_duration_per_task_ms": 47193.8, "avg_output_tokens_per_task": 2215.92, "last_run_timestamp": "2026-09-01T12:58:37.172Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.44642858482142855, "accuracy_std_dev": 0.022519182833890147, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.04059176116071428, "avg_duration_per_task_ms": 108020.11607142857, "avg_output_tokens_per_task": 1939.064732142857, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "WandB", "accuracy": 0.44571428571428573, "accuracy_std_dev": 0.046860627048162073, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.011708047142857144, "avg_duration_per_task_ms": 28176.38, "avg_output_tokens_per_task": 1525.6285714285714, "last_run_timestamp": "2026-06-20T16:39:11.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.445, "accuracy_std_dev": 0.001667000000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06940407, "avg_duration_per_task_ms": 279300.93, "avg_output_tokens_per_task": 3446.49, "last_run_timestamp": "2026-09-01T12:58:16.471Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.4444444444444444, "accuracy_std_dev": 0.02196410423850697, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.0480205202020202, "avg_duration_per_task_ms": 850576.6565656565, "avg_output_tokens_per_task": 104088.0202020202, "last_run_timestamp": "2026-09-01T17:31:48.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Friendli", "accuracy": 0.4442857142857143, "accuracy_std_dev": 0.04609218801400479, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.02215216285714286, "avg_duration_per_task_ms": 36972.67428571429, "avg_output_tokens_per_task": 1609.7742857142857, "last_run_timestamp": "2026-06-20T17:18:35.419Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.4411765294117647, "accuracy_std_dev": 0.05550646750988218, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.016216156470588234, "avg_duration_per_task_ms": 197255.68705882353, "avg_output_tokens_per_task": 10840.736470588236, "last_run_timestamp": "2026-08-01T17:38:32.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.44073454757929886, "accuracy_std_dev": 0.07859861818914539, "run_count": 24, "total_questions": 1198, "avg_cost_per_task_usd": 0.015726146076794657, "avg_duration_per_task_ms": 440405.62771285477, "avg_output_tokens_per_task": 13779.416527545909, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.44, "accuracy_std_dev": 0.013332999999999984, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02857548, "avg_duration_per_task_ms": 74491.22, "avg_output_tokens_per_task": 2449.6, "last_run_timestamp": "2026-09-01T17:30:53.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.44, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.7291186999999999, "avg_duration_per_task_ms": 35507.54, "avg_output_tokens_per_task": 3272.36, "last_run_timestamp": "2026-08-01T08:05:57.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.43972229166666665, "accuracy_std_dev": 0.05926957176640695, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.03497607833333332, "avg_duration_per_task_ms": 88448.07583333334, "avg_output_tokens_per_task": 2351.838333333333, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.43902436585365856, "accuracy_std_dev": 0.046630122309082575, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.15270590650406504, "avg_duration_per_task_ms": 175421.88617886178, "avg_output_tokens_per_task": 6311.357723577235, "last_run_timestamp": "2026-08-30T00:01:09.486Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.4383965367088608, "accuracy_std_dev": 0.03960836597472182, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.05240046329113924, "avg_duration_per_task_ms": 422067.2658227848, "avg_output_tokens_per_task": 20533.427848101266, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.4370816445783133, "accuracy_std_dev": 0.023605829668696253, "run_count": 20, "total_questions": 996, "avg_cost_per_task_usd": 0.07084799598393576, "avg_duration_per_task_ms": 649532.4809236948, "avg_output_tokens_per_task": 12783.55421686747, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.43666675, "accuracy_std_dev": 0.011055139582452123, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.21359883, "avg_duration_per_task_ms": 47801.905, "avg_output_tokens_per_task": 2174.32, "last_run_timestamp": "2026-09-01T13:02:23.243Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.43666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014376880000000002, "avg_duration_per_task_ms": 93606.58, "avg_output_tokens_per_task": 10159.34, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.4359688229398664, "accuracy_std_dev": 0.11718231670692668, "run_count": 18, "total_questions": 898, "avg_cost_per_task_usd": 0.009257310690423163, "avg_duration_per_task_ms": 55520.83184855234, "avg_output_tokens_per_task": 1748.1336302895322, "last_run_timestamp": "2026-08-29T17:44:16.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.43594414571949, "accuracy_std_dev": 0.06435200699203476, "run_count": 22, "total_questions": 1098, "avg_cost_per_task_usd": 0.011169334244080141, "avg_duration_per_task_ms": 99438.00273224043, "avg_output_tokens_per_task": 1867.516393442623, "last_run_timestamp": "2026-08-29T16:11:10.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cerebras", "accuracy": 0.435, "accuracy_std_dev": 0.029580398915498084, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.012494922499999998, "avg_duration_per_task_ms": 17298.03, "avg_output_tokens_per_task": 1157.0075, "last_run_timestamp": "2026-05-23T08:10:41.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.43422520234113715, "accuracy_std_dev": 0.04823165186928528, "run_count": 24, "total_questions": 1196, "avg_cost_per_task_usd": 0.015668311036789298, "avg_duration_per_task_ms": 173534.67224080267, "avg_output_tokens_per_task": 2336.4021739130435, "last_run_timestamp": "2026-08-29T16:17:35.870Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.43401008121827417, "accuracy_std_dev": 0.029416728833002063, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.035802939086294415, "avg_duration_per_task_ms": 50594.96446700508, "avg_output_tokens_per_task": 1953.4467005076142, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.43236108333333334, "accuracy_std_dev": 0.05038259370549571, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.0171767125, "avg_duration_per_task_ms": 123156.45833333333, "avg_output_tokens_per_task": 2417.2791666666667, "last_run_timestamp": "2026-08-29T16:50:29.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.43172692570281124, "accuracy_std_dev": 0.039473708180097544, "run_count": 20, "total_questions": 996, "avg_cost_per_task_usd": 0.04319263554216868, "avg_duration_per_task_ms": 361663.5903614458, "avg_output_tokens_per_task": 8970.2640562249, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4311113333333333, "accuracy_std_dev": 0.006285236475706817, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023601506666666664, "avg_duration_per_task_ms": 216917.58666666667, "avg_output_tokens_per_task": 3146.213333333333, "last_run_timestamp": "2026-08-15T08:02:48.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.4310830042194093, "accuracy_std_dev": 0.06328266797880157, "run_count": 19, "total_questions": 948, "avg_cost_per_task_usd": 0.02028051582278481, "avg_duration_per_task_ms": 151585.09915611814, "avg_output_tokens_per_task": 10897.122362869199, "last_run_timestamp": "2026-08-01T17:38:29.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.4302777916666667, "accuracy_std_dev": 0.04311821560545223, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.02642653999999999, "avg_duration_per_task_ms": 53021.551666666666, "avg_output_tokens_per_task": 2356.3941666666665, "last_run_timestamp": "2026-08-29T16:10:58.039Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.43, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02958018, "avg_duration_per_task_ms": 73337.54, "avg_output_tokens_per_task": 2441.2, "last_run_timestamp": "2026-09-01T17:31:46.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.43, "accuracy_std_dev": 0.0033330000000000026, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23500686999999998, "avg_duration_per_task_ms": 47776.24, "avg_output_tokens_per_task": 2493.12, "last_run_timestamp": "2026-09-01T13:00:52.000Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.42857142857142855, "accuracy_std_dev": 0.03681171064778607, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.02296006857142857, "avg_duration_per_task_ms": 94686.24857142857, "avg_output_tokens_per_task": 2287.175714285714, "last_run_timestamp": "2026-06-20T17:24:59.951Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.42799069860627176, "accuracy_std_dev": 0.06327294802110665, "run_count": 23, "total_questions": 1148, "avg_cost_per_task_usd": 0.013504289198606273, "avg_duration_per_task_ms": 250638.181184669, "avg_output_tokens_per_task": 12498.641986062717, "last_run_timestamp": "2026-08-22T16:08:38.678Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.42499063860830527, "accuracy_std_dev": 0.04030000519747692, "run_count": 18, "total_questions": 891, "avg_cost_per_task_usd": 0.020175196408529744, "avg_duration_per_task_ms": 230086.4859708193, "avg_output_tokens_per_task": 2539.8159371492707, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.423333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02960835, "avg_duration_per_task_ms": 524604.61, "avg_output_tokens_per_task": 20056.49, "last_run_timestamp": "2026-09-01T12:41:25.851Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.4203703888888889, "accuracy_std_dev": 0.04136388473339579, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.01674081333333333, "avg_duration_per_task_ms": 216767.63555555555, "avg_output_tokens_per_task": 11157.038888888888, "last_run_timestamp": "2026-08-01T17:38:35.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4183674081632653, "accuracy_std_dev": 0.008472500000000022, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.048346357142857135, "avg_duration_per_task_ms": 818319.5204081633, "avg_output_tokens_per_task": 111719.33673469388, "last_run_timestamp": "2026-09-01T17:32:02.570Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.41666675, "accuracy_std_dev": 0.013743584364622648, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.027539789999999998, "avg_duration_per_task_ms": 73803.845, "avg_output_tokens_per_task": 2931.665, "last_run_timestamp": "2026-09-01T17:32:02.600Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.413888875, "accuracy_std_dev": 0.04059353491866295, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.03607687833333334, "avg_duration_per_task_ms": 151776.1875, "avg_output_tokens_per_task": 3792.6, "last_run_timestamp": "2026-08-29T16:11:10.099Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.4136363181818182, "accuracy_std_dev": 0.04711807131739889, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.02628960454545455, "avg_duration_per_task_ms": 119464.79909090909, "avg_output_tokens_per_task": 2606.97, "last_run_timestamp": "2026-08-29T16:31:33.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.413333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06003468, "avg_duration_per_task_ms": 392895.94, "avg_output_tokens_per_task": 17058.96, "last_run_timestamp": "2026-08-01T08:06:29.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.41228081754385965, "accuracy_std_dev": 0.14934019625078246, "run_count": 6, "total_questions": 285, "avg_cost_per_task_usd": 0.03923479298245614, "avg_duration_per_task_ms": 212298.04912280702, "avg_output_tokens_per_task": 88632.91228070176, "last_run_timestamp": "2026-08-29T17:06:38.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41142871428571426, "accuracy_std_dev": 0.022737753675420126, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014496525714285714, "avg_duration_per_task_ms": 78866.04571428572, "avg_output_tokens_per_task": 9061.385714285714, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "AtlasCloud", "accuracy": 0.40956652017937223, "accuracy_std_dev": 0.04505436755688195, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.7669130919282512, "avg_duration_per_task_ms": 375326.5717488789, "avg_output_tokens_per_task": 14899.289237668161, "last_run_timestamp": "2026-07-01T04:28:59.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.4070327283236994, "accuracy_std_dev": 0.022772654274634263, "run_count": 21, "total_questions": 1038, "avg_cost_per_task_usd": 0.006255626204238921, "avg_duration_per_task_ms": 19009.97302504817, "avg_output_tokens_per_task": 331.504816955684, "last_run_timestamp": "2026-08-15T14:15:11.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.40666650000000004, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06123743, "avg_duration_per_task_ms": 234369.43, "avg_output_tokens_per_task": 8023.68, "last_run_timestamp": "2026-09-01T12:41:31.874Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.40379423577235773, "accuracy_std_dev": 0.19339763406453553, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.01662922357723577, "avg_duration_per_task_ms": 1820667.5203252032, "avg_output_tokens_per_task": 111556.55284552845, "last_run_timestamp": "2026-08-29T17:12:32.860Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.40347830434782606, "accuracy_std_dev": 0.03645492416006468, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.017620779130434784, "avg_duration_per_task_ms": 171847.04, "avg_output_tokens_per_task": 2749.1495652173912, "last_run_timestamp": "2026-08-29T16:17:08.788Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.4032566666666667, "accuracy_std_dev": 0.0558289469572196, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.22458803448275863, "avg_duration_per_task_ms": 344124.2744252874, "avg_output_tokens_per_task": 17850.82471264368, "last_run_timestamp": "2026-07-18T16:00:08.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.40094352201257866, "accuracy_std_dev": 0.042095981346942486, "run_count": 13, "total_questions": 636, "avg_cost_per_task_usd": 0.05157444339622641, "avg_duration_per_task_ms": 190922.58805031446, "avg_output_tokens_per_task": 3934.8223270440253, "last_run_timestamp": "2026-09-26T08:09:25.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.40055558333333335, "accuracy_std_dev": 0.02035940571929975, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.009198192499999995, "avg_duration_per_task_ms": 42365.11, "avg_output_tokens_per_task": 393.8333333333333, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.4, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.1315643, "avg_duration_per_task_ms": 71996.84, "avg_output_tokens_per_task": 3249.82, "last_run_timestamp": "2026-08-01T08:06:02.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.4, "accuracy_std_dev": 0.019626323972325204, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020620846666666668, "avg_duration_per_task_ms": 194089.08666666667, "avg_output_tokens_per_task": 3830.9066666666668, "last_run_timestamp": "2026-08-15T08:02:44.458Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.39416666666666667, "accuracy_std_dev": 0.026056356301720738, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.008334941666666665, "avg_duration_per_task_ms": 21964.856666666667, "avg_output_tokens_per_task": 381.19166666666666, "last_run_timestamp": "2026-08-29T16:10:09.208Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Inceptron", "accuracy": 0.3933333333333333, "accuracy_std_dev": 0.03197221015541812, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.004903370000000001, "avg_duration_per_task_ms": 11677.77, "avg_output_tokens_per_task": 233.46333333333334, "last_run_timestamp": "2026-05-16T08:28:13.794Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.39, "accuracy_std_dev": 0.014782396941407474, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.028387520000000006, "avg_duration_per_task_ms": 78569.82333333333, "avg_output_tokens_per_task": 4258.516666666666, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3860600450751252, "accuracy_std_dev": 0.0672991061241096, "run_count": 24, "total_questions": 1198, "avg_cost_per_task_usd": 0.03661595993322203, "avg_duration_per_task_ms": 170385.86227045074, "avg_output_tokens_per_task": 2372.5033388981637, "last_run_timestamp": "2026-08-29T16:13:54.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.3856944583333333, "accuracy_std_dev": 0.07826521681329195, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.038991605833333345, "avg_duration_per_task_ms": 59134.56, "avg_output_tokens_per_task": 1840.6966666666667, "last_run_timestamp": "2026-08-29T16:15:12.659Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.38527770833333336, "accuracy_std_dev": 0.033332128566743696, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.04909040916666666, "avg_duration_per_task_ms": 79084.84416666666, "avg_output_tokens_per_task": 2047.975, "last_run_timestamp": "2026-08-29T16:33:39.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.3837681304347826, "accuracy_std_dev": 0.032112499031163325, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.016648859999999998, "avg_duration_per_task_ms": 124100.78260869565, "avg_output_tokens_per_task": 2798.5713043478263, "last_run_timestamp": "2026-08-29T16:10:20.675Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.38358462814070354, "accuracy_std_dev": 0.035037700884760284, "run_count": 24, "total_questions": 1194, "avg_cost_per_task_usd": 0.013517370184254609, "avg_duration_per_task_ms": 48228.662479061975, "avg_output_tokens_per_task": 593.5661641541038, "last_run_timestamp": "2026-08-29T16:14:38.085Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.38333349999999994, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07100836, "avg_duration_per_task_ms": 283936.54, "avg_output_tokens_per_task": 7506.19, "last_run_timestamp": "2026-09-01T12:58:37.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.38313723529411764, "accuracy_std_dev": 0.04464047933477813, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.01614717176470588, "avg_duration_per_task_ms": 39209.35647058824, "avg_output_tokens_per_task": 2326.121176470588, "last_run_timestamp": "2026-07-18T08:03:14.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.38311113333333335, "accuracy_std_dev": 0.05225957113597045, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.012468884, "avg_duration_per_task_ms": 66002.08133333334, "avg_output_tokens_per_task": 1901.1146666666666, "last_run_timestamp": "2026-06-30T19:11:05.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "AtlasCloud", "accuracy": 0.38285714285714284, "accuracy_std_dev": 0.03194382824999699, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.01867379142857143, "avg_duration_per_task_ms": 97094.68714285715, "avg_output_tokens_per_task": 2232.7085714285713, "last_run_timestamp": "2026-06-20T16:46:21.602Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.3822342530282638, "accuracy_std_dev": 0.06305786163883313, "run_count": 15, "total_questions": 743, "avg_cost_per_task_usd": 0.009679043068640645, "avg_duration_per_task_ms": 96270.86137281293, "avg_output_tokens_per_task": 2609.127860026918, "last_run_timestamp": "2026-08-29T16:17:15.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.3822221666666667, "accuracy_std_dev": 0.15283338826307627, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.030608421666666673, "avg_duration_per_task_ms": 214180.72166666668, "avg_output_tokens_per_task": 2714.745, "last_run_timestamp": "2026-08-29T16:33:29.143Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.3796296111111111, "accuracy_std_dev": 0.024492127175298337, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.03998439333333333, "avg_duration_per_task_ms": 13676.166666666666, "avg_output_tokens_per_task": 510.91333333333336, "last_run_timestamp": "2026-07-18T16:00:09.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.3793939090909091, "accuracy_std_dev": 0.05088740122422613, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.039116065454545455, "avg_duration_per_task_ms": 58462.97090909091, "avg_output_tokens_per_task": 3651.5763636363636, "last_run_timestamp": "2026-08-29T16:25:50.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.3744443333333334, "accuracy_std_dev": 0.01940465134674902, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06861020666666667, "avg_duration_per_task_ms": 17591.08, "avg_output_tokens_per_task": 793.6233333333333, "last_run_timestamp": "2026-08-29T16:18:42.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.3709090909090909, "accuracy_std_dev": 0.09668333303668551, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.04139647545454545, "avg_duration_per_task_ms": 141167.0809090909, "avg_output_tokens_per_task": 5048.265454545454, "last_run_timestamp": "2026-08-29T16:11:34.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Venice", "accuracy": 0.37, "accuracy_std_dev": 0.04898979485566354, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04354687000000001, "avg_duration_per_task_ms": 64045.5575, "avg_output_tokens_per_task": 2587.9975, "last_run_timestamp": "2026-05-23T09:30:59.805Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.36666699999999997, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.80181, "avg_duration_per_task_ms": 63430.68, "avg_output_tokens_per_task": 3064.6, "last_run_timestamp": "2026-08-01T08:06:07.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.3628333, "accuracy_std_dev": 0.054805493135360074, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.019168134000000003, "avg_duration_per_task_ms": 43635.281, "avg_output_tokens_per_task": 2269.592, "last_run_timestamp": "2026-08-01T17:38:37.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3574878985507246, "accuracy_std_dev": 0.02831418715169525, "run_count": 14, "total_questions": 690, "avg_cost_per_task_usd": 0.06476017536231883, "avg_duration_per_task_ms": 326809.4072463768, "avg_output_tokens_per_task": 6876.569565217392, "last_run_timestamp": "2026-09-26T08:14:46.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.3555554444444445, "accuracy_std_dev": 0.3294587300632867, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021335997777777782, "avg_duration_per_task_ms": 65350.72222222222, "avg_output_tokens_per_task": 11586.551111111112, "last_run_timestamp": "2026-09-26T16:00:24.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.35151509090909094, "accuracy_std_dev": 0.04450312047374685, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.023887789090909096, "avg_duration_per_task_ms": 76519.86636363636, "avg_output_tokens_per_task": 3035.681818181818, "last_run_timestamp": "2026-08-29T16:13:32.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.3461934413580247, "accuracy_std_dev": 0.030214013647418478, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.07313393055555555, "avg_duration_per_task_ms": 211770.98302469135, "avg_output_tokens_per_task": 6862.71450617284, "last_run_timestamp": "2026-09-26T08:01:16.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.34058, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.11353834782608695, "avg_duration_per_task_ms": 397172.5, "avg_output_tokens_per_task": 14067.434782608696, "last_run_timestamp": "2026-08-01T08:05:57.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.33351874638487206, "accuracy_std_dev": 0.08113788331325603, "run_count": 18, "total_questions": 899, "avg_cost_per_task_usd": 0.02762981312569522, "avg_duration_per_task_ms": 64964.9043381535, "avg_output_tokens_per_task": 3953.5528364849833, "last_run_timestamp": "2026-07-25T08:07:19.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.3268333, "accuracy_std_dev": 0.04588001392556459, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.020283186000000005, "avg_duration_per_task_ms": 68109.949, "avg_output_tokens_per_task": 2751.024, "last_run_timestamp": "2026-08-01T17:38:36.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Novita", "accuracy": 0.3257778, "accuracy_std_dev": 0.06723601430602502, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.010765525333333335, "avg_duration_per_task_ms": 44415.11066666667, "avg_output_tokens_per_task": 1680.8693333333333, "last_run_timestamp": "2026-07-01T01:59:56.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.3242857857142857, "accuracy_std_dev": 0.042134165636483804, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.07503298571428571, "avg_duration_per_task_ms": 131659.5957142857, "avg_output_tokens_per_task": 7119.688571428572, "last_run_timestamp": "2026-09-26T08:09:25.338Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.31383302376599637, "accuracy_std_dev": 0.12215659515952029, "run_count": 22, "total_questions": 1094, "avg_cost_per_task_usd": 0.10114592687385739, "avg_duration_per_task_ms": 309255.3491773309, "avg_output_tokens_per_task": 6164.430530164534, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.31037033333333336, "accuracy_std_dev": 0.06921873819598595, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.006170470000000001, "avg_duration_per_task_ms": 68229.11888888889, "avg_output_tokens_per_task": 1564.5266666666666, "last_run_timestamp": "2026-08-22T16:04:58.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Fireworks", "accuracy": 0.3, "accuracy_std_dev": 0.12837445228704972, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.168574564, "avg_duration_per_task_ms": 489750.116, "avg_output_tokens_per_task": 17487.74, "last_run_timestamp": "2026-05-09T17:11:48.691Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Groq", "accuracy": 0.29666649999999994, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.05697704, "avg_duration_per_task_ms": 17166.81, "avg_output_tokens_per_task": 1588.56, "last_run_timestamp": "2026-07-04T16:00:20.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.293888875, "accuracy_std_dev": 0.03696429813269432, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.10173790416666667, "avg_duration_per_task_ms": 65960.12166666667, "avg_output_tokens_per_task": 1260.8666666666666, "last_run_timestamp": "2026-08-29T17:05:59.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.2926605816513761, "accuracy_std_dev": 0.03933661777666503, "run_count": 22, "total_questions": 1090, "avg_cost_per_task_usd": 0.012492933944954132, "avg_duration_per_task_ms": 163693.22110091744, "avg_output_tokens_per_task": 2119.8, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.2904954257028112, "accuracy_std_dev": 0.19463944059054425, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.1360842630522088, "avg_duration_per_task_ms": 731552.421686747, "avg_output_tokens_per_task": 5033.198795180723, "last_run_timestamp": "2026-09-26T08:05:39.969Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.28177779999999997, "accuracy_std_dev": 0.018129828637542793, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.0062486493333333325, "avg_duration_per_task_ms": 80386.89066666667, "avg_output_tokens_per_task": 1861.8466666666666, "last_run_timestamp": "2026-07-11T08:03:53.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.27, "accuracy_std_dev": 0.03, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02140288, "avg_duration_per_task_ms": 65620.09, "avg_output_tokens_per_task": 3949.38, "last_run_timestamp": "2026-09-01T17:29:05.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.266667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02052575, "avg_duration_per_task_ms": 64186.84, "avg_output_tokens_per_task": 3663.19, "last_run_timestamp": "2026-09-01T17:31:47.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.26041658333333334, "accuracy_std_dev": 0.01347799999999999, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.07850963541666667, "avg_duration_per_task_ms": 588191.84375, "avg_output_tokens_per_task": 5667.5625, "last_run_timestamp": "2026-09-01T12:59:55.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Chutes", "accuracy": 0.26, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09888683999999999, "avg_duration_per_task_ms": 848196.66, "avg_output_tokens_per_task": 16087.38, "last_run_timestamp": "2026-05-02T21:11:19.074Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.25, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08593323, "avg_duration_per_task_ms": 547480.87, "avg_output_tokens_per_task": 5216.43, "last_run_timestamp": "2026-09-01T12:57:03.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Friendli", "accuracy": 0.24666666666666667, "accuracy_std_dev": 0.029814239699997202, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07892822666666666, "avg_duration_per_task_ms": 52263.34333333333, "avg_output_tokens_per_task": 10821.06, "last_run_timestamp": "2026-05-16T08:51:28.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "WandB", "accuracy": 0.2057142857142857, "accuracy_std_dev": 0.05038626311013829, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.08984258285714286, "avg_duration_per_task_ms": 35641.26, "avg_output_tokens_per_task": 1649.8585714285714, "last_run_timestamp": "2026-06-20T08:23:22.767Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.19841299319727893, "accuracy_std_dev": 0.016383133956060492, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.011659061224489795, "avg_duration_per_task_ms": 134707.1224489796, "avg_output_tokens_per_task": 5676.0204081632655, "last_run_timestamp": "2026-09-01T13:00:50.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.18712804910714284, "accuracy_std_dev": 0.063489276040705, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.019668446428571427, "avg_duration_per_task_ms": 108649.33705357143, "avg_output_tokens_per_task": 2703.4866071428573, "last_run_timestamp": "2026-07-18T08:04:09.749Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Chutes", "accuracy": 0.18, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015152460000000001, "avg_duration_per_task_ms": 1356163.7, "avg_output_tokens_per_task": 13773.38, "last_run_timestamp": "2026-05-02T10:35:28.048Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.17888888888888888, "accuracy_std_dev": 0.06174408353742521, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021073126666666667, "avg_duration_per_task_ms": 119786.34888888888, "avg_output_tokens_per_task": 2837.891111111111, "last_run_timestamp": "2026-07-18T08:04:25.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01877496, "avg_duration_per_task_ms": 63137.36, "avg_output_tokens_per_task": 2797.32, "last_run_timestamp": "2026-08-01T08:06:16.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.016978299999999998, "avg_duration_per_task_ms": 57790.52, "avg_output_tokens_per_task": 2492.08, "last_run_timestamp": "2026-08-01T08:06:07.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.1366665, "accuracy_std_dev": 0.003333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00694106, "avg_duration_per_task_ms": 56167.4, "avg_output_tokens_per_task": 2292.83, "last_run_timestamp": "2026-09-01T17:31:40.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.125, "accuracy_std_dev": 0.020750046855368784, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.00834746, "avg_duration_per_task_ms": 82234.105, "avg_output_tokens_per_task": 2517.12, "last_run_timestamp": "2026-09-01T17:31:43.416Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.1033335, "accuracy_std_dev": 0.003333499999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0070183400000000005, "avg_duration_per_task_ms": 61321.36, "avg_output_tokens_per_task": 2324.09, "last_run_timestamp": "2026-09-01T17:31:46.301Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.033333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0007856200000000001, "avg_duration_per_task_ms": 1400.1, "avg_output_tokens_per_task": 126.5, "last_run_timestamp": "2026-06-30T19:06:38.991Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.013333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00098206, "avg_duration_per_task_ms": 2461.54, "avg_output_tokens_per_task": 61.9, "last_run_timestamp": "2026-07-01T01:58:01.413Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.006667000000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00013588, "avg_duration_per_task_ms": 446, "avg_output_tokens_per_task": 9.72, "last_run_timestamp": "2026-06-30T19:02:54.924Z" } ] }