{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-09-21", "retrieved_at": "2026-09-21T01:46:02.485Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's newest 10,000 non-zero-score runs: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2068, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.955357, "accuracy_std_dev": null, "run_count": 1, "total_questions": 56, "avg_cost_per_task_usd": 0.002410089285714286, "avg_duration_per_task_ms": 98368.17857142857, "avg_output_tokens_per_task": 4744.517857142857, "last_run_timestamp": "2026-09-16T18:25:41.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2003321212121212, "avg_duration_per_task_ms": 124514.04545454546, "avg_output_tokens_per_task": 16571.29797979798, "last_run_timestamp": "2026-09-01T17:50:10.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra", "model_slug": "sakana/fugu-ultra", "model_name": "Fugu Ultra", "provider_name": "auto-routing", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 1.148739303030303, "avg_duration_per_task_ms": 328779.10101010103, "avg_output_tokens_per_task": 33271.606060606064, "last_run_timestamp": "2026-09-15T01:07:47.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.116425398989899, "avg_duration_per_task_ms": 44572.09595959596, "avg_output_tokens_per_task": 2176.570707070707, "last_run_timestamp": "2026-09-06T03:17:58.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.945286, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2012271818181818, "avg_duration_per_task_ms": 134249.20707070708, "avg_output_tokens_per_task": 16644.886363636364, "last_run_timestamp": "2026-09-01T17:53:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02981064646464646, "avg_duration_per_task_ms": 28082.027777777777, "avg_output_tokens_per_task": 7803.272727272727, "last_run_timestamp": "2026-09-01T13:28:27.192Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.20185933333333334, "avg_duration_per_task_ms": 126427.48484848485, "avg_output_tokens_per_task": 16698.565656565657, "last_run_timestamp": "2026-09-01T17:51:04.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9436415317919074, "accuracy_std_dev": 0.0020135000000000014, "run_count": 2, "total_questions": 346, "avg_cost_per_task_usd": 0.09933485549132949, "avg_duration_per_task_ms": 198116.95664739885, "avg_output_tokens_per_task": 1767.9046242774566, "last_run_timestamp": "2026-09-06T03:05:46.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.942761, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.030509156565656566, "avg_duration_per_task_ms": 68136.79292929293, "avg_output_tokens_per_task": 7993.409090909091, "last_run_timestamp": "2026-09-01T13:25:46.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.9377104999999999, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.30815713636363634, "avg_duration_per_task_ms": 160641.4873737374, "avg_output_tokens_per_task": 10142.272727272728, "last_run_timestamp": "2026-09-01T13:20:32.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.93771, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.4446258484848485, "avg_duration_per_task_ms": 116119.66666666667, "avg_output_tokens_per_task": 11825.727272727272, "last_run_timestamp": "2026-08-01T08:45:01.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.9343432500000001, "accuracy_std_dev": 0.006628136630117082, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.1361736212121212, "avg_duration_per_task_ms": 193730.40025252526, "avg_output_tokens_per_task": 8868.685606060606, "last_run_timestamp": "2026-09-19T09:16:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.9335082598425197, "accuracy_std_dev": 0.011397999999999964, "run_count": 2, "total_questions": 381, "avg_cost_per_task_usd": 0.0213603280839895, "avg_duration_per_task_ms": 53235.17585301837, "avg_output_tokens_per_task": 7688.425196850394, "last_run_timestamp": "2026-09-01T13:29:18.783Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-ultra-v2", "model_slug": "sakana/fugu-ultra-v2", "model_name": "Fugu Ultra V2", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.5025571414141414, "avg_duration_per_task_ms": 194915.41919191918, "avg_output_tokens_per_task": 13806.227272727272, "last_run_timestamp": "2026-09-15T01:07:48.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.12205839393939394, "avg_duration_per_task_ms": 394338.13636363635, "avg_output_tokens_per_task": 20077.80808080808, "last_run_timestamp": "2026-09-01T13:22:57.685Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9324968387096775, "accuracy_std_dev": 0.005080239036370865, "run_count": 3, "total_questions": 558, "avg_cost_per_task_usd": 0.0715788476702509, "avg_duration_per_task_ms": 64320.11827956989, "avg_output_tokens_per_task": 11534.012544802867, "last_run_timestamp": "2026-09-01T17:51:06.380Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.930976, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.32996825757575754, "avg_duration_per_task_ms": 166479.3383838384, "avg_output_tokens_per_task": 10868.631313131313, "last_run_timestamp": "2026-09-01T13:22:14.451Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.9305555, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.08957261616161614, "avg_duration_per_task_ms": 260418.87121212122, "avg_output_tokens_per_task": 8649.169191919193, "last_run_timestamp": "2026-09-19T09:50:43.404Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9299660000000001, "accuracy_std_dev": 0.009009433123121587, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.32134650707070705, "avg_duration_per_task_ms": 152235.298989899, "avg_output_tokens_per_task": 9994.927272727273, "last_run_timestamp": "2026-09-01T13:24:49.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9292930000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14407252272727272, "avg_duration_per_task_ms": 58692.59848484849, "avg_output_tokens_per_task": 15884.012626262625, "last_run_timestamp": "2026-09-01T17:55:40.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.9290078158277567, "accuracy_std_dev": 0.008556345120594102, "run_count": 19, "total_questions": 3437, "avg_cost_per_task_usd": 0.15611512365434976, "avg_duration_per_task_ms": 137193.63078265928, "avg_output_tokens_per_task": 8758.286005237125, "last_run_timestamp": "2026-09-19T08:58:50.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.92887175, "accuracy_std_dev": 0.005239941859171739, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06947331565656567, "avg_duration_per_task_ms": 67244.06060606061, "avg_output_tokens_per_task": 11635.200757575758, "last_run_timestamp": "2026-09-01T17:56:37.544Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06683859343434344, "avg_duration_per_task_ms": 59600.07575757576, "avg_output_tokens_per_task": 11674.472222222223, "last_run_timestamp": "2026-09-01T17:57:17.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.927609, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14183102777777779, "avg_duration_per_task_ms": 79740.97727272728, "avg_output_tokens_per_task": 15646.176767676769, "last_run_timestamp": "2026-09-01T18:06:42.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9259257500000001, "accuracy_std_dev": 0.004610545920766882, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.1489979318181818, "avg_duration_per_task_ms": 84816.23358585859, "avg_output_tokens_per_task": 15665.375, "last_run_timestamp": "2026-09-01T17:51:30.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9254603241206031, "accuracy_std_dev": 0.020055971452135855, "run_count": 3, "total_questions": 398, "avg_cost_per_task_usd": 0.08006439949748743, "avg_duration_per_task_ms": 226382.3567839196, "avg_output_tokens_per_task": 6134.640703517588, "last_run_timestamp": "2026-09-05T08:30:49.088Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.925347078125, "accuracy_std_dev": 0.005810499999999996, "run_count": 2, "total_questions": 384, "avg_cost_per_task_usd": 0.3411057265625, "avg_duration_per_task_ms": 143482.140625, "avg_output_tokens_per_task": 10207.984375, "last_run_timestamp": "2026-09-01T13:21:58.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9240362551020408, "accuracy_std_dev": 0.010411205213614818, "run_count": 3, "total_questions": 588, "avg_cost_per_task_usd": 0.11667940476190476, "avg_duration_per_task_ms": 366799.1649659864, "avg_output_tokens_per_task": 19230.027210884353, "last_run_timestamp": "2026-09-01T13:27:43.904Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9225940304841601, "accuracy_std_dev": 0.0076093798566288715, "run_count": 9, "total_questions": 1673, "avg_cost_per_task_usd": 0.14876478720860728, "avg_duration_per_task_ms": 123940.04423191871, "avg_output_tokens_per_task": 9777.336521219366, "last_run_timestamp": "2026-09-19T09:02:06.672Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.922559, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.4976751313131313, "avg_duration_per_task_ms": 91843.67676767676, "avg_output_tokens_per_task": 12034.212121212122, "last_run_timestamp": "2026-08-01T08:41:32.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.9221377500000001, "accuracy_std_dev": 0.007277616312193182, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.1708342297979798, "avg_duration_per_task_ms": 352657.0984848485, "avg_output_tokens_per_task": 9736.763888888889, "last_run_timestamp": "2026-09-19T08:58:45.410Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9215687647058822, "accuracy_std_dev": 0.0038094199325123296, "run_count": 3, "total_questions": 561, "avg_cost_per_task_usd": 0.10569379500891266, "avg_duration_per_task_ms": 54609.55971479501, "avg_output_tokens_per_task": 3484.6114081996434, "last_run_timestamp": "2026-09-01T13:22:25.850Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9212118, "accuracy_std_dev": 0.008675973568424486, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1201016404040404, "avg_duration_per_task_ms": 61346.69292929293, "avg_output_tokens_per_task": 3579.7959595959596, "last_run_timestamp": "2026-09-01T13:28:53.908Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.92087525, "accuracy_std_dev": 0.005952071335888052, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.137514726010101, "avg_duration_per_task_ms": 557212.4103535353, "avg_output_tokens_per_task": 37117.12752525252, "last_run_timestamp": "2026-09-19T09:21:26.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.920875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.4329253232323232, "avg_duration_per_task_ms": 113725.84848484848, "avg_output_tokens_per_task": 11493.873737373737, "last_run_timestamp": "2026-08-01T08:53:39.678Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.9208333333333333, "accuracy_std_dev": 0.005676000000000014, "run_count": 2, "total_questions": 360, "avg_cost_per_task_usd": 0.0029125277777777778, "avg_duration_per_task_ms": 461662.2555555556, "avg_output_tokens_per_task": 10228.08888888889, "last_run_timestamp": "2026-09-19T09:15:41.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.9200335, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07324645454545455, "avg_duration_per_task_ms": 236783.91666666666, "avg_output_tokens_per_task": 25006.20707070707, "last_run_timestamp": "2026-09-20T10:38:54.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.9194322857142857, "accuracy_std_dev": 0.008972904191656873, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.17085740764790766, "avg_duration_per_task_ms": 272930.75685425685, "avg_output_tokens_per_task": 13023.35064935065, "last_run_timestamp": "2026-09-19T08:49:38.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191915, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07126612626262625, "avg_duration_per_task_ms": 64525.257575757576, "avg_output_tokens_per_task": 3520.8156565656564, "last_run_timestamp": "2026-09-01T13:21:08.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9185987869565218, "accuracy_std_dev": 0.013465663103476456, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.1389484079710145, "avg_duration_per_task_ms": 369171.10942028987, "avg_output_tokens_per_task": 10521.670289855072, "last_run_timestamp": "2026-09-19T08:59:31.029Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.9185184, "accuracy_std_dev": 0.0060794207816205845, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1914820111111111, "avg_duration_per_task_ms": 356646.5292929293, "avg_output_tokens_per_task": 43359.274747474745, "last_run_timestamp": "2026-09-20T10:34:51.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.9176481996466431, "accuracy_std_dev": 0.014209394819477809, "run_count": 9, "total_questions": 1698, "avg_cost_per_task_usd": 0.118061925795053, "avg_duration_per_task_ms": 124721.6089517079, "avg_output_tokens_per_task": 7956.564782096584, "last_run_timestamp": "2026-09-19T09:54:41.293Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.9163074598792063, "accuracy_std_dev": 0.01614503396730882, "run_count": 6, "total_questions": 1159, "avg_cost_per_task_usd": 0.007566448662640208, "avg_duration_per_task_ms": 499842.77135461604, "avg_output_tokens_per_task": 19527.920621225196, "last_run_timestamp": "2026-09-20T10:39:48.638Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.9155657692307693, "accuracy_std_dev": 0.012784482626358756, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.02835553613053613, "avg_duration_per_task_ms": 214027.3453768454, "avg_output_tokens_per_task": 23521.37567987568, "last_run_timestamp": "2026-09-19T16:08:21.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.914983, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07349445707070708, "avg_duration_per_task_ms": 64386.71464646464, "avg_output_tokens_per_task": 3569.0353535353534, "last_run_timestamp": "2026-09-01T13:22:37.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.91456225, "accuracy_std_dev": 0.004014799893830335, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.10835830176767677, "avg_duration_per_task_ms": 621940.4457070708, "avg_output_tokens_per_task": 40133.22601010101, "last_run_timestamp": "2026-09-19T08:53:55.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9138044000000001, "accuracy_std_dev": 0.003749296606031599, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.28000743232323233, "avg_duration_per_task_ms": 356725.703030303, "avg_output_tokens_per_task": 38588.78080808081, "last_run_timestamp": "2026-09-19T08:49:14.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.9133028235677082, "accuracy_std_dev": 0.006692965934471823, "run_count": 8, "total_questions": 1536, "avg_cost_per_task_usd": 0.1390292682291667, "avg_duration_per_task_ms": 291227.6529947917, "avg_output_tokens_per_task": 9127.681640625, "last_run_timestamp": "2026-09-19T08:55:48.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9132995, "accuracy_std_dev": 0.01430950000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.15749233333333332, "avg_duration_per_task_ms": 547358.5101010101, "avg_output_tokens_per_task": 39453.482323232325, "last_run_timestamp": "2026-09-19T08:48:18.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.9122337978723405, "accuracy_std_dev": 0.011253882096414569, "run_count": 5, "total_questions": 940, "avg_cost_per_task_usd": 0.01295165319148936, "avg_duration_per_task_ms": 355502.26914893615, "avg_output_tokens_per_task": 16382.894680851065, "last_run_timestamp": "2026-09-20T10:40:15.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9102129999999999, "accuracy_std_dev": 0.002861334420627323, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0912948888888889, "avg_duration_per_task_ms": 155548.53703703705, "avg_output_tokens_per_task": 27869.387205387204, "last_run_timestamp": "2026-09-01T13:31:50.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.9099324999999999, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.030082277777777777, "avg_duration_per_task_ms": 178098.15404040404, "avg_output_tokens_per_task": 24895.11111111111, "last_run_timestamp": "2026-09-19T08:54:37.084Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.909652, "accuracy_std_dev": 0.0020998715833752102, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03531527104377104, "avg_duration_per_task_ms": 174484.34343434343, "avg_output_tokens_per_task": 29238.693602693602, "last_run_timestamp": "2026-09-19T08:07:24.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stealth/union-alpha", "model_slug": "stealth/union-alpha", "model_name": "Union Alpha", "provider_name": "auto-routing", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 146448.00505050505, "avg_output_tokens_per_task": 4567.823232323232, "last_run_timestamp": "2026-09-15T01:07:47.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.9090906666666667, "accuracy_std_dev": 0.0036369299629709197, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.037642313131313124, "avg_duration_per_task_ms": 128088.45959595959, "avg_output_tokens_per_task": 28275.836700336702, "last_run_timestamp": "2026-09-19T08:49:31.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.9087186329479767, "accuracy_std_dev": 0.004670283372078221, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.0291304161849711, "avg_duration_per_task_ms": 405484.31502890174, "avg_output_tokens_per_task": 30120.609826589596, "last_run_timestamp": "2026-09-19T16:37:44.222Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9086567723440133, "accuracy_std_dev": 0.002243205890386943, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.0342340826306914, "avg_duration_per_task_ms": 141097.99494097807, "avg_output_tokens_per_task": 28836.671163575043, "last_run_timestamp": "2026-09-19T08:47:59.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto", "model_slug": "openrouter/auto", "model_name": "Auto Router", "provider_name": "auto-routing", "accuracy": 0.907407, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00787879292929293, "avg_duration_per_task_ms": 237113.06565656565, "avg_output_tokens_per_task": 23447.141414141413, "last_run_timestamp": "2026-09-15T01:07:47.971Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.9070287642172525, "accuracy_std_dev": 0.016195054520979538, "run_count": 8, "total_questions": 1565, "avg_cost_per_task_usd": 0.1865306019169329, "avg_duration_per_task_ms": 516947.6555910543, "avg_output_tokens_per_task": 12424.835782747603, "last_run_timestamp": "2026-09-19T08:02:29.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9067542764456982, "accuracy_std_dev": 0.021950605534673606, "run_count": 11, "total_questions": 2127, "avg_cost_per_task_usd": 0.0353538086506817, "avg_duration_per_task_ms": 260805.93888105312, "avg_output_tokens_per_task": 29657.3121767748, "last_run_timestamp": "2026-09-19T16:10:18.592Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.906285, "accuracy_std_dev": 0.004827531943619518, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03418770707070708, "avg_duration_per_task_ms": 132654.23905723906, "avg_output_tokens_per_task": 28802.200336700338, "last_run_timestamp": "2026-09-19T08:48:26.475Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.9056333457943925, "accuracy_std_dev": 0.02007868381757514, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.03042957359813084, "avg_duration_per_task_ms": 291576.496105919, "avg_output_tokens_per_task": 25313.03971962617, "last_run_timestamp": "2026-09-19T16:48:55.177Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.9054903294117648, "accuracy_std_dev": 0.002950830881105991, "run_count": 3, "total_questions": 425, "avg_cost_per_task_usd": 0.0986494188235294, "avg_duration_per_task_ms": 590399.5082352941, "avg_output_tokens_per_task": 26203.308235294116, "last_run_timestamp": "2026-09-05T08:31:24.281Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.904882, "accuracy_std_dev": 0.020544122168315374, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.13592807828282827, "avg_duration_per_task_ms": 578809.478114478, "avg_output_tokens_per_task": 38384.35353535353, "last_run_timestamp": "2026-09-19T08:57:25.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.9042877457886676, "accuracy_std_dev": 0.034874145815887096, "run_count": 14, "total_questions": 2612, "avg_cost_per_task_usd": 0.1541255895865237, "avg_duration_per_task_ms": 517283.3177641654, "avg_output_tokens_per_task": 8744.607963246553, "last_run_timestamp": "2026-09-19T12:44:31.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.9041560076335877, "accuracy_std_dev": 0.003276000000000001, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.021076463104325698, "avg_duration_per_task_ms": 791723.9363867685, "avg_output_tokens_per_task": 17407.793893129772, "last_run_timestamp": "2026-09-19T09:03:39.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.9040400000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02929719696969697, "avg_duration_per_task_ms": 146549.648989899, "avg_output_tokens_per_task": 27342.593434343435, "last_run_timestamp": "2026-09-01T13:36:23.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9037548157099697, "accuracy_std_dev": 0.016887706798429505, "run_count": 12, "total_questions": 2317, "avg_cost_per_task_usd": 0.031995520932239965, "avg_duration_per_task_ms": 304192.94389296504, "avg_output_tokens_per_task": 26670.303409581356, "last_run_timestamp": "2026-09-19T16:45:47.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.9031985, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14676221212121213, "avg_duration_per_task_ms": 106397.32070707071, "avg_output_tokens_per_task": 9655.194444444445, "last_run_timestamp": "2026-09-01T17:54:59.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.9031985, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.043649184343434344, "avg_duration_per_task_ms": 141329.08080808082, "avg_output_tokens_per_task": 28078.323232323233, "last_run_timestamp": "2026-09-01T13:21:17.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.9013081934015927, "accuracy_std_dev": 0.01190368377259848, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.03069646018202503, "avg_duration_per_task_ms": 230610.76109215018, "avg_output_tokens_per_task": 25458.812855517634, "last_run_timestamp": "2026-09-19T16:27:17.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.9012855000000001, "accuracy_std_dev": 0.00564207676746779, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.21591182460973365, "avg_duration_per_task_ms": 95090.06290174472, "avg_output_tokens_per_task": 7868.12924701561, "last_run_timestamp": "2026-09-19T08:52:57.055Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.9006734999999999, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03283414393939394, "avg_duration_per_task_ms": 255154.72727272726, "avg_output_tokens_per_task": 28959.247474747473, "last_run_timestamp": "2026-09-19T09:07:59.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.9006733333333333, "accuracy_std_dev": 0.007653512933432718, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.12550200336700337, "avg_duration_per_task_ms": 30223.45286195286, "avg_output_tokens_per_task": 2299.0185185185187, "last_run_timestamp": "2026-09-19T08:48:46.513Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.9006732857142857, "accuracy_std_dev": 0.005831732595140517, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.20125868181818185, "avg_duration_per_task_ms": 98286.09018759019, "avg_output_tokens_per_task": 7826.790764790765, "last_run_timestamp": "2026-09-19T08:55:19.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.9006730000000001, "accuracy_std_dev": 0.007008969420202443, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1542171708754209, "avg_duration_per_task_ms": 127676.21296296296, "avg_output_tokens_per_task": 9507.647306397306, "last_run_timestamp": "2026-09-01T17:57:22.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.9003924999999999, "accuracy_std_dev": 0.005181281268502329, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.23008475925925925, "avg_duration_per_task_ms": 111602.88636363637, "avg_output_tokens_per_task": 9003.594276094276, "last_run_timestamp": "2026-09-19T08:48:22.873Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9002196347031963, "accuracy_std_dev": 0.02634610017364999, "run_count": 10, "total_questions": 1971, "avg_cost_per_task_usd": 0.043411074581430745, "avg_duration_per_task_ms": 257713.3805175038, "avg_output_tokens_per_task": 31013.896499238966, "last_run_timestamp": "2026-09-19T16:06:20.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.8998315, "accuracy_std_dev": 0.007575499999999957, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13947029797979799, "avg_duration_per_task_ms": 103780.42929292929, "avg_output_tokens_per_task": 9170.424242424242, "last_run_timestamp": "2026-09-01T17:56:45.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.8996633999999999, "accuracy_std_dev": 0.005792805834826492, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007343192929292929, "avg_duration_per_task_ms": 219337.0404040404, "avg_output_tokens_per_task": 25809.762626262625, "last_run_timestamp": "2026-08-08T09:30:52.456Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.8993387243502051, "accuracy_std_dev": 0.026386926357751014, "run_count": 8, "total_questions": 1462, "avg_cost_per_task_usd": 0.027840691518467853, "avg_duration_per_task_ms": 740625.9062927497, "avg_output_tokens_per_task": 24883.40560875513, "last_run_timestamp": "2026-09-12T16:17:57.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.8992302857142858, "accuracy_std_dev": 0.03276088896279607, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04223221067821068, "avg_duration_per_task_ms": 271453.88095238095, "avg_output_tokens_per_task": 31163.87950937951, "last_run_timestamp": "2026-09-19T16:00:30.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.8990668549618319, "accuracy_std_dev": 0.0016184999999999672, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.01465777099236641, "avg_duration_per_task_ms": 713790.7251908397, "avg_output_tokens_per_task": 29013.936386768448, "last_run_timestamp": "2026-09-19T09:05:22.577Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.8989897500000001, "accuracy_std_dev": 0.005831706820262806, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008321901515151515, "avg_duration_per_task_ms": 193168.351010101, "avg_output_tokens_per_task": 25995.64835858586, "last_run_timestamp": "2026-09-19T09:01:52.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8986529999999999, "accuracy_std_dev": 0.004441544821343102, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03175560707070707, "avg_duration_per_task_ms": 205238.4111111111, "avg_output_tokens_per_task": 26427.771717171716, "last_run_timestamp": "2026-09-19T09:12:15.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.8985087142857142, "accuracy_std_dev": 0.025191571054701628, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.15058746031746031, "avg_duration_per_task_ms": 621838.7965367965, "avg_output_tokens_per_task": 38175.91341991342, "last_run_timestamp": "2026-09-19T09:15:37.835Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.898148, "accuracy_std_dev": 0.014408658178331523, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.035229258838383834, "avg_duration_per_task_ms": 168991.77904040404, "avg_output_tokens_per_task": 26420.739898989897, "last_run_timestamp": "2026-09-19T08:49:42.594Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.898148, "accuracy_std_dev": 0.04651015131398736, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1383203263888889, "avg_duration_per_task_ms": 228916.913510101, "avg_output_tokens_per_task": 9157.267676767677, "last_run_timestamp": "2026-09-19T08:48:42.819Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.8978918260869566, "accuracy_std_dev": 0.006291373173206031, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.2227677929292929, "avg_duration_per_task_ms": 95189.05489679403, "avg_output_tokens_per_task": 8106.728590250329, "last_run_timestamp": "2026-09-19T08:48:23.715Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.8977272500000001, "accuracy_std_dev": 0.05811571909292271, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.09910069823232324, "avg_duration_per_task_ms": 237858.36363636365, "avg_output_tokens_per_task": 8860.338383838383, "last_run_timestamp": "2026-09-19T08:47:59.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.8970657142857142, "accuracy_std_dev": 0.01398216255207516, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.2014812243867244, "avg_duration_per_task_ms": 97566.17965367965, "avg_output_tokens_per_task": 7846.960317460317, "last_run_timestamp": "2026-09-19T08:48:23.322Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.8969693999999999, "accuracy_std_dev": 0.016713400846027716, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.027060835353535352, "avg_duration_per_task_ms": 212549.52424242423, "avg_output_tokens_per_task": 25322.420202020203, "last_run_timestamp": "2026-09-19T08:53:38.936Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.8964645000000001, "accuracy_std_dev": 0.007575499999999957, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03375818181818182, "avg_duration_per_task_ms": 56405.70202020202, "avg_output_tokens_per_task": 3599.8333333333335, "last_run_timestamp": "2026-09-01T13:22:46.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.8962534351585013, "accuracy_std_dev": 0.007566251387483333, "run_count": 16, "total_questions": 3123, "avg_cost_per_task_usd": 0.12569597342299071, "avg_duration_per_task_ms": 58925.71245597182, "avg_output_tokens_per_task": 4794.491194364393, "last_run_timestamp": "2026-09-19T09:01:07.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.8960958972972972, "accuracy_std_dev": 0.005877557334660528, "run_count": 3, "total_questions": 555, "avg_cost_per_task_usd": 0.1555473855855856, "avg_duration_per_task_ms": 83534.15135135135, "avg_output_tokens_per_task": 9299.446846846848, "last_run_timestamp": "2026-09-01T17:56:10.791Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.8960617898734176, "accuracy_std_dev": 0.01049307968079437, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.011295574261603377, "avg_duration_per_task_ms": 279006.29071729956, "avg_output_tokens_per_task": 26796.40464135021, "last_run_timestamp": "2026-09-19T09:37:05.170Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.8960286613718412, "accuracy_std_dev": 0.061702303840823536, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.14792670397111912, "avg_duration_per_task_ms": 306430.2418772563, "avg_output_tokens_per_task": 10222.844765342961, "last_run_timestamp": "2026-09-19T09:07:35.756Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8959034999999999, "accuracy_std_dev": 0.009139365308925972, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.19779903198653198, "avg_duration_per_task_ms": 96138.98316498316, "avg_output_tokens_per_task": 7715.625420875421, "last_run_timestamp": "2026-09-19T08:50:35.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.8953344392265193, "accuracy_std_dev": 0.018557569482759252, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.02011020073664825, "avg_duration_per_task_ms": 228595.4673112339, "avg_output_tokens_per_task": 26682.067219152854, "last_run_timestamp": "2026-09-19T08:54:33.078Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.8951176999999999, "accuracy_std_dev": 0.006210770774227599, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.20379388838383836, "avg_duration_per_task_ms": 100040.05353535354, "avg_output_tokens_per_task": 7943.0055555555555, "last_run_timestamp": "2026-09-19T08:51:26.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.894795448275862, "accuracy_std_dev": 0.021807656270218488, "run_count": 6, "total_questions": 1044, "avg_cost_per_task_usd": 0.16143976149425288, "avg_duration_per_task_ms": 585867.4003831417, "avg_output_tokens_per_task": 37446.46647509579, "last_run_timestamp": "2026-09-19T09:12:47.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.8942333620253164, "accuracy_std_dev": 0.016976412073488586, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.12454873333333333, "avg_duration_per_task_ms": 626537.8101265823, "avg_output_tokens_per_task": 36557.81012658228, "last_run_timestamp": "2026-09-19T09:12:43.627Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AkashML", "accuracy": 0.893939, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03405869696969697, "avg_duration_per_task_ms": 153813.0883838384, "avg_output_tokens_per_task": 9711.138888888889, "last_run_timestamp": "2026-06-20T09:09:33.534Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.893728625, "accuracy_std_dev": 0.007454970371126546, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1200770505050505, "avg_duration_per_task_ms": 58158.018939393936, "avg_output_tokens_per_task": 4592.037878787879, "last_run_timestamp": "2026-09-19T09:07:46.594Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.8932607767114523, "accuracy_std_dev": 0.024235313237801887, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.043238984005118365, "avg_duration_per_task_ms": 591335.9296225208, "avg_output_tokens_per_task": 23011.071657069737, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8931731201011378, "accuracy_std_dev": 0.020887674587840063, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.03478943742098609, "avg_duration_per_task_ms": 165557.26927939316, "avg_output_tokens_per_task": 26087.041719342604, "last_run_timestamp": "2026-09-19T09:00:03.663Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8930976666666668, "accuracy_std_dev": 0.010182556173290759, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05509197895622895, "avg_duration_per_task_ms": 49604.97474747475, "avg_output_tokens_per_task": 3808.635521885522, "last_run_timestamp": "2026-09-01T13:23:50.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.8928136074289877, "accuracy_std_dev": 0.017730075367313348, "run_count": 7, "total_questions": 1373, "avg_cost_per_task_usd": 0.11131419956300073, "avg_duration_per_task_ms": 377366.75018208305, "avg_output_tokens_per_task": 7606.375091041515, "last_run_timestamp": "2026-09-19T09:20:00.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.892567903763697, "accuracy_std_dev": 0.02022831814050264, "run_count": 11, "total_questions": 2099, "avg_cost_per_task_usd": 0.15797696188661267, "avg_duration_per_task_ms": 338559.5883754169, "avg_output_tokens_per_task": 39638.616484040016, "last_run_timestamp": "2026-09-19T08:51:08.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8924730132827325, "accuracy_std_dev": 0.008810976361192897, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.22325217647058823, "avg_duration_per_task_ms": 107407.18848829855, "avg_output_tokens_per_task": 8706.91587602783, "last_run_timestamp": "2026-09-19T08:46:15.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.892396125, "accuracy_std_dev": 0.00693984391822862, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.20264107049663296, "avg_duration_per_task_ms": 70639.64036195286, "avg_output_tokens_per_task": 7379.623106060606, "last_run_timestamp": "2026-09-19T08:49:53.737Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8918440425531915, "accuracy_std_dev": 0.0082731599766957, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.050205347517730495, "avg_duration_per_task_ms": 44870.67021276596, "avg_output_tokens_per_task": 3654.1524822695037, "last_run_timestamp": "2026-09-01T13:21:31.062Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.8918245230352304, "accuracy_std_dev": 0.00608573196090658, "run_count": 4, "total_questions": 738, "avg_cost_per_task_usd": 0.0061554742547425476, "avg_duration_per_task_ms": 427061.5867208672, "avg_output_tokens_per_task": 12975.516260162602, "last_run_timestamp": "2026-09-19T09:12:08.423Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.8914630362257793, "accuracy_std_dev": 0.03662992869870515, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.12755125863521483, "avg_duration_per_task_ms": 562774.203032856, "avg_output_tokens_per_task": 38228.18028643639, "last_run_timestamp": "2026-09-19T09:07:20.465Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Decart", "accuracy": 0.8914139999999999, "accuracy_std_dev": 0.007575999999999972, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03735867676767677, "avg_duration_per_task_ms": 78882.84090909091, "avg_output_tokens_per_task": 10904.39393939394, "last_run_timestamp": "2026-06-20T08:34:12.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.8908169457700652, "accuracy_std_dev": 0.02099999680855173, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.13300457700650758, "avg_duration_per_task_ms": 536629.229934924, "avg_output_tokens_per_task": 37314.125090383226, "last_run_timestamp": "2026-09-19T08:51:35.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8905724999999999, "accuracy_std_dev": 0.011784499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.027024376262626264, "avg_duration_per_task_ms": 278293.2196969697, "avg_output_tokens_per_task": 26273.641414141413, "last_run_timestamp": "2026-09-19T09:00:31.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.890572, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.21863568181818183, "avg_duration_per_task_ms": 106364.56818181818, "avg_output_tokens_per_task": 8520.704545454546, "last_run_timestamp": "2026-09-01T13:30:25.730Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.8905659347708894, "accuracy_std_dev": 0.017636022008661704, "run_count": 10, "total_questions": 1855, "avg_cost_per_task_usd": 0.09373495849056605, "avg_duration_per_task_ms": 523293.3725067385, "avg_output_tokens_per_task": 28466.21778975741, "last_run_timestamp": "2026-09-19T08:54:04.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.8903317142857142, "accuracy_std_dev": 0.024732969688912254, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.08410881962481961, "avg_duration_per_task_ms": 171127.59163059163, "avg_output_tokens_per_task": 28479.191197691198, "last_run_timestamp": "2026-09-19T16:11:37.780Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.8897304999999999, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0318945303030303, "avg_duration_per_task_ms": 52439.469696969696, "avg_output_tokens_per_task": 3386.712121212121, "last_run_timestamp": "2026-09-01T13:38:25.421Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8893674137931035, "accuracy_std_dev": 0.02203700000000003, "run_count": 2, "total_questions": 348, "avg_cost_per_task_usd": 0.045426718390804595, "avg_duration_per_task_ms": 158494.20977011495, "avg_output_tokens_per_task": 32753.790229885057, "last_run_timestamp": "2026-09-19T09:02:12.681Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.8892205742320818, "accuracy_std_dev": 0.012673561529979113, "run_count": 12, "total_questions": 2344, "avg_cost_per_task_usd": 0.0036068412969283274, "avg_duration_per_task_ms": 283009.04991467576, "avg_output_tokens_per_task": 19743.632679180886, "last_run_timestamp": "2026-09-19T08:50:41.183Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.888735406077348, "accuracy_std_dev": 0.010037165766694254, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.2139424162062615, "avg_duration_per_task_ms": 103563.50506445672, "avg_output_tokens_per_task": 8329.639042357274, "last_run_timestamp": "2026-09-19T08:49:51.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8881117317228225, "accuracy_std_dev": 0.00876730099564649, "run_count": 24, "total_questions": 4719, "avg_cost_per_task_usd": 0.13360808010171646, "avg_duration_per_task_ms": 59682.03581267218, "avg_output_tokens_per_task": 4779.846789574062, "last_run_timestamp": "2026-09-19T08:56:54.905Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.8876260833333333, "accuracy_std_dev": 0.013293729934686846, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.011911787037037038, "avg_duration_per_task_ms": 214527.07491582492, "avg_output_tokens_per_task": 24222.33712121212, "last_run_timestamp": "2026-09-19T08:49:45.167Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.887310375, "accuracy_std_dev": 0.006640514916734627, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.23988401546717172, "avg_duration_per_task_ms": 84791.94412878787, "avg_output_tokens_per_task": 8924.59659090909, "last_run_timestamp": "2026-09-19T08:49:34.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.8872055000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20363688131313135, "avg_duration_per_task_ms": 99426.9191919192, "avg_output_tokens_per_task": 7920.752525252526, "last_run_timestamp": "2026-09-01T13:21:53.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8872051666666666, "accuracy_std_dev": 0.008961113052455555, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01774598653198653, "avg_duration_per_task_ms": 138138.01430976432, "avg_output_tokens_per_task": 26885.06818181818, "last_run_timestamp": "2026-09-19T08:49:05.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8864601803278688, "accuracy_std_dev": 0.006211138077858382, "run_count": 3, "total_questions": 549, "avg_cost_per_task_usd": 0.009764420765027322, "avg_duration_per_task_ms": 50630.12750455373, "avg_output_tokens_per_task": 7246.790528233151, "last_run_timestamp": "2026-09-01T17:51:11.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.8864530691399664, "accuracy_std_dev": 0.01029973730112676, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.023206354974704888, "avg_duration_per_task_ms": 81414.45531197301, "avg_output_tokens_per_task": 7880.962900505902, "last_run_timestamp": "2026-09-01T17:55:01.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.886142711026616, "accuracy_std_dev": 0.009981418278950896, "run_count": 8, "total_questions": 1578, "avg_cost_per_task_usd": 0.2087646856780735, "avg_duration_per_task_ms": 102336.10709759188, "avg_output_tokens_per_task": 8124.274397972116, "last_run_timestamp": "2026-09-19T08:49:20.918Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sakana/fugu-max", "model_slug": "sakana/fugu-max", "model_name": "Fugu Max", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07555469191919191, "avg_duration_per_task_ms": 225091.94949494948, "avg_output_tokens_per_task": 12279.550505050505, "last_run_timestamp": "2026-09-15T01:07:47.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Parasail", "accuracy": 0.8855216666666667, "accuracy_std_dev": 0.016666035428046132, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.042562936026936025, "avg_duration_per_task_ms": 323944.202020202, "avg_output_tokens_per_task": 15151.54377104377, "last_run_timestamp": "2026-05-23T08:52:13.739Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.8855215, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.026893237373737375, "avg_duration_per_task_ms": 167702.77272727274, "avg_output_tokens_per_task": 22217.964646464647, "last_run_timestamp": "2026-09-19T09:51:03.686Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.8846799999999999, "accuracy_std_dev": 0.01856306569508389, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.12417679966329967, "avg_duration_per_task_ms": 507255.2138047138, "avg_output_tokens_per_task": 31430.781144781144, "last_run_timestamp": "2026-09-19T09:49:23.372Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.8845707161572053, "accuracy_std_dev": 0.02143221713073307, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.16085208820960697, "avg_duration_per_task_ms": 756655.7283842795, "avg_output_tokens_per_task": 40813.194759825325, "last_run_timestamp": "2026-09-16T00:37:42.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.8835445189873418, "accuracy_std_dev": 0.001982499999999998, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.012081701265822782, "avg_duration_per_task_ms": 336769.7392405063, "avg_output_tokens_per_task": 21488.718987341774, "last_run_timestamp": "2026-09-19T11:58:10.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.8834176976530144, "accuracy_std_dev": 0.013545671626692473, "run_count": 11, "total_questions": 2173, "avg_cost_per_task_usd": 0.007175545329038196, "avg_duration_per_task_ms": 505749.09387942933, "avg_output_tokens_per_task": 25759.638748274276, "last_run_timestamp": "2026-09-19T09:05:08.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.8833586219178081, "accuracy_std_dev": 0.04650342639234633, "run_count": 17, "total_questions": 3285, "avg_cost_per_task_usd": 0.07316709345509893, "avg_duration_per_task_ms": 692682.2697108067, "avg_output_tokens_per_task": 32513.825570776255, "last_run_timestamp": "2026-08-29T20:15:38.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.8829965, "accuracy_std_dev": 0.03256836877631423, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.12880823737373737, "avg_duration_per_task_ms": 39004.51136363636, "avg_output_tokens_per_task": 2367.6944444444443, "last_run_timestamp": "2026-09-19T08:49:32.428Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.8829965, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13198183333333333, "avg_duration_per_task_ms": 254155.68181818182, "avg_output_tokens_per_task": 43876.38636363636, "last_run_timestamp": "2026-09-01T17:57:36.744Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8827861249999999, "accuracy_std_dev": 0.010063667341450408, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01960655429292929, "avg_duration_per_task_ms": 366878.014520202, "avg_output_tokens_per_task": 24088.609848484848, "last_run_timestamp": "2026-09-19T08:54:33.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.8825515716096324, "accuracy_std_dev": 0.020268991676360943, "run_count": 12, "total_questions": 2367, "avg_cost_per_task_usd": 0.1375574414871145, "avg_duration_per_task_ms": 860088.601182932, "avg_output_tokens_per_task": 38561.9945078158, "last_run_timestamp": "2026-08-29T18:18:56.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8820443201861292, "accuracy_std_dev": 0.010098014499560479, "run_count": 24, "total_questions": 4513, "avg_cost_per_task_usd": 0.13465608021271883, "avg_duration_per_task_ms": 61985.66430312431, "avg_output_tokens_per_task": 4903.735430977177, "last_run_timestamp": "2026-09-19T08:56:11.770Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.8815657186311787, "accuracy_std_dev": 0.013649359425343822, "run_count": 12, "total_questions": 2367, "avg_cost_per_task_usd": 0.10104644275454162, "avg_duration_per_task_ms": 571611.1580059147, "avg_output_tokens_per_task": 26348.45078158006, "last_run_timestamp": "2026-09-19T09:12:25.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.8813909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 163, "avg_cost_per_task_usd": 0.009804907975460122, "avg_duration_per_task_ms": 1011753.5766871165, "avg_output_tokens_per_task": 19559.061349693253, "last_run_timestamp": "2026-09-14T09:44:57.474Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.881313, "accuracy_std_dev": 0.007576000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.027364550505050506, "avg_duration_per_task_ms": 154677.17424242425, "avg_output_tokens_per_task": 22608.323232323233, "last_run_timestamp": "2026-09-19T09:33:09.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.881313, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.029419752525252522, "avg_duration_per_task_ms": 107618.89141414141, "avg_output_tokens_per_task": 24385.08585858586, "last_run_timestamp": "2026-09-19T08:52:26.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8812490225563911, "accuracy_std_dev": 0.01232224163211534, "run_count": 26, "total_questions": 5054, "avg_cost_per_task_usd": 0.17062608369608234, "avg_duration_per_task_ms": 343916.4879303522, "avg_output_tokens_per_task": 27421.04273842501, "last_run_timestamp": "2026-09-19T09:01:57.132Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.8805556673913044, "accuracy_std_dev": 0.0379554198818583, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.13285437536231884, "avg_duration_per_task_ms": 766731.7239130435, "avg_output_tokens_per_task": 33250.103623188406, "last_run_timestamp": "2026-09-19T09:21:26.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.880471125, "accuracy_std_dev": 0.005389907917522819, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.005834660353535354, "avg_duration_per_task_ms": 362869.7544191919, "avg_output_tokens_per_task": 28491.27462121212, "last_run_timestamp": "2026-09-19T08:50:07.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.8802532590673576, "accuracy_std_dev": 0.005991714074916775, "run_count": 3, "total_questions": 579, "avg_cost_per_task_usd": 0.14682200690846287, "avg_duration_per_task_ms": 338397.36614853196, "avg_output_tokens_per_task": 33348.9585492228, "last_run_timestamp": "2026-07-18T08:26:40.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8798644568527919, "accuracy_std_dev": 0.004527248931623837, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.0029938477157360403, "avg_duration_per_task_ms": 316818.90016920475, "avg_output_tokens_per_task": 18321.88663282572, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.879729960954447, "accuracy_std_dev": 0.011206573386241868, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.004548976861894432, "avg_duration_per_task_ms": 362426.65871294285, "avg_output_tokens_per_task": 18848.88069414317, "last_run_timestamp": "2026-09-19T11:00:11.912Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8796295, "accuracy_std_dev": 0.006574181641086607, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015792595959595958, "avg_duration_per_task_ms": 304804.172979798, "avg_output_tokens_per_task": 24118.43686868687, "last_run_timestamp": "2026-09-19T08:49:45.204Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8796293333333333, "accuracy_std_dev": 0.013874043218262744, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.18146528114478117, "avg_duration_per_task_ms": 307037.70454545453, "avg_output_tokens_per_task": 45544.973905723906, "last_run_timestamp": "2026-09-19T08:48:01.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8792085, "accuracy_std_dev": 0.026919662038183172, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.14250958080808082, "avg_duration_per_task_ms": 42415.96590909091, "avg_output_tokens_per_task": 2641.520202020202, "last_run_timestamp": "2026-09-19T08:47:22.067Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.879189, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.020723571428571428, "avg_duration_per_task_ms": 141095.54497354498, "avg_output_tokens_per_task": 17734.21164021164, "last_run_timestamp": "2026-09-15T00:33:26.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "WandB", "accuracy": 0.8791716692015209, "accuracy_std_dev": 0.009087349844013949, "run_count": 4, "total_questions": 789, "avg_cost_per_task_usd": 0.05804701140684411, "avg_duration_per_task_ms": 798447.8948035488, "avg_output_tokens_per_task": 16682.354879594423, "last_run_timestamp": "2026-07-11T08:42:46.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8790546552363299, "accuracy_std_dev": 0.030844451113161732, "run_count": 6, "total_questions": 1079, "avg_cost_per_task_usd": 0.1346450278035218, "avg_duration_per_task_ms": 284974.0537534754, "avg_output_tokens_per_task": 31126.302131603337, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8788689803571428, "accuracy_std_dev": 0.011915461634880886, "run_count": 17, "total_questions": 3360, "avg_cost_per_task_usd": 0.0034551410714285715, "avg_duration_per_task_ms": 175260.2726190476, "avg_output_tokens_per_task": 17584.270833333332, "last_run_timestamp": "2026-08-29T18:22:03.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13332395959595958, "avg_duration_per_task_ms": 138478.24242424243, "avg_output_tokens_per_task": 9426.934343434343, "last_run_timestamp": "2026-08-01T08:50:28.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8787875, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0075271136363636365, "avg_duration_per_task_ms": 80896.84595959596, "avg_output_tokens_per_task": 8213.79797979798, "last_run_timestamp": "2026-09-01T17:51:13.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8787801424227066, "accuracy_std_dev": 0.016401176693323544, "run_count": 11, "total_questions": 1973, "avg_cost_per_task_usd": 0.011341015205271159, "avg_duration_per_task_ms": 261740.6791687785, "avg_output_tokens_per_task": 24571.48302078054, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8785529767441861, "accuracy_std_dev": 0.101852, "run_count": 2, "total_questions": 258, "avg_cost_per_task_usd": 0.1083256472868217, "avg_duration_per_task_ms": 620596.2751937985, "avg_output_tokens_per_task": 16165.639534883721, "last_run_timestamp": "2026-09-01T16:19:34.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.8781917204968944, "accuracy_std_dev": 0.004843231342812369, "run_count": 5, "total_questions": 966, "avg_cost_per_task_usd": 0.005745680124223602, "avg_duration_per_task_ms": 442762.4254658385, "avg_output_tokens_per_task": 20256.71635610766, "last_run_timestamp": "2026-07-18T08:31:23.912Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8781237785467129, "accuracy_std_dev": 0.021132034607833364, "run_count": 9, "total_questions": 1734, "avg_cost_per_task_usd": 0.12840682410611304, "avg_duration_per_task_ms": 62804.37370242215, "avg_output_tokens_per_task": 4913.2906574394465, "last_run_timestamp": "2026-09-19T08:47:57.110Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.8779459999999999, "accuracy_std_dev": 0.014310000000000045, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13208357575757576, "avg_duration_per_task_ms": 257170.34595959596, "avg_output_tokens_per_task": 43907.46464646464, "last_run_timestamp": "2026-09-01T17:57:59.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.877945875, "accuracy_std_dev": 0.018131774468026423, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0226289154040404, "avg_duration_per_task_ms": 227816.96022727274, "avg_output_tokens_per_task": 25365.86047979798, "last_run_timestamp": "2026-09-19T09:08:32.908Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8775250000000001, "accuracy_std_dev": 0.012731145529762818, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.021682643939393943, "avg_duration_per_task_ms": 244444.21085858587, "avg_output_tokens_per_task": 24838.689393939392, "last_run_timestamp": "2026-09-19T09:00:03.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.8771930050933787, "accuracy_std_dev": 0.007637161296363126, "run_count": 3, "total_questions": 589, "avg_cost_per_task_usd": 0.007654222410865875, "avg_duration_per_task_ms": 996245.6400679117, "avg_output_tokens_per_task": 30379.964346349745, "last_run_timestamp": "2026-09-19T08:50:20.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8771045000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14050905303030303, "avg_duration_per_task_ms": 185398.66919191918, "avg_output_tokens_per_task": 46713.32070707071, "last_run_timestamp": "2026-09-01T17:54:30.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Nebius", "accuracy": 0.8771041666666666, "accuracy_std_dev": 0.010779529510707913, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0291337297979798, "avg_duration_per_task_ms": 93389.48653198653, "avg_output_tokens_per_task": 8046.875420875421, "last_run_timestamp": "2026-05-23T16:38:19.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8771039999999999, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.007407005050505051, "avg_duration_per_task_ms": 78924.21717171717, "avg_output_tokens_per_task": 8106.714646464647, "last_run_timestamp": "2026-09-01T17:50:43.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.8770844662731874, "accuracy_std_dev": 0.007890082234467962, "run_count": 18, "total_questions": 3558, "avg_cost_per_task_usd": 0.005329554525014053, "avg_duration_per_task_ms": 182892.29510961214, "avg_output_tokens_per_task": 18839.3898257448, "last_run_timestamp": "2026-08-29T19:12:25.584Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.8770211787234045, "accuracy_std_dev": 0.03195742141593335, "run_count": 6, "total_questions": 1175, "avg_cost_per_task_usd": 0.0032351353191489365, "avg_duration_per_task_ms": 354647.9063829787, "avg_output_tokens_per_task": 26251.109787234043, "last_run_timestamp": "2026-09-19T08:54:45.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.8766337242567858, "accuracy_std_dev": 0.03920192550002771, "run_count": 12, "total_questions": 2321, "avg_cost_per_task_usd": 0.0058388018095648425, "avg_duration_per_task_ms": 129752.1714778113, "avg_output_tokens_per_task": 22107.688496337785, "last_run_timestamp": "2026-09-19T08:56:52.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8765966933797908, "accuracy_std_dev": 0.020189124617118887, "run_count": 6, "total_questions": 1148, "avg_cost_per_task_usd": 0.004125730836236933, "avg_duration_per_task_ms": 453615.28397212544, "avg_output_tokens_per_task": 16054.564459930314, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8762625, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.032276419191919196, "avg_duration_per_task_ms": 263049.4646464646, "avg_output_tokens_per_task": 26729.401515151516, "last_run_timestamp": "2026-09-19T08:59:52.508Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.8761783907455013, "accuracy_std_dev": 0.007822499999999954, "run_count": 2, "total_questions": 389, "avg_cost_per_task_usd": 0.048952943444730085, "avg_duration_per_task_ms": 221005.3881748072, "avg_output_tokens_per_task": 16160.848329048844, "last_run_timestamp": "2026-09-19T08:57:27.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.876137452982811, "accuracy_std_dev": 0.011389182123401155, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.004450751263902932, "avg_duration_per_task_ms": 154446.49140546005, "avg_output_tokens_per_task": 22487.894843276037, "last_run_timestamp": "2026-09-19T09:48:52.067Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.875421, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022698737373737374, "avg_duration_per_task_ms": 167536.68686868687, "avg_output_tokens_per_task": 25022.651515151516, "last_run_timestamp": "2026-09-16T10:28:37.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.8751755374894692, "accuracy_std_dev": 0.015517134880153898, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.08454973546756529, "avg_duration_per_task_ms": 186945.5796124684, "avg_output_tokens_per_task": 21577.509688289807, "last_run_timestamp": "2026-09-19T09:33:49.568Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.875, "accuracy_std_dev": 0.012205000000000021, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.015933075757575757, "avg_duration_per_task_ms": 228129.89141414143, "avg_output_tokens_per_task": 23888.01767676768, "last_run_timestamp": "2026-09-19T10:55:25.751Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.875, "accuracy_std_dev": 0.004187691070267732, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07034678914141415, "avg_duration_per_task_ms": 185029.36994949495, "avg_output_tokens_per_task": 11452.73106060606, "last_run_timestamp": "2026-09-19T09:06:46.387Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.8745582826855123, "accuracy_std_dev": 0.047501460387912024, "run_count": 8, "total_questions": 1415, "avg_cost_per_task_usd": 0.07384584240282685, "avg_duration_per_task_ms": 519959.2473498233, "avg_output_tokens_per_task": 21893.836042402825, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.8744172980132449, "accuracy_std_dev": 0.016480704550770754, "run_count": 7, "total_questions": 1359, "avg_cost_per_task_usd": 0.005027246504782929, "avg_duration_per_task_ms": 379290.6011773363, "avg_output_tokens_per_task": 17886.896983075792, "last_run_timestamp": "2026-07-18T08:30:06.336Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.8743103404255319, "accuracy_std_dev": 0.09209312024616473, "run_count": 13, "total_questions": 2538, "avg_cost_per_task_usd": 0.02429028802206462, "avg_duration_per_task_ms": 194909.10914105596, "avg_output_tokens_per_task": 20523.450748620962, "last_run_timestamp": "2026-09-19T16:05:49.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.8740457236641223, "accuracy_std_dev": 0.03655758641704891, "run_count": 10, "total_questions": 1965, "avg_cost_per_task_usd": 0.12762344173027992, "avg_duration_per_task_ms": 372867.7043256998, "avg_output_tokens_per_task": 30124.432061068703, "last_run_timestamp": "2026-09-19T08:47:35.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8739243333333333, "accuracy_std_dev": 0.016541355096981753, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.10989428507295174, "avg_duration_per_task_ms": 217232.23681257013, "avg_output_tokens_per_task": 31249.776094276094, "last_run_timestamp": "2026-09-19T08:49:55.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.8734939514185776, "accuracy_std_dev": 0.033103140956105115, "run_count": 13, "total_questions": 2573, "avg_cost_per_task_usd": 0.15498554061406922, "avg_duration_per_task_ms": 438624.14224640495, "avg_output_tokens_per_task": 11693.160124368442, "last_run_timestamp": "2026-09-19T08:57:31.387Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.8728743269428278, "accuracy_std_dev": 0.022779113769318488, "run_count": 15, "total_questions": 2921, "avg_cost_per_task_usd": 0.060696329681615875, "avg_duration_per_task_ms": 329375.96234166384, "avg_output_tokens_per_task": 22546.172543649434, "last_run_timestamp": "2026-08-22T17:03:07.750Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.8728068947368423, "accuracy_std_dev": 0.01632482305148274, "run_count": 15, "total_questions": 2964, "avg_cost_per_task_usd": 0.004530426113360323, "avg_duration_per_task_ms": 167318.0539811066, "avg_output_tokens_per_task": 16025.914979757084, "last_run_timestamp": "2026-08-08T10:06:09.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.8727017508992805, "accuracy_std_dev": 0.010551952378424571, "run_count": 17, "total_questions": 3336, "avg_cost_per_task_usd": 0.08340600749400481, "avg_duration_per_task_ms": 697270.5386690648, "avg_output_tokens_per_task": 29384.263788968827, "last_run_timestamp": "2026-08-29T19:07:02.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.8724410619469029, "accuracy_std_dev": 0.010962697967207792, "run_count": 19, "total_questions": 3729, "avg_cost_per_task_usd": 0.0055334333601501735, "avg_duration_per_task_ms": 182246.17109144543, "avg_output_tokens_per_task": 19566.488871010995, "last_run_timestamp": "2026-08-29T18:42:44.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8721201154090044, "accuracy_std_dev": 0.010011587092433206, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.007726445783132531, "avg_duration_per_task_ms": 299427.08941027266, "avg_output_tokens_per_task": 27338.933417882054, "last_run_timestamp": "2026-09-19T08:58:27.800Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8720540000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.22419905555555555, "avg_duration_per_task_ms": 276263.6666666667, "avg_output_tokens_per_task": 22325.227272727272, "last_run_timestamp": "2026-09-01T13:30:31.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.8720024103343466, "accuracy_std_dev": 0.014437391891889622, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.0073382553191489374, "avg_duration_per_task_ms": 410218.9604863222, "avg_output_tokens_per_task": 21333.504559270517, "last_run_timestamp": "2026-09-19T08:51:18.222Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8718390517241379, "accuracy_std_dev": 0.07995783917690623, "run_count": 10, "total_questions": 1740, "avg_cost_per_task_usd": 0.009114119540229886, "avg_duration_per_task_ms": 138759.1695402299, "avg_output_tokens_per_task": 23586.4683908046, "last_run_timestamp": "2026-09-20T10:35:58.112Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.8718132857142857, "accuracy_std_dev": 0.01994854442032793, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.12823083044733047, "avg_duration_per_task_ms": 62432.78354978355, "avg_output_tokens_per_task": 4914.397546897547, "last_run_timestamp": "2026-09-19T08:48:19.751Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.8717413166815345, "accuracy_std_dev": 0.019041701395297674, "run_count": 17, "total_questions": 3363, "avg_cost_per_task_usd": 0.07283155159084152, "avg_duration_per_task_ms": 379368.3589057389, "avg_output_tokens_per_task": 21568.800475765685, "last_run_timestamp": "2026-08-29T18:39:19.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.8716215506756757, "accuracy_std_dev": 0.01729230466578475, "run_count": 18, "total_questions": 3552, "avg_cost_per_task_usd": 0.004909025900900901, "avg_duration_per_task_ms": 310490.77252252254, "avg_output_tokens_per_task": 17361.359797297297, "last_run_timestamp": "2026-08-29T18:13:33.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.871542434599156, "accuracy_std_dev": 0.007338207797953819, "run_count": 11, "total_questions": 2133, "avg_cost_per_task_usd": 0.0043919292076887015, "avg_duration_per_task_ms": 387373.37552742613, "avg_output_tokens_per_task": 23530.04781997187, "last_run_timestamp": "2026-08-29T19:32:50.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8715012977099236, "accuracy_std_dev": 0.0011400000000000299, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.04716040458015267, "avg_duration_per_task_ms": 197916.213740458, "avg_output_tokens_per_task": 39122.49618320611, "last_run_timestamp": "2026-09-19T10:12:37.404Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8714924444444443, "accuracy_std_dev": 0.016244831438618754, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.007635035353535354, "avg_duration_per_task_ms": 200792.30808080808, "avg_output_tokens_per_task": 27092.780583613916, "last_run_timestamp": "2026-09-19T08:57:19.814Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.871212, "accuracy_std_dev": 0.010943000000000036, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.25165717424242423, "avg_duration_per_task_ms": 248967.9292929293, "avg_output_tokens_per_task": 23863.48232323232, "last_run_timestamp": "2026-09-01T13:53:50.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8710768244274808, "accuracy_std_dev": 0.016176058048857262, "run_count": 4, "total_questions": 786, "avg_cost_per_task_usd": 0.009818778625954197, "avg_duration_per_task_ms": 366203.1501272265, "avg_output_tokens_per_task": 20813.51526717557, "last_run_timestamp": "2026-09-19T08:49:15.112Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.8710124489402697, "accuracy_std_dev": 0.014206989600055803, "run_count": 16, "total_questions": 3114, "avg_cost_per_task_usd": 0.005702225112395633, "avg_duration_per_task_ms": 187380.58863198457, "avg_output_tokens_per_task": 21072.890494540785, "last_run_timestamp": "2026-08-29T19:20:21.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.8710116425661915, "accuracy_std_dev": 0.01025225770257459, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.025059682281059068, "avg_duration_per_task_ms": 328330.6405295316, "avg_output_tokens_per_task": 20642.19348268839, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.8705363341869399, "accuracy_std_dev": 0.026338858318477553, "run_count": 12, "total_questions": 2343, "avg_cost_per_task_usd": 0.09737285574050364, "avg_duration_per_task_ms": 684757.7733674776, "avg_output_tokens_per_task": 25874.87921468203, "last_run_timestamp": "2026-09-19T08:51:08.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.87037, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07184554292929292, "avg_duration_per_task_ms": 295285.3813131313, "avg_output_tokens_per_task": 23811.56313131313, "last_run_timestamp": "2026-09-12T10:35:19.982Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.8700592503816793, "accuracy_std_dev": 0.011904817070833141, "run_count": 10, "total_questions": 1965, "avg_cost_per_task_usd": 0.03175990890585242, "avg_duration_per_task_ms": 495176.59541984735, "avg_output_tokens_per_task": 27384.936386768448, "last_run_timestamp": "2026-08-29T19:08:19.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.8699495, "accuracy_std_dev": 0.002104500000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00502325, "avg_duration_per_task_ms": 389434.8611111111, "avg_output_tokens_per_task": 17944.10101010101, "last_run_timestamp": "2026-08-08T10:05:07.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.8699310434782609, "accuracy_std_dev": 0.01348825804164903, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.02950167588932806, "avg_duration_per_task_ms": 201139.55204216074, "avg_output_tokens_per_task": 12529.4356609574, "last_run_timestamp": "2026-08-29T18:36:08.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.8698725770862801, "accuracy_std_dev": 0.02689058259409786, "run_count": 4, "total_questions": 707, "avg_cost_per_task_usd": 0.01008102828854314, "avg_duration_per_task_ms": 338323.4582743989, "avg_output_tokens_per_task": 19986.214992927864, "last_run_timestamp": "2026-09-19T08:51:05.252Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.8698646204481794, "accuracy_std_dev": 0.029668031806137727, "run_count": 8, "total_questions": 1428, "avg_cost_per_task_usd": 0.09692743837535013, "avg_duration_per_task_ms": 221464.25770308124, "avg_output_tokens_per_task": 26428.57492997199, "last_run_timestamp": "2026-09-19T08:45:54.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.8695285000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10538721464646464, "avg_duration_per_task_ms": 581981.0101010101, "avg_output_tokens_per_task": 34695.7398989899, "last_run_timestamp": "2026-08-08T09:03:54.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.8692809647058825, "accuracy_std_dev": 0.009104176033557385, "run_count": 4, "total_questions": 765, "avg_cost_per_task_usd": 0.0047775359477124175, "avg_duration_per_task_ms": 482494.045751634, "avg_output_tokens_per_task": 26353.58431372549, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8691077500000002, "accuracy_std_dev": 0.03719680227099502, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.11411050757575758, "avg_duration_per_task_ms": 27732.646464646463, "avg_output_tokens_per_task": 2074.2247474747473, "last_run_timestamp": "2026-09-19T08:48:16.582Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.868687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13888562626262627, "avg_duration_per_task_ms": 1028019.2020202021, "avg_output_tokens_per_task": 46184.969696969696, "last_run_timestamp": "2026-07-18T16:16:27.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.8679501874999999, "accuracy_std_dev": 0.017033291102348476, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.03806877335858586, "avg_duration_per_task_ms": 189574.66571969696, "avg_output_tokens_per_task": 10526.75946969697, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8676765999999999, "accuracy_std_dev": 0.004832456584388534, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.06976842626262626, "avg_duration_per_task_ms": 222247.99696969698, "avg_output_tokens_per_task": 11360.827272727272, "last_run_timestamp": "2026-09-19T08:56:34.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.8672080501968503, "accuracy_std_dev": 0.10319503473951104, "run_count": 12, "total_questions": 2032, "avg_cost_per_task_usd": 0.019514511811023625, "avg_duration_per_task_ms": 272320.6092519685, "avg_output_tokens_per_task": 16198, "last_run_timestamp": "2026-09-19T16:07:15.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.86700325, "accuracy_std_dev": 0.011963515314801908, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07822817297979798, "avg_duration_per_task_ms": 107799.15151515152, "avg_output_tokens_per_task": 12625.300505050505, "last_run_timestamp": "2026-09-19T08:50:00.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.8669794312026003, "accuracy_std_dev": 0.015914615486612767, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.06791193391115927, "avg_duration_per_task_ms": 196363.11339833876, "avg_output_tokens_per_task": 18755.92163235825, "last_run_timestamp": "2026-08-29T16:55:12.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.866898125, "accuracy_std_dev": 0.008034667532597418, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.006507090909090909, "avg_duration_per_task_ms": 263155.34027777775, "avg_output_tokens_per_task": 18228.347222222223, "last_run_timestamp": "2026-09-19T08:55:40.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.8667228333333332, "accuracy_std_dev": 0.006024375774486715, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0256143728956229, "avg_duration_per_task_ms": 228795.63636363635, "avg_output_tokens_per_task": 21105.421717171717, "last_run_timestamp": "2026-09-19T09:03:56.948Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.8665198811881186, "accuracy_std_dev": 0.012337540068582453, "run_count": 14, "total_questions": 2727, "avg_cost_per_task_usd": 0.14467475100843422, "avg_duration_per_task_ms": 785535.8151815182, "avg_output_tokens_per_task": 36269.02383571691, "last_run_timestamp": "2026-08-29T18:24:25.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.8664904348591549, "accuracy_std_dev": 0.01831953501771554, "run_count": 7, "total_questions": 1136, "avg_cost_per_task_usd": 0.0762857491197183, "avg_duration_per_task_ms": 481292.88996478874, "avg_output_tokens_per_task": 19103.125880281692, "last_run_timestamp": "2026-08-15T08:47:41.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.8663463077140436, "accuracy_std_dev": 0.022216806823277722, "run_count": 18, "total_questions": 3539, "avg_cost_per_task_usd": 0.08553019892625037, "avg_duration_per_task_ms": 515037.7468211359, "avg_output_tokens_per_task": 28564.16643119525, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.8663298, "accuracy_std_dev": 0.011398071405286056, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.06935939393939393, "avg_duration_per_task_ms": 219280.1888888889, "avg_output_tokens_per_task": 11290.334343434344, "last_run_timestamp": "2026-09-19T09:01:59.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Wafer", "accuracy": 0.8661615, "accuracy_std_dev": 0.012626499999999985, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0186262601010101, "avg_duration_per_task_ms": 28912.50505050505, "avg_output_tokens_per_task": 7135.919191919192, "last_run_timestamp": "2026-06-13T16:14:13.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.8660516956521741, "accuracy_std_dev": 0.012674417624678604, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.04402415107597715, "avg_duration_per_task_ms": 197822.74681598594, "avg_output_tokens_per_task": 12519.180720245937, "last_run_timestamp": "2026-08-29T18:39:14.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.8657404999999998, "accuracy_std_dev": 0.014549128315813283, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07900070454545453, "avg_duration_per_task_ms": 194604.26641414143, "avg_output_tokens_per_task": 12749.18813131313, "last_run_timestamp": "2026-09-19T08:51:22.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.8657200967741937, "accuracy_std_dev": 0.019433673652366045, "run_count": 30, "total_questions": 5177, "avg_cost_per_task_usd": 0.14950793490438477, "avg_duration_per_task_ms": 230585.83755070504, "avg_output_tokens_per_task": 27022.21209194514, "last_run_timestamp": "2026-09-19T09:13:15.177Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.86532, "accuracy_std_dev": 0.008780061799326916, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02537659898989899, "avg_duration_per_task_ms": 218871.01414141414, "avg_output_tokens_per_task": 20892.29595959596, "last_run_timestamp": "2026-09-19T08:55:39.417Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8649832, "accuracy_std_dev": 0.04797704545050687, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.10750342222222221, "avg_duration_per_task_ms": 531124.404040404, "avg_output_tokens_per_task": 27624.18686868687, "last_run_timestamp": "2026-09-19T08:57:20.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.8646244400527008, "accuracy_std_dev": 0.0070903829233687965, "run_count": 8, "total_questions": 1518, "avg_cost_per_task_usd": 0.0050142358366271405, "avg_duration_per_task_ms": 140856.23188405798, "avg_output_tokens_per_task": 17854.836627140976, "last_run_timestamp": "2026-08-08T09:25:45.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.8643639115417018, "accuracy_std_dev": 0.014757024377299852, "run_count": 18, "total_questions": 3561, "avg_cost_per_task_usd": 0.04002094074698118, "avg_duration_per_task_ms": 443712.06318449875, "avg_output_tokens_per_task": 21300.27857343443, "last_run_timestamp": "2026-08-29T19:37:04.387Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AkashML", "accuracy": 0.8643577142857142, "accuracy_std_dev": 0.0204579248543673, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07737239971139971, "avg_duration_per_task_ms": 285766.67604617606, "avg_output_tokens_per_task": 19368.568542568544, "last_run_timestamp": "2026-06-13T09:38:25.896Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.8642561170313988, "accuracy_std_dev": 0.018479984718270198, "run_count": 16, "total_questions": 3153, "avg_cost_per_task_usd": 0.06389940437678403, "avg_duration_per_task_ms": 380883.4934982556, "avg_output_tokens_per_task": 19730.762765620046, "last_run_timestamp": "2026-08-29T19:18:11.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.8641172857142856, "accuracy_std_dev": 0.03201454919570288, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10434612698412697, "avg_duration_per_task_ms": 510444.2525252525, "avg_output_tokens_per_task": 34101.318903318905, "last_run_timestamp": "2026-09-19T08:48:26.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8640852666666665, "accuracy_std_dev": 0.013018156579519569, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.003674580134680135, "avg_duration_per_task_ms": 299779.9195286195, "avg_output_tokens_per_task": 17340.830976430978, "last_run_timestamp": "2026-08-22T17:16:03.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.8638886961538461, "accuracy_std_dev": 0.011642989955736238, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.10131371474358974, "avg_duration_per_task_ms": 295529.557051282, "avg_output_tokens_per_task": 25493.882692307692, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8638342436974789, "accuracy_std_dev": 0.020982236666300873, "run_count": 11, "total_questions": 2142, "avg_cost_per_task_usd": 0.12425809803921568, "avg_duration_per_task_ms": 840368.0368814192, "avg_output_tokens_per_task": 43681.42903828198, "last_run_timestamp": "2026-08-29T19:01:27.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.8636361428571427, "accuracy_std_dev": 0.00964989106420482, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.8530851695526697, "avg_duration_per_task_ms": 482874.69696969696, "avg_output_tokens_per_task": 33932.781385281385, "last_run_timestamp": "2026-09-19T08:55:16.214Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.863636, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0938164393939394, "avg_duration_per_task_ms": 667365.3939393939, "avg_output_tokens_per_task": 23446.50505050505, "last_run_timestamp": "2026-09-10T01:44:54.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.8632135551128817, "accuracy_std_dev": 0.07509086757181263, "run_count": 4, "total_questions": 753, "avg_cost_per_task_usd": 0.12150484462151397, "avg_duration_per_task_ms": 758279.0185922974, "avg_output_tokens_per_task": 31021.026560424965, "last_run_timestamp": "2026-09-19T08:59:57.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.8631552857142857, "accuracy_std_dev": 0.0099276693094788, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.8424338816738816, "avg_duration_per_task_ms": 415013.15367965365, "avg_output_tokens_per_task": 33506.266955266954, "last_run_timestamp": "2026-09-19T08:48:50.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.8630124096311478, "accuracy_std_dev": 0.020757306507348198, "run_count": 25, "total_questions": 4880, "avg_cost_per_task_usd": 0.24381632479508195, "avg_duration_per_task_ms": 339721.7473360656, "avg_output_tokens_per_task": 44446.558196721315, "last_run_timestamp": "2026-09-19T08:58:34.250Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8626651566632756, "accuracy_std_dev": 0.015616368164205143, "run_count": 5, "total_questions": 983, "avg_cost_per_task_usd": 0.006657713123092574, "avg_duration_per_task_ms": 243535.13428280773, "avg_output_tokens_per_task": 13080.57985757884, "last_run_timestamp": "2026-09-19T09:17:42.829Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.862393223076923, "accuracy_std_dev": 0.009661955971101298, "run_count": 4, "total_questions": 780, "avg_cost_per_task_usd": 0.10223442820512822, "avg_duration_per_task_ms": 195837.56794871794, "avg_output_tokens_per_task": 26387.1858974359, "last_run_timestamp": "2026-09-19T08:47:19.823Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8623737499999999, "accuracy_std_dev": 0.003834422340783562, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07836570454545455, "avg_duration_per_task_ms": 88417.3308080808, "avg_output_tokens_per_task": 12812.738636363636, "last_run_timestamp": "2026-09-19T08:48:01.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.8623736666666667, "accuracy_std_dev": 0.019269903780824178, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.08775729166666667, "avg_duration_per_task_ms": 407569.07954545453, "avg_output_tokens_per_task": 38211.61952861953, "last_run_timestamp": "2026-09-19T09:10:50.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.8619530000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008309641414141413, "avg_duration_per_task_ms": 730613.3737373737, "avg_output_tokens_per_task": 29358.666666666668, "last_run_timestamp": "2026-07-31T04:09:58.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8619530000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023946904040404043, "avg_duration_per_task_ms": 411455.5404040404, "avg_output_tokens_per_task": 47654.88383838384, "last_run_timestamp": "2026-09-19T11:21:43.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.8619525, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.223144601010101, "avg_duration_per_task_ms": 273332.86363636365, "avg_output_tokens_per_task": 22220.545454545456, "last_run_timestamp": "2026-09-01T13:23:04.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8613915000000001, "accuracy_std_dev": 0.04603966955336814, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.004525512626262627, "avg_duration_per_task_ms": 565963.2584175084, "avg_output_tokens_per_task": 18978.082491582492, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8613799792531119, "accuracy_std_dev": 0.01086146499258382, "run_count": 11, "total_questions": 2169, "avg_cost_per_task_usd": 0.0035420834485938215, "avg_duration_per_task_ms": 187394.91378515444, "avg_output_tokens_per_task": 20972.25633932688, "last_run_timestamp": "2026-08-29T19:40:39.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8612793999999999, "accuracy_std_dev": 0.013988220381449522, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.006820060606060606, "avg_duration_per_task_ms": 354799.9131313131, "avg_output_tokens_per_task": 22824.536363636365, "last_run_timestamp": "2026-09-19T09:05:29.545Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.8612792, "accuracy_std_dev": 0.03136197207702347, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005120404040404041, "avg_duration_per_task_ms": 530889.794949495, "avg_output_tokens_per_task": 17542.636363636364, "last_run_timestamp": "2026-09-05T09:22:54.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8611109999999997, "accuracy_std_dev": 0.008672208830511384, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.8817781500721501, "avg_duration_per_task_ms": 416732.5119047619, "avg_output_tokens_per_task": 33399.694083694085, "last_run_timestamp": "2026-09-19T09:02:03.655Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.8609416158536587, "accuracy_std_dev": 0.007965914224996389, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.016509900406504064, "avg_duration_per_task_ms": 333963.73475609755, "avg_output_tokens_per_task": 32804.009146341465, "last_run_timestamp": "2026-09-19T08:48:55.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.8608576565349543, "accuracy_std_dev": 0.016168604733866183, "run_count": 15, "total_questions": 2961, "avg_cost_per_task_usd": 0.015263644376899694, "avg_duration_per_task_ms": 308595.9824383654, "avg_output_tokens_per_task": 17709.83485309017, "last_run_timestamp": "2026-08-08T09:24:32.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.860810656081081, "accuracy_std_dev": 0.017596670139546283, "run_count": 8, "total_questions": 1480, "avg_cost_per_task_usd": 0.0684703912162162, "avg_duration_per_task_ms": 339840.025, "avg_output_tokens_per_task": 20257.95, "last_run_timestamp": "2026-07-18T08:30:34.727Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.86079326201373, "accuracy_std_dev": 0.028859126736614872, "run_count": 10, "total_questions": 1748, "avg_cost_per_task_usd": 0.010031786613272312, "avg_duration_per_task_ms": 339107.3318077803, "avg_output_tokens_per_task": 18845.827231121282, "last_run_timestamp": "2026-09-19T08:55:09.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8606902500000001, "accuracy_std_dev": 0.01603212873225199, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.008977529040404041, "avg_duration_per_task_ms": 694440.691919192, "avg_output_tokens_per_task": 30908.54419191919, "last_run_timestamp": "2026-09-19T08:48:13.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.8605240633981405, "accuracy_std_dev": 0.05128936013142842, "run_count": 18, "total_questions": 3549, "avg_cost_per_task_usd": 0.00447025641025641, "avg_duration_per_task_ms": 218107.38489715412, "avg_output_tokens_per_task": 16009.49056072133, "last_run_timestamp": "2026-08-29T18:30:11.054Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8604095833333332, "accuracy_std_dev": 0.019983922335460633, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.11414123358585856, "avg_duration_per_task_ms": 818760.5311447812, "avg_output_tokens_per_task": 36533.946127946125, "last_run_timestamp": "2026-08-29T18:43:04.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.8603764494556766, "accuracy_std_dev": 0.016565195248170166, "run_count": 10, "total_questions": 1929, "avg_cost_per_task_usd": 0.10031787610160703, "avg_duration_per_task_ms": 637584.9020217729, "avg_output_tokens_per_task": 34282.6588906169, "last_run_timestamp": "2026-09-19T08:55:10.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.860269, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.20759804545454547, "avg_duration_per_task_ms": 271144.0252525252, "avg_output_tokens_per_task": 20665.12626262626, "last_run_timestamp": "2026-09-01T13:31:11.391Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8601822310030396, "accuracy_std_dev": 0.013547575585321513, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.007783161094224923, "avg_duration_per_task_ms": 526386.7112462006, "avg_output_tokens_per_task": 27793.246200607904, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8601241573604061, "accuracy_std_dev": 0.012372890590139204, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.10654743824027074, "avg_duration_per_task_ms": 370641.0829103215, "avg_output_tokens_per_task": 25074.472081218275, "last_run_timestamp": "2026-09-19T08:51:20.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.8600690598563449, "accuracy_std_dev": 0.020027284386563163, "run_count": 19, "total_questions": 3759, "avg_cost_per_task_usd": 0.05487559829741953, "avg_duration_per_task_ms": 344231.89066241024, "avg_output_tokens_per_task": 20069.8159084863, "last_run_timestamp": "2026-08-29T19:21:48.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.86000405028463, "accuracy_std_dev": 0.023944982277086784, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.04536207748260594, "avg_duration_per_task_ms": 361772.752055661, "avg_output_tokens_per_task": 22068.88678051866, "last_run_timestamp": "2026-08-29T19:11:30.664Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.8599289999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 188, "avg_cost_per_task_usd": 0.023036143617021276, "avg_duration_per_task_ms": 1599986.2287234042, "avg_output_tokens_per_task": 29232.622340425532, "last_run_timestamp": "2026-07-18T08:57:54.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.8594949584487533, "accuracy_std_dev": 0.016664788929539907, "run_count": 11, "total_questions": 2166, "avg_cost_per_task_usd": 0.14950503924284395, "avg_duration_per_task_ms": 850309.7391505078, "avg_output_tokens_per_task": 43660.16805170822, "last_run_timestamp": "2026-08-29T20:16:17.828Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.8591069047619047, "accuracy_std_dev": 0.034057120266518005, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.03376736123136124, "avg_duration_per_task_ms": 210716.60485810487, "avg_output_tokens_per_task": 11637.827561327562, "last_run_timestamp": "2026-08-15T15:55:02.752Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.8589856042216361, "accuracy_std_dev": 0.022745485097766183, "run_count": 23, "total_questions": 4548, "avg_cost_per_task_usd": 0.03739281288478453, "avg_duration_per_task_ms": 298793.5393579595, "avg_output_tokens_per_task": 12826.183597185576, "last_run_timestamp": "2026-08-29T19:19:13.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.858586, "accuracy_std_dev": 0.020201999999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.025399686868686867, "avg_duration_per_task_ms": 99978.351010101, "avg_output_tokens_per_task": 16731.46212121212, "last_run_timestamp": "2026-09-19T09:57:08.766Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8585858571428572, "accuracy_std_dev": 0.008098936198725138, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.8266230122655123, "avg_duration_per_task_ms": 471406.1378066378, "avg_output_tokens_per_task": 32893.57142857143, "last_run_timestamp": "2026-09-19T08:54:22.602Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.8583117441860466, "accuracy_std_dev": 0.020029378052937655, "run_count": 4, "total_questions": 774, "avg_cost_per_task_usd": 0.0894643488372093, "avg_duration_per_task_ms": 522454.9418604651, "avg_output_tokens_per_task": 23617.786821705427, "last_run_timestamp": "2026-09-19T09:16:39.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8581650000000001, "accuracy_std_dev": 0.016424948356083192, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.10795924494949495, "avg_duration_per_task_ms": 770514.4431818182, "avg_output_tokens_per_task": 28271.43686868687, "last_run_timestamp": "2026-09-19T08:46:12.784Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.8580918152173914, "accuracy_std_dev": 0.03107799783834439, "run_count": 17, "total_questions": 3312, "avg_cost_per_task_usd": 0.06328980283816425, "avg_duration_per_task_ms": 802747.4701086957, "avg_output_tokens_per_task": 28225.76388888889, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8576652329113923, "accuracy_std_dev": 0.03114981165807159, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.06335243122362869, "avg_duration_per_task_ms": 565982.5426160338, "avg_output_tokens_per_task": 40503.44135021097, "last_run_timestamp": "2026-09-19T08:47:25.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8575758, "accuracy_std_dev": 0.00835660171122207, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0665558909090909, "avg_duration_per_task_ms": 212021.95252525253, "avg_output_tokens_per_task": 10824.585858585859, "last_run_timestamp": "2026-09-19T08:47:49.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.8575267943548386, "accuracy_std_dev": 0.0136809305307607, "run_count": 4, "total_questions": 744, "avg_cost_per_task_usd": 0.18561700403225806, "avg_duration_per_task_ms": 513052.8830645161, "avg_output_tokens_per_task": 55552.403225806454, "last_run_timestamp": "2026-07-18T08:26:48.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.857418, "accuracy_std_dev": null, "run_count": 1, "total_questions": 173, "avg_cost_per_task_usd": 0.08778269942196533, "avg_duration_per_task_ms": 317630.5202312139, "avg_output_tokens_per_task": 36575.57225433526, "last_run_timestamp": "2026-08-04T03:12:38.954Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.8574163891625615, "accuracy_std_dev": 0.01729438534295256, "run_count": 13, "total_questions": 2436, "avg_cost_per_task_usd": 0.14581712520525447, "avg_duration_per_task_ms": 191299.41174055828, "avg_output_tokens_per_task": 26193.477422003285, "last_run_timestamp": "2026-09-19T09:04:15.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.8573686660341556, "accuracy_std_dev": 0.03152328839896587, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.09650723655913979, "avg_duration_per_task_ms": 712555.4617330803, "avg_output_tokens_per_task": 37848.16002530044, "last_run_timestamp": "2026-09-12T08:54:06.518Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.8570308061104582, "accuracy_std_dev": 0.02562810322823216, "run_count": 13, "total_questions": 2553, "avg_cost_per_task_usd": 0.07877737015276144, "avg_duration_per_task_ms": 570287.8503721112, "avg_output_tokens_per_task": 34464.188405797104, "last_run_timestamp": "2026-09-19T08:48:29.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.856844813993174, "accuracy_std_dev": 0.04300853558120186, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.08689430546075085, "avg_duration_per_task_ms": 500988.61945392494, "avg_output_tokens_per_task": 22774.62059158134, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8568008017241379, "accuracy_std_dev": 0.060206459555848316, "run_count": 4, "total_questions": 696, "avg_cost_per_task_usd": 0.011371310344827586, "avg_duration_per_task_ms": 603206.4712643678, "avg_output_tokens_per_task": 22523.2183908046, "last_run_timestamp": "2026-09-19T08:56:14.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8567280117827869, "accuracy_std_dev": 0.019208364564689006, "run_count": 10, "total_questions": 1952, "avg_cost_per_task_usd": 0.009705317622950821, "avg_duration_per_task_ms": 263176.2423155738, "avg_output_tokens_per_task": 17464.506147540982, "last_run_timestamp": "2026-08-29T18:07:13.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.8564651992882563, "accuracy_std_dev": 0.004182357947378454, "run_count": 3, "total_questions": 562, "avg_cost_per_task_usd": 0.137316371886121, "avg_duration_per_task_ms": 228809.79359430604, "avg_output_tokens_per_task": 31529.871886120996, "last_run_timestamp": "2026-09-19T08:49:57.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.856341, "accuracy_std_dev": 0.010617665907973035, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.8414909006734006, "avg_duration_per_task_ms": 451224.2822671156, "avg_output_tokens_per_task": 33486.77272727273, "last_run_timestamp": "2026-09-19T08:45:30.738Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.8562288, "accuracy_std_dev": 0.011497073373689477, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1913848090909091, "avg_duration_per_task_ms": 350329.27575757576, "avg_output_tokens_per_task": 55974.5, "last_run_timestamp": "2026-08-29T18:20:49.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.8560606556149731, "accuracy_std_dev": 0.0169669220414311, "run_count": 10, "total_questions": 1870, "avg_cost_per_task_usd": 0.046046802673796794, "avg_duration_per_task_ms": 755872.1957219251, "avg_output_tokens_per_task": 23501.898395721924, "last_run_timestamp": "2026-07-18T16:22:39.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8555779231547017, "accuracy_std_dev": 0.009816647688493245, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.016417139534883725, "avg_duration_per_task_ms": 493380.05561172904, "avg_output_tokens_per_task": 32598.366026289183, "last_run_timestamp": "2026-09-19T08:48:55.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.855499071018968, "accuracy_std_dev": 0.058276609033653, "run_count": 23, "total_questions": 4534, "avg_cost_per_task_usd": 0.04388444067048963, "avg_duration_per_task_ms": 191885.59814733127, "avg_output_tokens_per_task": 12114.957873842082, "last_run_timestamp": "2026-08-29T18:30:43.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.8554747697565143, "accuracy_std_dev": 0.03528402563356005, "run_count": 12, "total_questions": 2341, "avg_cost_per_task_usd": 0.16198979666809055, "avg_duration_per_task_ms": 293938.29090132425, "avg_output_tokens_per_task": 36709.38530542503, "last_run_timestamp": "2026-09-19T08:57:22.475Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.855459142857143, "accuracy_std_dev": 0.01202022619740304, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.7458562337662338, "avg_duration_per_task_ms": 493748.36435786437, "avg_output_tokens_per_task": 29676.530303030304, "last_run_timestamp": "2026-09-19T08:57:48.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.8552118345784417, "accuracy_std_dev": 0.013541336870486602, "run_count": 5, "total_questions": 937, "avg_cost_per_task_usd": 0.009773050160085377, "avg_duration_per_task_ms": 36346.31910352188, "avg_output_tokens_per_task": 8563.84204909285, "last_run_timestamp": "2026-08-01T18:26:19.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.8549781428571429, "accuracy_std_dev": 0.02627276912061813, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007480496392496394, "avg_duration_per_task_ms": 490043.57647907647, "avg_output_tokens_per_task": 24203.634920634922, "last_run_timestamp": "2026-09-19T08:48:10.197Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8546835012658227, "accuracy_std_dev": 0.02399739526567811, "run_count": 11, "total_questions": 1975, "avg_cost_per_task_usd": 0.12927437518987342, "avg_duration_per_task_ms": 439949.8698734177, "avg_output_tokens_per_task": 41561.01063291139, "last_run_timestamp": "2026-08-29T19:17:01.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.8546576666666666, "accuracy_std_dev": 0.005555300251911549, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.12576473232323232, "avg_duration_per_task_ms": 209920.99494949495, "avg_output_tokens_per_task": 28358.969696969696, "last_run_timestamp": "2026-09-19T08:49:27.482Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8546573333333333, "accuracy_std_dev": 0.015141104458467416, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007946281144781145, "avg_duration_per_task_ms": 637289.4747474748, "avg_output_tokens_per_task": 29143.16835016835, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8543789735234216, "accuracy_std_dev": 0.012989646298494788, "run_count": 5, "total_questions": 982, "avg_cost_per_task_usd": 0.0073387342158859475, "avg_duration_per_task_ms": 333764.4317718941, "avg_output_tokens_per_task": 21754.163951120165, "last_run_timestamp": "2026-09-19T08:47:03.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8539195888324873, "accuracy_std_dev": 0.008792242881591056, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.11491216243654823, "avg_duration_per_task_ms": 321574.34010152286, "avg_output_tokens_per_task": 29798.65143824027, "last_run_timestamp": "2026-09-19T08:46:08.188Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.853063631284916, "accuracy_std_dev": 0.013358496890952795, "run_count": 19, "total_questions": 3759, "avg_cost_per_task_usd": 0.017711467411545622, "avg_duration_per_task_ms": 155839.47140196862, "avg_output_tokens_per_task": 11900.042564511838, "last_run_timestamp": "2026-08-01T17:57:47.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8526935, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20698726515151514, "avg_duration_per_task_ms": 257611.93181818182, "avg_output_tokens_per_task": 19625.295454545456, "last_run_timestamp": "2026-09-01T13:50:27.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8525731428571429, "accuracy_std_dev": 0.020985589732198676, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.8386075396825398, "avg_duration_per_task_ms": 455420.84199134196, "avg_output_tokens_per_task": 33379.707070707074, "last_run_timestamp": "2026-09-19T08:53:57.572Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.8525641846153847, "accuracy_std_dev": 0.0069550738853012715, "run_count": 4, "total_questions": 780, "avg_cost_per_task_usd": 0.1463613051282051, "avg_duration_per_task_ms": 253560.13461538462, "avg_output_tokens_per_task": 27986.41794871795, "last_run_timestamp": "2026-09-19T10:21:04.775Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.8524682087067861, "accuracy_std_dev": 0.02973171903175686, "run_count": 12, "total_questions": 2343, "avg_cost_per_task_usd": 0.02875936875800256, "avg_duration_per_task_ms": 363978.1912078532, "avg_output_tokens_per_task": 24367.118224498507, "last_run_timestamp": "2026-08-29T19:03:08.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.8522881577868852, "accuracy_std_dev": 0.009664339658766143, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.09954490881147542, "avg_duration_per_task_ms": 487265.8411885246, "avg_output_tokens_per_task": 30959.861680327867, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.8520922857142859, "accuracy_std_dev": 0.06769645437022032, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.04623776310726311, "avg_duration_per_task_ms": 201066.01322751323, "avg_output_tokens_per_task": 13148.871332371333, "last_run_timestamp": "2026-08-29T20:00:02.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.85189953626943, "accuracy_std_dev": 0.030767744842407633, "run_count": 4, "total_questions": 772, "avg_cost_per_task_usd": 0.137902896373057, "avg_duration_per_task_ms": 375133.5362694301, "avg_output_tokens_per_task": 31541, "last_run_timestamp": "2026-09-19T08:53:57.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.851852, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.16585097979797978, "avg_duration_per_task_ms": 389107.60101010103, "avg_output_tokens_per_task": 37536.631313131315, "last_run_timestamp": "2026-08-04T03:12:10.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.8517631052631578, "accuracy_std_dev": 0.01675925147146067, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.013288521265284426, "avg_duration_per_task_ms": 130487.94444444444, "avg_output_tokens_per_task": 10986.221956406167, "last_run_timestamp": "2026-08-01T17:57:21.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8516789626604435, "accuracy_std_dev": 0.034038116974527216, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.05071204317386231, "avg_duration_per_task_ms": 213874.00194476856, "avg_output_tokens_per_task": 21039.041618047453, "last_run_timestamp": "2026-09-19T09:37:29.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8515202907075875, "accuracy_std_dev": 0.012396151064835489, "run_count": 6, "total_questions": 1173, "avg_cost_per_task_usd": 0.12419511594202898, "avg_duration_per_task_ms": 467009.2557544757, "avg_output_tokens_per_task": 31591.142369991474, "last_run_timestamp": "2026-09-19T09:07:57.675Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.8513301536491676, "accuracy_std_dev": 0.05979411713817116, "run_count": 12, "total_questions": 2343, "avg_cost_per_task_usd": 0.06459188518992744, "avg_duration_per_task_ms": 616134.8638497653, "avg_output_tokens_per_task": 36840.9726845924, "last_run_timestamp": "2026-09-19T09:39:16.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8512884334600759, "accuracy_std_dev": 0.0436746758115393, "run_count": 12, "total_questions": 2367, "avg_cost_per_task_usd": 0.11391814955640052, "avg_duration_per_task_ms": 499191.1106886354, "avg_output_tokens_per_task": 32549.244613434726, "last_run_timestamp": "2026-09-19T08:49:34.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8509483597560974, "accuracy_std_dev": 0.02777727136059266, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.10137330792682926, "avg_duration_per_task_ms": 326460.5447154472, "avg_output_tokens_per_task": 22810.232723577235, "last_run_timestamp": "2026-09-19T08:57:56.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8508814908551657, "accuracy_std_dev": 0.01345103975540779, "run_count": 31, "total_questions": 6069, "avg_cost_per_task_usd": 0.03916498055692866, "avg_duration_per_task_ms": 432081.76124567474, "avg_output_tokens_per_task": 22652.797495468774, "last_run_timestamp": "2026-08-29T19:16:17.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.8505806741803278, "accuracy_std_dev": 0.015646816333043635, "run_count": 5, "total_questions": 976, "avg_cost_per_task_usd": 0.017172275614754096, "avg_duration_per_task_ms": 299428.9518442623, "avg_output_tokens_per_task": 34120.667008196724, "last_run_timestamp": "2026-09-19T08:49:36.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.8504337702909648, "accuracy_std_dev": 0.025173874720431894, "run_count": 10, "total_questions": 1959, "avg_cost_per_task_usd": 0.05186597600816743, "avg_duration_per_task_ms": 525779.0051046453, "avg_output_tokens_per_task": 20693.85911179173, "last_run_timestamp": "2026-08-01T17:46:48.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.8502667601600985, "accuracy_std_dev": 0.030732490705134455, "run_count": 17, "total_questions": 3248, "avg_cost_per_task_usd": 0.10197107050492611, "avg_duration_per_task_ms": 818392.5018472907, "avg_output_tokens_per_task": 31384.871613300493, "last_run_timestamp": "2026-08-29T18:13:11.491Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.8501585778210115, "accuracy_std_dev": 0.036849849445430474, "run_count": 27, "total_questions": 4626, "avg_cost_per_task_usd": 0.010481716817985303, "avg_duration_per_task_ms": 412689.4900562041, "avg_output_tokens_per_task": 15437.374621703415, "last_run_timestamp": "2026-09-19T08:48:53.177Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.8499811111111111, "accuracy_std_dev": 0.015580772491506915, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.7504352328843996, "avg_duration_per_task_ms": 500085.27890011226, "avg_output_tokens_per_task": 29943.05948372615, "last_run_timestamp": "2026-09-19T09:04:24.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.8499715520304567, "accuracy_std_dev": 0.014031811236486494, "run_count": 12, "total_questions": 2364, "avg_cost_per_task_usd": 0.027818640862944163, "avg_duration_per_task_ms": 497777.6065989848, "avg_output_tokens_per_task": 25887.815143824027, "last_run_timestamp": "2026-08-29T19:21:17.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.8497797692307693, "accuracy_std_dev": 0.008887516934129454, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.0686196585081585, "avg_duration_per_task_ms": 218386.79564879564, "avg_output_tokens_per_task": 21351.092074592074, "last_run_timestamp": "2026-09-19T09:06:57.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.8495459182879378, "accuracy_std_dev": 0.02771513034399036, "run_count": 8, "total_questions": 1542, "avg_cost_per_task_usd": 0.028309669909208815, "avg_duration_per_task_ms": 547916.742542153, "avg_output_tokens_per_task": 27038.87094682231, "last_run_timestamp": "2026-09-19T08:49:45.084Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.8490153107221006, "accuracy_std_dev": 0.02104665580299624, "run_count": 7, "total_questions": 1371, "avg_cost_per_task_usd": 0.04150975711159738, "avg_duration_per_task_ms": 669606.0547045952, "avg_output_tokens_per_task": 34749.842450765864, "last_run_timestamp": "2026-08-29T19:20:22.044Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.8488110578734859, "accuracy_std_dev": 0.0374690880393425, "run_count": 4, "total_questions": 743, "avg_cost_per_task_usd": 0.12476096096904442, "avg_duration_per_task_ms": 559359.8277254375, "avg_output_tokens_per_task": 33151.71467025572, "last_run_timestamp": "2026-09-19T08:53:56.961Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8487651666666665, "accuracy_std_dev": 0.03381563814971148, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.8327803324915825, "avg_duration_per_task_ms": 429212.7272727273, "avg_output_tokens_per_task": 33275.729797979795, "last_run_timestamp": "2026-09-19T08:47:59.678Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8487252142857141, "accuracy_std_dev": 0.017377716902395323, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.7963357799422798, "avg_duration_per_task_ms": 477422.2904040404, "avg_output_tokens_per_task": 30185.76370851371, "last_run_timestamp": "2026-09-19T08:57:19.905Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.8486816237288136, "accuracy_std_dev": 0.03080583684772884, "run_count": 18, "total_questions": 3540, "avg_cost_per_task_usd": 0.21800511779661022, "avg_duration_per_task_ms": 505891.4338983051, "avg_output_tokens_per_task": 41719.584463276835, "last_run_timestamp": "2026-09-19T08:54:22.411Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.20520298484848487, "avg_duration_per_task_ms": 278139.5202020202, "avg_output_tokens_per_task": 20422.565656565657, "last_run_timestamp": "2026-09-01T13:34:32.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.8484846666666668, "accuracy_std_dev": 0.022121414381142606, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1076985303030303, "avg_duration_per_task_ms": 498790.32828282827, "avg_output_tokens_per_task": 31614.96127946128, "last_run_timestamp": "2026-08-29T19:12:34.610Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8483078331388565, "accuracy_std_dev": 0.009457638031564467, "run_count": 13, "total_questions": 2571, "avg_cost_per_task_usd": 0.045122924542979385, "avg_duration_per_task_ms": 260125.00777907428, "avg_output_tokens_per_task": 41250.62932711007, "last_run_timestamp": "2026-09-19T16:06:22.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.8481442625127681, "accuracy_std_dev": 0.03159535831352445, "run_count": 15, "total_questions": 2937, "avg_cost_per_task_usd": 0.07499461457269323, "avg_duration_per_task_ms": 825143.8052434457, "avg_output_tokens_per_task": 32030.652706843717, "last_run_timestamp": "2026-08-29T18:42:20.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.8481386725888325, "accuracy_std_dev": 0.07204977338092783, "run_count": 12, "total_questions": 2364, "avg_cost_per_task_usd": 0.15121076776649744, "avg_duration_per_task_ms": 469840.51480541454, "avg_output_tokens_per_task": 34451.947969543144, "last_run_timestamp": "2026-09-19T08:48:42.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.8481244738853504, "accuracy_std_dev": 0.01045396470643235, "run_count": 12, "total_questions": 2355, "avg_cost_per_task_usd": 0.025721170276008492, "avg_duration_per_task_ms": 552477.8963906582, "avg_output_tokens_per_task": 26613.83906581741, "last_run_timestamp": "2026-09-19T09:19:53.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Together", "accuracy": 0.8481239999999999, "accuracy_std_dev": 0.019416861332357486, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.0265733189033189, "avg_duration_per_task_ms": 50796.17424242424, "avg_output_tokens_per_task": 7342.486652236652, "last_run_timestamp": "2026-06-20T16:15:59.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.848103465060241, "accuracy_std_dev": 0.01673605276986749, "run_count": 19, "total_questions": 3735, "avg_cost_per_task_usd": 0.017738112449799195, "avg_duration_per_task_ms": 321964.4128514056, "avg_output_tokens_per_task": 14661.384471218205, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8480308582278481, "accuracy_std_dev": 0.00841927700239488, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.035698729535864976, "avg_duration_per_task_ms": 353629.90632911393, "avg_output_tokens_per_task": 36124.08481012658, "last_run_timestamp": "2026-09-19T16:18:47.983Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.8480290637966805, "accuracy_std_dev": 0.06331280368811666, "run_count": 10, "total_questions": 1928, "avg_cost_per_task_usd": 0.05295942946058091, "avg_duration_per_task_ms": 591223.8205394191, "avg_output_tokens_per_task": 18578.95072614108, "last_run_timestamp": "2026-09-12T09:19:57.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.8480035714285714, "accuracy_std_dev": 0.016064940156903722, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.7519250108225108, "avg_duration_per_task_ms": 501563.3658008658, "avg_output_tokens_per_task": 29978.645743145742, "last_run_timestamp": "2026-09-19T08:50:15.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.8477077692307692, "accuracy_std_dev": 0.03663217256656703, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.0030205528360528365, "avg_duration_per_task_ms": 496322.17443667445, "avg_output_tokens_per_task": 11617.942113442114, "last_run_timestamp": "2026-07-18T08:42:46.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.847642875, "accuracy_std_dev": 0.03659437500640468, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.17585300694444442, "avg_duration_per_task_ms": 44732.63005050505, "avg_output_tokens_per_task": 3313.816919191919, "last_run_timestamp": "2026-09-19T08:59:16.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.8474537731821213, "accuracy_std_dev": 0.016586662965662614, "run_count": 23, "total_questions": 4497, "avg_cost_per_task_usd": 0.048555820102290405, "avg_duration_per_task_ms": 531371.0297976428, "avg_output_tokens_per_task": 24551.97487213698, "last_run_timestamp": "2026-08-30T00:19:02.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.8471111408, "accuracy_std_dev": 0.011562823272593973, "run_count": 19, "total_questions": 3750, "avg_cost_per_task_usd": 0.011580457866666668, "avg_duration_per_task_ms": 318015.93173333333, "avg_output_tokens_per_task": 13008.763733333333, "last_run_timestamp": "2026-08-15T14:57:08.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.8469509168474333, "accuracy_std_dev": 0.01813136080720607, "run_count": 21, "total_questions": 4149, "avg_cost_per_task_usd": 0.01589632562063148, "avg_duration_per_task_ms": 154447.76644974694, "avg_output_tokens_per_task": 13181.300795372379, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8468012500000001, "accuracy_std_dev": 0.031201721542047957, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.11716699116161616, "avg_duration_per_task_ms": 561187.7866161616, "avg_output_tokens_per_task": 31215.503787878788, "last_run_timestamp": "2026-08-29T18:26:17.965Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.8466981191037736, "accuracy_std_dev": 0.02482932381358783, "run_count": 5, "total_questions": 848, "avg_cost_per_task_usd": 0.009916996462264151, "avg_duration_per_task_ms": 317031.18396226416, "avg_output_tokens_per_task": 19623.935141509435, "last_run_timestamp": "2026-09-19T08:58:01.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.8465876167512689, "accuracy_std_dev": 0.014084137220399971, "run_count": 12, "total_questions": 2364, "avg_cost_per_task_usd": 0.10131879653130285, "avg_duration_per_task_ms": 853989.2901861252, "avg_output_tokens_per_task": 31653.363790186126, "last_run_timestamp": "2026-09-19T17:02:15.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8464216922102598, "accuracy_std_dev": 0.016722260700560206, "run_count": 8, "total_questions": 1579, "avg_cost_per_task_usd": 0.08943012222925902, "avg_duration_per_task_ms": 420229.20645978465, "avg_output_tokens_per_task": 22253.619379354022, "last_run_timestamp": "2026-09-19T08:55:13.302Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8462402666666666, "accuracy_std_dev": 0.020217635616681023, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.15834877643097642, "avg_duration_per_task_ms": 154755.59292929294, "avg_output_tokens_per_task": 15633.021548821549, "last_run_timestamp": "2026-09-19T08:49:28.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8459593333333334, "accuracy_std_dev": 0.04023697333989667, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1740725378787879, "avg_duration_per_task_ms": 45559.39730639731, "avg_output_tokens_per_task": 3286.5547138047136, "last_run_timestamp": "2026-09-19T08:50:24.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.8459295011933173, "accuracy_std_dev": 0.01249384613206028, "run_count": 13, "total_questions": 2514, "avg_cost_per_task_usd": 0.023641319013524262, "avg_duration_per_task_ms": 567109.6607000795, "avg_output_tokens_per_task": 26995.523468575975, "last_run_timestamp": "2026-09-19T09:00:43.108Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "WandB", "accuracy": 0.8458599302325581, "accuracy_std_dev": 0.025858269701844596, "run_count": 15, "total_questions": 2967, "avg_cost_per_task_usd": 0.012189807549713516, "avg_duration_per_task_ms": 182654.23222109876, "avg_output_tokens_per_task": 10138.908324907314, "last_run_timestamp": "2026-07-18T16:18:27.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8452890331905781, "accuracy_std_dev": 0.011290865996902098, "run_count": 5, "total_questions": 934, "avg_cost_per_task_usd": 0.01568154282655246, "avg_duration_per_task_ms": 212340.7505353319, "avg_output_tokens_per_task": 31157.209850107065, "last_run_timestamp": "2026-09-19T08:57:46.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.8451892474226804, "accuracy_std_dev": 0.006483228158872687, "run_count": 5, "total_questions": 970, "avg_cost_per_task_usd": 0.007323164948453608, "avg_duration_per_task_ms": 502972.8824742268, "avg_output_tokens_per_task": 18045.067010309278, "last_run_timestamp": "2026-08-29T18:17:30.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.845117875, "accuracy_std_dev": 0.04165596517738336, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.17059957196969697, "avg_duration_per_task_ms": 43267.32512626263, "avg_output_tokens_per_task": 3208.438762626263, "last_run_timestamp": "2026-09-19T08:45:37.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.8443808801141771, "accuracy_std_dev": 0.0190832292537191, "run_count": 16, "total_questions": 3153, "avg_cost_per_task_usd": 0.02911775134792261, "avg_duration_per_task_ms": 432929.5179194418, "avg_output_tokens_per_task": 23367.47732318427, "last_run_timestamp": "2026-08-29T19:06:57.766Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.8441076, "accuracy_std_dev": 0.01595676654714231, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.10875679292929294, "avg_duration_per_task_ms": 315962.43434343435, "avg_output_tokens_per_task": 27643.872727272726, "last_run_timestamp": "2026-09-19T08:54:22.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.843887868147448, "accuracy_std_dev": 0.023115623631383364, "run_count": 11, "total_questions": 2116, "avg_cost_per_task_usd": 0.14197860255198488, "avg_duration_per_task_ms": 310798.73534971644, "avg_output_tokens_per_task": 37568.326559546316, "last_run_timestamp": "2026-09-19T08:48:08.180Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.843855, "accuracy_std_dev": 0.020699899528741687, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.019635045454545454, "avg_duration_per_task_ms": 388426.77146464644, "avg_output_tokens_per_task": 35270.70202020202, "last_run_timestamp": "2026-09-19T08:58:24.996Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "ModelRun", "accuracy": 0.8435248151724137, "accuracy_std_dev": 0.047063095761904726, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.04366690390804597, "avg_duration_per_task_ms": 260610.5043678161, "avg_output_tokens_per_task": 21718.556781609197, "last_run_timestamp": "2026-07-11T08:27:26.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8435148007968127, "accuracy_std_dev": 0.009824077042964072, "run_count": 16, "total_questions": 3012, "avg_cost_per_task_usd": 0.0045806557104913674, "avg_duration_per_task_ms": 254608.46049136785, "avg_output_tokens_per_task": 11284.547476759628, "last_run_timestamp": "2026-08-29T18:39:38.845Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Baidu", "accuracy": 0.8434342499999999, "accuracy_std_dev": 0.01988383838541994, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00799521717171717, "avg_duration_per_task_ms": 74649.10037878787, "avg_output_tokens_per_task": 7345.006313131313, "last_run_timestamp": "2026-06-06T16:35:15.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.843434, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02832048484848485, "avg_duration_per_task_ms": 290952.4191919192, "avg_output_tokens_per_task": 15039.919191919193, "last_run_timestamp": "2026-09-19T08:54:33.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.8434186912820514, "accuracy_std_dev": 0.048959906763890886, "run_count": 15, "total_questions": 2925, "avg_cost_per_task_usd": 0.12929724376068377, "avg_duration_per_task_ms": 178496.35042735044, "avg_output_tokens_per_task": 32231.285811965812, "last_run_timestamp": "2026-07-18T08:26:40.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.8431370045248868, "accuracy_std_dev": 0.004260933614280586, "run_count": 7, "total_questions": 1326, "avg_cost_per_task_usd": 0.039398003770739064, "avg_duration_per_task_ms": 50754.730015082954, "avg_output_tokens_per_task": 14781.932126696833, "last_run_timestamp": "2026-09-01T17:50:42.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.8430610700389105, "accuracy_std_dev": 0.010186499999999987, "run_count": 2, "total_questions": 257, "avg_cost_per_task_usd": 0.03866427237354086, "avg_duration_per_task_ms": 140782.22178988328, "avg_output_tokens_per_task": 15417.210116731518, "last_run_timestamp": "2026-09-19T08:50:06.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8427455454545453, "accuracy_std_dev": 0.01932638720348021, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.16901573622589527, "avg_duration_per_task_ms": 157077.36363636365, "avg_output_tokens_per_task": 15869.985537190083, "last_run_timestamp": "2026-09-19T08:55:16.112Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8426068009188362, "accuracy_std_dev": 0.022699620798815105, "run_count": 10, "total_questions": 1959, "avg_cost_per_task_usd": 0.12467511332312406, "avg_duration_per_task_ms": 432582.16334864724, "avg_output_tokens_per_task": 32390.03164880041, "last_run_timestamp": "2026-09-19T08:49:25.281Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.8425974369531652, "accuracy_std_dev": 0.020837639567858918, "run_count": 10, "total_questions": 1943, "avg_cost_per_task_usd": 0.18937706690684508, "avg_duration_per_task_ms": 350006.53216675244, "avg_output_tokens_per_task": 43348.177560473494, "last_run_timestamp": "2026-09-19T08:51:18.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.8425927500000001, "accuracy_std_dev": 0.014939580472272304, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.10931991035353535, "avg_duration_per_task_ms": 266096.30303030304, "avg_output_tokens_per_task": 24660.041666666668, "last_run_timestamp": "2026-09-12T08:53:18.149Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.8425925, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06301305808080808, "avg_duration_per_task_ms": 90351.56313131313, "avg_output_tokens_per_task": 13876.954545454546, "last_run_timestamp": "2026-09-01T17:51:50.923Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8425808221092757, "accuracy_std_dev": 0.0344104863252307, "run_count": 12, "total_questions": 2361, "avg_cost_per_task_usd": 0.17626905887335875, "avg_duration_per_task_ms": 323040.35662854725, "avg_output_tokens_per_task": 40000.04659042779, "last_run_timestamp": "2026-09-19T09:14:29.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.8423992077798862, "accuracy_std_dev": 0.022061600294686224, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.1114357827324478, "avg_duration_per_task_ms": 504012.30803289055, "avg_output_tokens_per_task": 34670.05819101835, "last_run_timestamp": "2026-08-29T20:19:40.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.8415372905296948, "accuracy_std_dev": 0.0365919096817196, "run_count": 19, "total_questions": 3738, "avg_cost_per_task_usd": 0.0653075548421616, "avg_duration_per_task_ms": 587777.9748528625, "avg_output_tokens_per_task": 26128.800428036382, "last_run_timestamp": "2026-08-29T18:40:35.044Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.8414075126536513, "accuracy_std_dev": 0.022108573391159597, "run_count": 21, "total_questions": 4149, "avg_cost_per_task_usd": 0.0148124528802121, "avg_duration_per_task_ms": 317232.769824054, "avg_output_tokens_per_task": 14106.06435285611, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.8411896666666666, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.140150531986532, "avg_duration_per_task_ms": 165547.89562289562, "avg_output_tokens_per_task": 31680.003367003366, "last_run_timestamp": "2026-09-19T09:01:25.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DekaLLM", "accuracy": 0.8410117804878049, "accuracy_std_dev": 0.0033504999999999785, "run_count": 2, "total_questions": 369, "avg_cost_per_task_usd": 0.07247923035230353, "avg_duration_per_task_ms": 402624.47967479675, "avg_output_tokens_per_task": 25758.56639566396, "last_run_timestamp": "2026-07-04T09:58:40.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.840988087912088, "accuracy_std_dev": 0.024478415329178844, "run_count": 18, "total_questions": 3549, "avg_cost_per_task_usd": 0.12527278360101435, "avg_duration_per_task_ms": 593414.8410819949, "avg_output_tokens_per_task": 31740.369118061426, "last_run_timestamp": "2026-08-29T18:39:39.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "SiliconFlow", "accuracy": 0.8409090999999999, "accuracy_std_dev": 0.016326424363282965, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.029859258080808075, "avg_duration_per_task_ms": 210807.68282828282, "avg_output_tokens_per_task": 13522.216161616161, "last_run_timestamp": "2026-06-06T17:25:31.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.8408720981210859, "accuracy_std_dev": 0.02022286087457656, "run_count": 22, "total_questions": 4311, "avg_cost_per_task_usd": 0.05355351844119695, "avg_duration_per_task_ms": 553888.6455578752, "avg_output_tokens_per_task": 24570.292275574113, "last_run_timestamp": "2026-08-30T00:28:09.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8408546584615385, "accuracy_std_dev": 0.004112945970955614, "run_count": 5, "total_questions": 975, "avg_cost_per_task_usd": 0.09125773538461539, "avg_duration_per_task_ms": 289113.2892307692, "avg_output_tokens_per_task": 22386.701538461537, "last_run_timestamp": "2026-09-19T08:59:39.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8405005064184852, "accuracy_std_dev": 0.014666451484675618, "run_count": 16, "total_questions": 3116, "avg_cost_per_task_usd": 0.14646416527599487, "avg_duration_per_task_ms": 284641.20763799746, "avg_output_tokens_per_task": 33175.712451861364, "last_run_timestamp": "2026-08-29T19:00:02.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.8402252180451129, "accuracy_std_dev": 0.006276000000000004, "run_count": 2, "total_questions": 266, "avg_cost_per_task_usd": 0.00410540977443609, "avg_duration_per_task_ms": 368590.0263157895, "avg_output_tokens_per_task": 8011.255639097744, "last_run_timestamp": "2026-09-19T08:47:33.865Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.8400309335394126, "accuracy_std_dev": 0.029718375539176087, "run_count": 4, "total_questions": 647, "avg_cost_per_task_usd": 0.07242874188562597, "avg_duration_per_task_ms": 447585.9366306028, "avg_output_tokens_per_task": 23737.97372488408, "last_run_timestamp": "2026-09-19T08:52:28.200Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8398567499999999, "accuracy_std_dev": 0.038088117043738205, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.17395044507575758, "avg_duration_per_task_ms": 45323.468434343435, "avg_output_tokens_per_task": 3273.729797979798, "last_run_timestamp": "2026-09-19T08:52:27.050Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.839856625, "accuracy_std_dev": 0.02003363270813297, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.15796164583333333, "avg_duration_per_task_ms": 153579.38446969696, "avg_output_tokens_per_task": 15584.020833333334, "last_run_timestamp": "2026-09-19T08:49:06.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.8397368503538927, "accuracy_std_dev": 0.015255152920898565, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.12783742972699697, "avg_duration_per_task_ms": 159433.03235591506, "avg_output_tokens_per_task": 28836.38321536906, "last_run_timestamp": "2026-09-19T09:00:01.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.839052991845056, "accuracy_std_dev": 0.019945151183070924, "run_count": 15, "total_questions": 2943, "avg_cost_per_task_usd": 0.053893335372069334, "avg_duration_per_task_ms": 628913.658851512, "avg_output_tokens_per_task": 30616.803601766904, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.83880475, "accuracy_std_dev": 0.009249480644203778, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.07424993055555555, "avg_duration_per_task_ms": 281356.54671717173, "avg_output_tokens_per_task": 20276.51388888889, "last_run_timestamp": "2026-09-19T08:49:08.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.8386599858635525, "accuracy_std_dev": 0.19405393544087168, "run_count": 10, "total_questions": 1627, "avg_cost_per_task_usd": 0.14442266195451753, "avg_duration_per_task_ms": 280013.19237861095, "avg_output_tokens_per_task": 11154.37738168408, "last_run_timestamp": "2026-09-20T10:39:59.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8384512456608813, "accuracy_std_dev": 0.021133434301110997, "run_count": 8, "total_questions": 1498, "avg_cost_per_task_usd": 0.0535803865153538, "avg_duration_per_task_ms": 282678.4526034713, "avg_output_tokens_per_task": 22384.684245660883, "last_run_timestamp": "2026-09-19T10:08:51.605Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.8384059735973598, "accuracy_std_dev": 0.03372879005412596, "run_count": 14, "total_questions": 2727, "avg_cost_per_task_usd": 0.05032389952328565, "avg_duration_per_task_ms": 247684.96699669966, "avg_output_tokens_per_task": 22104.830583058305, "last_run_timestamp": "2026-09-19T09:24:21.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Parasail", "accuracy": 0.8383835714285715, "accuracy_std_dev": 0.009733641278093865, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.031499565656565656, "avg_duration_per_task_ms": 532165.215007215, "avg_output_tokens_per_task": 15048.665223665224, "last_run_timestamp": "2026-06-06T16:02:18.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.8382395753791257, "accuracy_std_dev": 0.015894466289989074, "run_count": 17, "total_questions": 3363, "avg_cost_per_task_usd": 0.016586219744275946, "avg_duration_per_task_ms": 262022.0906928338, "avg_output_tokens_per_task": 15243.496580434135, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.837957142668428, "accuracy_std_dev": 0.006945026007870667, "run_count": 4, "total_questions": 757, "avg_cost_per_task_usd": 0.019369532364597094, "avg_duration_per_task_ms": 59399.14531043593, "avg_output_tokens_per_task": 15971.39894319683, "last_run_timestamp": "2026-09-19T09:12:31.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8378685034013607, "accuracy_std_dev": 0.007616806900532539, "run_count": 5, "total_questions": 882, "avg_cost_per_task_usd": 0.018575892290249432, "avg_duration_per_task_ms": 593585.7845804988, "avg_output_tokens_per_task": 36956.14739229025, "last_run_timestamp": "2026-09-19T08:46:08.149Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.8377076351706036, "accuracy_std_dev": 0.022782891525221304, "run_count": 4, "total_questions": 762, "avg_cost_per_task_usd": 0.01399598031496063, "avg_duration_per_task_ms": 689440.656167979, "avg_output_tokens_per_task": 33978.16535433071, "last_run_timestamp": "2026-09-12T08:53:51.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8376065934065935, "accuracy_std_dev": 0.00571412028808169, "run_count": 3, "total_questions": 546, "avg_cost_per_task_usd": 0.036677783882783886, "avg_duration_per_task_ms": 47951.65201465201, "avg_output_tokens_per_task": 14583.362637362638, "last_run_timestamp": "2026-09-01T17:50:52.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8375420000000001, "accuracy_std_dev": 0.005646444323642987, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.08216196843434342, "avg_duration_per_task_ms": 323315.19444444444, "avg_output_tokens_per_task": 19594.77398989899, "last_run_timestamp": "2026-09-19T08:47:56.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.8374022479391249, "accuracy_std_dev": 0.08262800793142024, "run_count": 16, "total_questions": 3154, "avg_cost_per_task_usd": 0.09566040615091946, "avg_duration_per_task_ms": 471332.48065948003, "avg_output_tokens_per_task": 29376.78376664553, "last_run_timestamp": "2026-08-29T18:23:09.250Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8371893307626392, "accuracy_std_dev": 0.0077992202671491664, "run_count": 7, "total_questions": 1167, "avg_cost_per_task_usd": 0.01691520051413882, "avg_duration_per_task_ms": 189610.05741216795, "avg_output_tokens_per_task": 33640.42673521851, "last_run_timestamp": "2026-09-19T09:05:59.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8371357823193916, "accuracy_std_dev": 0.009460368146759528, "run_count": 6, "total_questions": 1052, "avg_cost_per_task_usd": 0.08486342680608365, "avg_duration_per_task_ms": 599922.150190114, "avg_output_tokens_per_task": 21002.96958174905, "last_run_timestamp": "2026-09-19T08:48:08.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.8369750858164482, "accuracy_std_dev": 0.010022153871541413, "run_count": 13, "total_questions": 2517, "avg_cost_per_task_usd": 0.06672872864521255, "avg_duration_per_task_ms": 709736.1537544696, "avg_output_tokens_per_task": 40652.02900278109, "last_run_timestamp": "2026-09-19T16:00:24.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.8368604243386243, "accuracy_std_dev": 0.053504834431884106, "run_count": 6, "total_questions": 945, "avg_cost_per_task_usd": 0.05630135555555556, "avg_duration_per_task_ms": 268944.617989418, "avg_output_tokens_per_task": 18648.48253968254, "last_run_timestamp": "2026-08-15T15:24:07.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8367553969581748, "accuracy_std_dev": 0.016143329277057448, "run_count": 20, "total_questions": 3945, "avg_cost_per_task_usd": 0.04780176628643852, "avg_duration_per_task_ms": 445163.75487959443, "avg_output_tokens_per_task": 25765.407351077312, "last_run_timestamp": "2026-08-22T17:01:35.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.8367000000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06203523232323233, "avg_duration_per_task_ms": 92697.9696969697, "avg_output_tokens_per_task": 13658.787878787878, "last_run_timestamp": "2026-09-01T17:49:43.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.83627925, "accuracy_std_dev": 0.037120416716242575, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015161977272727275, "avg_duration_per_task_ms": 425327.9482323232, "avg_output_tokens_per_task": 27500.13005050505, "last_run_timestamp": "2026-08-29T19:58:27.585Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.8352059691011233, "accuracy_std_dev": 0.03754617901216079, "run_count": 11, "total_questions": 2136, "avg_cost_per_task_usd": 0.14707353417602997, "avg_duration_per_task_ms": 365510.97191011236, "avg_output_tokens_per_task": 33578.668539325845, "last_run_timestamp": "2026-08-15T08:57:45.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.835017, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.036737896464646466, "avg_duration_per_task_ms": 49536.31060606061, "avg_output_tokens_per_task": 14606.348484848484, "last_run_timestamp": "2026-09-01T17:49:40.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.8350164999999999, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.16845593434343434, "avg_duration_per_task_ms": 341569.6590909091, "avg_output_tokens_per_task": 41205.416666666664, "last_run_timestamp": "2026-09-19T08:58:05.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.8349114289772727, "accuracy_std_dev": 0.012684612209645646, "run_count": 11, "total_questions": 2112, "avg_cost_per_task_usd": 0.09840011221590908, "avg_duration_per_task_ms": 269461.8380681818, "avg_output_tokens_per_task": 24744.351799242424, "last_run_timestamp": "2026-07-18T08:29:37.313Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8348017092511013, "accuracy_std_dev": 0.013128455115293883, "run_count": 7, "total_questions": 1362, "avg_cost_per_task_usd": 0.03501912334801763, "avg_duration_per_task_ms": 566392.0389133627, "avg_output_tokens_per_task": 32664.345814977973, "last_run_timestamp": "2026-08-29T18:47:45.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SambaNova", "accuracy": 0.8345957500000001, "accuracy_std_dev": 0.014451436422982317, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0074238952020202024, "avg_duration_per_task_ms": 38293.18308080808, "avg_output_tokens_per_task": 7280.891414141414, "last_run_timestamp": "2026-05-16T16:15:42.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.8344652224938874, "accuracy_std_dev": 0.01975287567409165, "run_count": 19, "total_questions": 3681, "avg_cost_per_task_usd": 0.003750411572942135, "avg_duration_per_task_ms": 316450.0222765553, "avg_output_tokens_per_task": 9605.333061668025, "last_run_timestamp": "2026-08-29T19:56:51.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.8344516577181208, "accuracy_std_dev": 0.048501499999999975, "run_count": 2, "total_questions": 298, "avg_cost_per_task_usd": 0.0903846644295302, "avg_duration_per_task_ms": 304392.4194630872, "avg_output_tokens_per_task": 22866.939597315435, "last_run_timestamp": "2026-07-18T08:27:05.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8341984152249134, "accuracy_std_dev": 0.029002188381032377, "run_count": 3, "total_questions": 578, "avg_cost_per_task_usd": 0.005492470588235294, "avg_duration_per_task_ms": 956260.6712802768, "avg_output_tokens_per_task": 15404.778546712803, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.8341078565737051, "accuracy_std_dev": 0.01236126339861748, "run_count": 10, "total_questions": 1506, "avg_cost_per_task_usd": 0.03458700464807437, "avg_duration_per_task_ms": 234385.25099601594, "avg_output_tokens_per_task": 27567.501328021248, "last_run_timestamp": "2026-09-19T16:00:28.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.8340128348623852, "accuracy_std_dev": 0.01276366393869724, "run_count": 20, "total_questions": 3924, "avg_cost_per_task_usd": 0.004765704128440367, "avg_duration_per_task_ms": 173295.0881753313, "avg_output_tokens_per_task": 15701.070846075434, "last_run_timestamp": "2026-08-29T18:49:32.713Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.8340065999999999, "accuracy_std_dev": 0.029823943324785232, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13819853030303028, "avg_duration_per_task_ms": 378353.51414141414, "avg_output_tokens_per_task": 35316.50404040404, "last_run_timestamp": "2026-09-19T08:49:31.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8338945000000001, "accuracy_std_dev": 0.036102903838657247, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.6825328661616162, "avg_duration_per_task_ms": 471720.8164983165, "avg_output_tokens_per_task": 27444.19781144781, "last_run_timestamp": "2026-09-19T08:48:28.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.833614, "accuracy_std_dev": 0.016210715242291687, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.012104537037037037, "avg_duration_per_task_ms": 473962.1228956229, "avg_output_tokens_per_task": 40122.851851851854, "last_run_timestamp": "2026-09-19T08:44:54.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.8334797826086957, "accuracy_std_dev": 0.038296180861868125, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.05379731664470795, "avg_duration_per_task_ms": 258855.73583662714, "avg_output_tokens_per_task": 12134.429732103645, "last_run_timestamp": "2026-08-29T18:50:46.982Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.8332221724061321, "accuracy_std_dev": 0.13397161638673358, "run_count": 23, "total_questions": 4501, "avg_cost_per_task_usd": 0.03938373672517219, "avg_duration_per_task_ms": 180863.35369917797, "avg_output_tokens_per_task": 10881.63741390802, "last_run_timestamp": "2026-08-29T19:36:49.687Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.833212137254902, "accuracy_std_dev": 0.02511813859250007, "run_count": 14, "total_questions": 2754, "avg_cost_per_task_usd": 0.06167101997095135, "avg_duration_per_task_ms": 884018.2875816993, "avg_output_tokens_per_task": 28448.484749455336, "last_run_timestamp": "2026-08-29T19:12:56.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.8329946475964793, "accuracy_std_dev": 0.03338342053015737, "run_count": 15, "total_questions": 2954, "avg_cost_per_task_usd": 0.11301010663507109, "avg_duration_per_task_ms": 707116.9400812457, "avg_output_tokens_per_task": 34269.73628977657, "last_run_timestamp": "2026-08-29T18:43:32.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.8328522857142855, "accuracy_std_dev": 0.011962448742559688, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.24642647763347764, "avg_duration_per_task_ms": 279472.34812409815, "avg_output_tokens_per_task": 54220.55988455989, "last_run_timestamp": "2026-09-17T02:17:44.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.8328282, "accuracy_std_dev": 0.014614290846975768, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13095260505050504, "avg_duration_per_task_ms": 155759.8383838384, "avg_output_tokens_per_task": 29521.12626262626, "last_run_timestamp": "2026-09-19T08:58:36.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.8327721666666664, "accuracy_std_dev": 0.03409742452598958, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.014290039562289561, "avg_duration_per_task_ms": 292947.5042087542, "avg_output_tokens_per_task": 11998.566077441077, "last_run_timestamp": "2026-07-18T09:15:42.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.832677994757536, "accuracy_std_dev": 0.01790311723380873, "run_count": 4, "total_questions": 763, "avg_cost_per_task_usd": 0.14453915727391875, "avg_duration_per_task_ms": 303619.376146789, "avg_output_tokens_per_task": 32661.047182175622, "last_run_timestamp": "2026-09-19T08:49:14.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Together", "accuracy": 0.8326117142857142, "accuracy_std_dev": 0.02081119738927845, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.05571586471861472, "avg_duration_per_task_ms": 232585.47619047618, "avg_output_tokens_per_task": 17366.68867243867, "last_run_timestamp": "2026-06-20T17:09:19.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.8324706727037516, "accuracy_std_dev": 0.015362295521421925, "run_count": 4, "total_questions": 773, "avg_cost_per_task_usd": 0.104053553686934, "avg_duration_per_task_ms": 725443.3247089263, "avg_output_tokens_per_task": 27644.671410090556, "last_run_timestamp": "2026-09-19T08:52:28.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.8319369960317462, "accuracy_std_dev": 0.026396632252184915, "run_count": 23, "total_questions": 4536, "avg_cost_per_task_usd": 0.04784231701940035, "avg_duration_per_task_ms": 507122.99823633156, "avg_output_tokens_per_task": 24258.074074074073, "last_run_timestamp": "2026-08-30T00:30:19.495Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.8318344082733814, "accuracy_std_dev": 0.03947741134153016, "run_count": 17, "total_questions": 3336, "avg_cost_per_task_usd": 0.14675382314148683, "avg_duration_per_task_ms": 651191.3980815348, "avg_output_tokens_per_task": 33576.36121103117, "last_run_timestamp": "2026-08-29T18:21:26.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.8316448145896658, "accuracy_std_dev": 0.03141045267820889, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.11912360688956432, "avg_duration_per_task_ms": 503318.0197568389, "avg_output_tokens_per_task": 34925.666666666664, "last_run_timestamp": "2026-09-19T08:47:24.459Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.8313797864493997, "accuracy_std_dev": 0.024754194188330587, "run_count": 18, "total_questions": 3498, "avg_cost_per_task_usd": 0.04931393767867353, "avg_duration_per_task_ms": 846895.9456832475, "avg_output_tokens_per_task": 28170.789879931388, "last_run_timestamp": "2026-08-22T17:50:04.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8313591965192169, "accuracy_std_dev": 0.02385695725514725, "run_count": 21, "total_questions": 4137, "avg_cost_per_task_usd": 0.015896513173797436, "avg_duration_per_task_ms": 285786.35170413344, "avg_output_tokens_per_task": 13215.3529127387, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.8312629347826086, "accuracy_std_dev": 0.044536486139877485, "run_count": 4, "total_questions": 644, "avg_cost_per_task_usd": 0.005694647515527951, "avg_duration_per_task_ms": 270388.4906832298, "avg_output_tokens_per_task": 13521.194099378881, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.8312575670329668, "accuracy_std_dev": 0.0356512774660921, "run_count": 14, "total_questions": 2730, "avg_cost_per_task_usd": 0.04862300952380953, "avg_duration_per_task_ms": 395101.81245421246, "avg_output_tokens_per_task": 19499.453846153847, "last_run_timestamp": "2026-08-01T17:53:10.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.8310886, "accuracy_std_dev": 0.05088763866506941, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.1784800898989899, "avg_duration_per_task_ms": 40054.59898989899, "avg_output_tokens_per_task": 3209.878451178451, "last_run_timestamp": "2026-09-19T09:08:31.785Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.8309340194750211, "accuracy_std_dev": 0.0227049955198312, "run_count": 6, "total_questions": 1181, "avg_cost_per_task_usd": 0.0975369652836579, "avg_duration_per_task_ms": 214278.25232853513, "avg_output_tokens_per_task": 28093.950889077052, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.830808, "accuracy_std_dev": 0.008551060402078817, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.09822325252525253, "avg_duration_per_task_ms": 297130.1747474747, "avg_output_tokens_per_task": 24088.215151515153, "last_run_timestamp": "2026-09-19T08:48:09.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.8306985375494073, "accuracy_std_dev": 0.018572923858850742, "run_count": 3, "total_questions": 506, "avg_cost_per_task_usd": 0.020810217391304345, "avg_duration_per_task_ms": 244282.4347826087, "avg_output_tokens_per_task": 41406.6837944664, "last_run_timestamp": "2026-09-19T08:48:21.497Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.8304598896551725, "accuracy_std_dev": 0.003983459286711617, "run_count": 6, "total_questions": 1160, "avg_cost_per_task_usd": 0.03665691206896552, "avg_duration_per_task_ms": 205169.7853448276, "avg_output_tokens_per_task": 13753.440517241379, "last_run_timestamp": "2026-09-19T08:48:39.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.830442043478261, "accuracy_std_dev": 0.013460761850905967, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.060090737154150205, "avg_duration_per_task_ms": 218919.57114624506, "avg_output_tokens_per_task": 27589.693895476503, "last_run_timestamp": "2026-08-30T00:09:46.578Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8303131074589128, "accuracy_std_dev": 0.009856971411895784, "run_count": 24, "total_questions": 4746, "avg_cost_per_task_usd": 0.00450906805731142, "avg_duration_per_task_ms": 297774.21533923305, "avg_output_tokens_per_task": 11688.514327855035, "last_run_timestamp": "2026-08-29T16:40:58.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "WandB", "accuracy": 0.8302964372990355, "accuracy_std_dev": 0.05435128994016608, "run_count": 5, "total_questions": 933, "avg_cost_per_task_usd": 0.004838944265809216, "avg_duration_per_task_ms": 341319.115755627, "avg_output_tokens_per_task": 13605.640943193997, "last_run_timestamp": "2026-07-18T08:04:02.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.8299663333333334, "accuracy_std_dev": 0.009322650368263805, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.04014494781144781, "avg_duration_per_task_ms": 219758.45622895623, "avg_output_tokens_per_task": 14463.257575757576, "last_run_timestamp": "2026-09-19T08:47:20.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.8297885767530007, "accuracy_std_dev": 0.0250017324703876, "run_count": 24, "total_questions": 4749, "avg_cost_per_task_usd": 0.16577993451252898, "avg_duration_per_task_ms": 150407.92208886083, "avg_output_tokens_per_task": 15324.479679932618, "last_run_timestamp": "2026-09-19T08:54:35.988Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.8295164461407973, "accuracy_std_dev": 0.030217333777287914, "run_count": 18, "total_questions": 3537, "avg_cost_per_task_usd": 0.13597140797285837, "avg_duration_per_task_ms": 506852.9443030817, "avg_output_tokens_per_task": 33591.725190839694, "last_run_timestamp": "2026-08-29T18:09:15.344Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8294639871023217, "accuracy_std_dev": 0.027475351240747944, "run_count": 18, "total_questions": 3489, "avg_cost_per_task_usd": 0.1216583038119805, "avg_duration_per_task_ms": 574849.784752078, "avg_output_tokens_per_task": 33861.400401261104, "last_run_timestamp": "2026-08-29T18:14:11.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.8294050555555555, "accuracy_std_dev": 0.021508202180853447, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.02960114337822672, "avg_duration_per_task_ms": 125113.44304152638, "avg_output_tokens_per_task": 16376.810606060606, "last_run_timestamp": "2026-08-29T18:13:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.829367646430954, "accuracy_std_dev": 0.05068897680083226, "run_count": 23, "total_questions": 4497, "avg_cost_per_task_usd": 0.10293063842561709, "avg_duration_per_task_ms": 624030.37914165, "avg_output_tokens_per_task": 32772.14164998888, "last_run_timestamp": "2026-08-29T18:11:51.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.8293240541237112, "accuracy_std_dev": 0.05663663093125379, "run_count": 12, "total_questions": 2328, "avg_cost_per_task_usd": 0.04663492611683849, "avg_duration_per_task_ms": 552324.0841924399, "avg_output_tokens_per_task": 22329.278350515466, "last_run_timestamp": "2026-09-19T08:14:35.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.829273741542625, "accuracy_std_dev": 0.020525043792584238, "run_count": 4, "total_questions": 739, "avg_cost_per_task_usd": 0.023301357239512856, "avg_duration_per_task_ms": 305516.3085250338, "avg_output_tokens_per_task": 46400.958051420836, "last_run_timestamp": "2026-09-19T08:49:44.918Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.8291479228699551, "accuracy_std_dev": 0.046765034341613804, "run_count": 34, "total_questions": 6690, "avg_cost_per_task_usd": 0.03550391076233183, "avg_duration_per_task_ms": 277215.60224215244, "avg_output_tokens_per_task": 18441.03587443946, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.828497570984456, "accuracy_std_dev": 0.015230029187102692, "run_count": 5, "total_questions": 965, "avg_cost_per_task_usd": 0.11257541554404144, "avg_duration_per_task_ms": 438124.0186528497, "avg_output_tokens_per_task": 31261.672538860104, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8283502494845361, "accuracy_std_dev": 0.01123608353475534, "run_count": 5, "total_questions": 970, "avg_cost_per_task_usd": 0.12046294226804122, "avg_duration_per_task_ms": 260289.6525773196, "avg_output_tokens_per_task": 28436.055670103095, "last_run_timestamp": "2026-09-19T08:59:56.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DekaLLM", "accuracy": 0.8282827500000001, "accuracy_std_dev": 0.02172313812936567, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0027490239898989898, "avg_duration_per_task_ms": 108260.87878787878, "avg_output_tokens_per_task": 2717.243686868687, "last_run_timestamp": "2026-05-16T08:55:42.302Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.82828275, "accuracy_std_dev": 0.014334603419261384, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.012366218434343434, "avg_duration_per_task_ms": 835296.3156565657, "avg_output_tokens_per_task": 36481.89646464647, "last_run_timestamp": "2026-09-19T08:15:51.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.8279570000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 186, "avg_cost_per_task_usd": 0.024709930107526885, "avg_duration_per_task_ms": 419969.8602150538, "avg_output_tokens_per_task": 20901.392473118278, "last_run_timestamp": "2026-08-01T18:03:43.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "AtlasCloud", "accuracy": 0.8276921538461538, "accuracy_std_dev": 0.027161692421496852, "run_count": 10, "total_questions": 1950, "avg_cost_per_task_usd": 0.032337288205128205, "avg_duration_per_task_ms": 326118.6758974359, "avg_output_tokens_per_task": 12882.684102564102, "last_run_timestamp": "2026-06-30T22:55:56.214Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.8275520710659898, "accuracy_std_dev": 0.0776517694741866, "run_count": 12, "total_questions": 2364, "avg_cost_per_task_usd": 0.07118247335025381, "avg_duration_per_task_ms": 485649.0562605753, "avg_output_tokens_per_task": 31102.425126903552, "last_run_timestamp": "2026-09-19T08:54:19.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.8273554038986354, "accuracy_std_dev": 0.018032038830334687, "run_count": 13, "total_questions": 2565, "avg_cost_per_task_usd": 0.020172549707602336, "avg_duration_per_task_ms": 335505.9828460039, "avg_output_tokens_per_task": 20130.481091617934, "last_run_timestamp": "2026-08-29T16:15:12.678Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8272305, "accuracy_std_dev": 0.016036241150905673, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.06078435227272727, "avg_duration_per_task_ms": 492639.2809343434, "avg_output_tokens_per_task": 47927.3244949495, "last_run_timestamp": "2026-09-19T16:08:54.164Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Together", "accuracy": 0.8271558080733944, "accuracy_std_dev": 0.040507064699838946, "run_count": 14, "total_questions": 2725, "avg_cost_per_task_usd": 0.016758301284403674, "avg_duration_per_task_ms": 220975.85504587155, "avg_output_tokens_per_task": 14210.697247706423, "last_run_timestamp": "2026-07-18T08:29:12.575Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.8267805538461538, "accuracy_std_dev": 0.020665297105829934, "run_count": 12, "total_questions": 2340, "avg_cost_per_task_usd": 0.08551232649572652, "avg_duration_per_task_ms": 577054.2572649573, "avg_output_tokens_per_task": 31689.464957264958, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8267103169705471, "accuracy_std_dev": 0.014502002597532615, "run_count": 11, "total_questions": 2139, "avg_cost_per_task_usd": 0.009508418419822345, "avg_duration_per_task_ms": 526491.7872837775, "avg_output_tokens_per_task": 33720.50303880318, "last_run_timestamp": "2026-09-19T09:13:55.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8264543613323679, "accuracy_std_dev": 0.018188360323715465, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.20616153874004342, "avg_duration_per_task_ms": 315117.1817523534, "avg_output_tokens_per_task": 46875.9225199131, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8263960270551509, "accuracy_std_dev": 0.04766683339597883, "run_count": 5, "total_questions": 961, "avg_cost_per_task_usd": 0.028155845993756503, "avg_duration_per_task_ms": 288712.7356919875, "avg_output_tokens_per_task": 8693.547346514048, "last_run_timestamp": "2026-09-19T09:20:52.921Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.8262626000000001, "accuracy_std_dev": 0.005773190715713467, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.1255757282828283, "avg_duration_per_task_ms": 138406.39494949495, "avg_output_tokens_per_task": 30506.08282828283, "last_run_timestamp": "2026-09-19T08:50:32.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.82617825, "accuracy_std_dev": 0.007374361408793328, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.045437868686868686, "avg_duration_per_task_ms": 57534.18939393939, "avg_output_tokens_per_task": 11194.286616161617, "last_run_timestamp": "2026-09-19T08:49:40.730Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8261094014447884, "accuracy_std_dev": 0.026551028658038853, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.2076026150670795, "avg_duration_per_task_ms": 857894.439628483, "avg_output_tokens_per_task": 47560.28689370485, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.8259803298319327, "accuracy_std_dev": 0.013069264196579715, "run_count": 5, "total_questions": 952, "avg_cost_per_task_usd": 0.04310651365546219, "avg_duration_per_task_ms": 411042.2016806723, "avg_output_tokens_per_task": 42229.66176470588, "last_run_timestamp": "2026-09-19T08:50:24.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.8259176777899344, "accuracy_std_dev": 0.050464276054948014, "run_count": 28, "total_questions": 5484, "avg_cost_per_task_usd": 0.030212785740335524, "avg_duration_per_task_ms": 236266.8163749088, "avg_output_tokens_per_task": 15214.504011670313, "last_run_timestamp": "2026-08-01T17:44:01.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8258873707224335, "accuracy_std_dev": 0.009676826643527784, "run_count": 6, "total_questions": 1052, "avg_cost_per_task_usd": 0.025350776615969583, "avg_duration_per_task_ms": 120122.08840304182, "avg_output_tokens_per_task": 20151.38783269962, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.8255112924187727, "accuracy_std_dev": 0.033062154820560075, "run_count": 21, "total_questions": 4155, "avg_cost_per_task_usd": 0.01651286329723225, "avg_duration_per_task_ms": 227671.96774969916, "avg_output_tokens_per_task": 13721.429602888087, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.825477, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04118768013468014, "avg_duration_per_task_ms": 215669.1313131313, "avg_output_tokens_per_task": 14940.681818181818, "last_run_timestamp": "2026-09-19T09:12:21.615Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.8253399469387755, "accuracy_std_dev": 0.06202361079266511, "run_count": 5, "total_questions": 980, "avg_cost_per_task_usd": 0.048117852040816325, "avg_duration_per_task_ms": 1077180.0714285714, "avg_output_tokens_per_task": 26155.976530612246, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.8252867162300974, "accuracy_std_dev": 0.1181226298338754, "run_count": 21, "total_questions": 3894, "avg_cost_per_task_usd": 0.03139470313302517, "avg_duration_per_task_ms": 234154.43913713406, "avg_output_tokens_per_task": 9983.509758602979, "last_run_timestamp": "2026-08-29T18:43:58.355Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.8246641421568626, "accuracy_std_dev": 0.04102567192972371, "run_count": 19, "total_questions": 3672, "avg_cost_per_task_usd": 0.09416541448801743, "avg_duration_per_task_ms": 832671.316993464, "avg_output_tokens_per_task": 28688.930555555555, "last_run_timestamp": "2026-08-29T18:08:11.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8245855497237569, "accuracy_std_dev": 0.05595579604141202, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.02354478360957643, "avg_duration_per_task_ms": 541510.1505524862, "avg_output_tokens_per_task": 25036.35497237569, "last_run_timestamp": "2026-09-19T09:09:19.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.8245443432510885, "accuracy_std_dev": 0.014070989097480181, "run_count": 21, "total_questions": 4134, "avg_cost_per_task_usd": 0.0048930220125786164, "avg_duration_per_task_ms": 408825.40662796324, "avg_output_tokens_per_task": 11390.059748427673, "last_run_timestamp": "2026-08-29T18:40:24.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.82449475, "accuracy_std_dev": 0.010869783171135461, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06350338131313131, "avg_duration_per_task_ms": 123563.64393939394, "avg_output_tokens_per_task": 13357.755050505051, "last_run_timestamp": "2026-09-01T17:56:39.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8242424, "accuracy_std_dev": 0.011692885317149064, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.043399320202020204, "avg_duration_per_task_ms": 227400.42525252525, "avg_output_tokens_per_task": 14348.484848484848, "last_run_timestamp": "2026-09-19T08:47:45.478Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.823863625, "accuracy_std_dev": 0.014332832029448166, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.31506537247474753, "avg_duration_per_task_ms": 289259.720959596, "avg_output_tokens_per_task": 20815.732954545456, "last_run_timestamp": "2026-09-19T08:48:25.679Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.8234257166446501, "accuracy_std_dev": 0.04369321987022243, "run_count": 23, "total_questions": 4542, "avg_cost_per_task_usd": 0.06675309929546455, "avg_duration_per_task_ms": 567531.9035667107, "avg_output_tokens_per_task": 30278.626376045795, "last_run_timestamp": "2026-08-29T18:40:30.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.82339096128561, "accuracy_std_dev": 0.01836790238580393, "run_count": 21, "total_questions": 4107, "avg_cost_per_task_usd": 0.10042522181641102, "avg_duration_per_task_ms": 563721.646457268, "avg_output_tokens_per_task": 31885.672510348188, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.8232322727272725, "accuracy_std_dev": 0.03750024958969282, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.00558554453627181, "avg_duration_per_task_ms": 239700.90036730946, "avg_output_tokens_per_task": 13322.357208448118, "last_run_timestamp": "2026-08-01T20:03:34.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.823232, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03873228787878788, "avg_duration_per_task_ms": 172542.09595959596, "avg_output_tokens_per_task": 15454.005050505051, "last_run_timestamp": "2026-09-20T21:05:40.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.8231507132003797, "accuracy_std_dev": 0.013738132759397979, "run_count": 16, "total_questions": 3159, "avg_cost_per_task_usd": 0.012798601772712881, "avg_duration_per_task_ms": 385156.8765432099, "avg_output_tokens_per_task": 12700.649889205444, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.823107677419355, "accuracy_std_dev": 0.0400712127273121, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.164333871600253, "avg_duration_per_task_ms": 161114.86337760912, "avg_output_tokens_per_task": 16212.869070208728, "last_run_timestamp": "2026-09-19T08:49:31.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8222913416149069, "accuracy_std_dev": 0.020103371140184446, "run_count": 5, "total_questions": 966, "avg_cost_per_task_usd": 0.13681437163561078, "avg_duration_per_task_ms": 249682.33954451347, "avg_output_tokens_per_task": 30864.992753623188, "last_run_timestamp": "2026-09-19T08:50:10.014Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.8222221999999999, "accuracy_std_dev": 0.01398833591818555, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.034438841414141406, "avg_duration_per_task_ms": 232730.4090909091, "avg_output_tokens_per_task": 13762.19696969697, "last_run_timestamp": "2026-09-19T08:49:03.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.8221166532066507, "accuracy_std_dev": 0.015661128267637187, "run_count": 13, "total_questions": 2526, "avg_cost_per_task_usd": 0.03727458986539984, "avg_duration_per_task_ms": 329571.73871733964, "avg_output_tokens_per_task": 20674.40657165479, "last_run_timestamp": "2026-08-29T16:44:34.185Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.82211, "accuracy_std_dev": 0.02580143550915981, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03809178872053872, "avg_duration_per_task_ms": 290291.4090909091, "avg_output_tokens_per_task": 14257.26936026936, "last_run_timestamp": "2026-09-19T08:52:03.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8219697499999998, "accuracy_std_dev": 0.013060489125124707, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03950909848484849, "avg_duration_per_task_ms": 229156.16414141413, "avg_output_tokens_per_task": 15405.453282828283, "last_run_timestamp": "2026-09-19T08:48:23.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.8219452102185287, "accuracy_std_dev": 0.034130220185287316, "run_count": 17, "total_questions": 3249, "avg_cost_per_task_usd": 0.08630355401662049, "avg_duration_per_task_ms": 387562.971683595, "avg_output_tokens_per_task": 22693.282856263464, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "auto-routing", "accuracy": 0.8215312538860104, "accuracy_std_dev": 0.022946010202695863, "run_count": 3, "total_questions": 579, "avg_cost_per_task_usd": 0.08296633678756477, "avg_duration_per_task_ms": 239744.6183074266, "avg_output_tokens_per_task": 23823.246977547496, "last_run_timestamp": "2026-06-30T21:35:33.459Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.8214834459833794, "accuracy_std_dev": 0.011465056212986574, "run_count": 11, "total_questions": 2166, "avg_cost_per_task_usd": 0.005549661126500463, "avg_duration_per_task_ms": 487617.81809787627, "avg_output_tokens_per_task": 16572.74192059095, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8210970962025316, "accuracy_std_dev": 0.02845442397599282, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.039231337130801694, "avg_duration_per_task_ms": 342482.6898734177, "avg_output_tokens_per_task": 40890.57890295359, "last_run_timestamp": "2026-09-19T16:14:18.749Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Together", "accuracy": 0.8209874444444444, "accuracy_std_dev": 0.015449980784677811, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.07123941301907968, "avg_duration_per_task_ms": 202727.39393939395, "avg_output_tokens_per_task": 16398.18462401796, "last_run_timestamp": "2026-06-20T09:16:50.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.8209175000000001, "accuracy_std_dev": 0.025544439297036866, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.03782663825757576, "avg_duration_per_task_ms": 314167.50126262626, "avg_output_tokens_per_task": 39526.941287878784, "last_run_timestamp": "2026-09-19T16:25:45.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Ambient", "accuracy": 0.8207071666666668, "accuracy_std_dev": 0.015360537180457651, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008930407407407408, "avg_duration_per_task_ms": 167713.53535353535, "avg_output_tokens_per_task": 8891.933501683501, "last_run_timestamp": "2026-06-13T16:35:11.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.8206819729372937, "accuracy_std_dev": 0.01766511701862355, "run_count": 23, "total_questions": 4545, "avg_cost_per_task_usd": 0.004627064906490648, "avg_duration_per_task_ms": 269164.16699669964, "avg_output_tokens_per_task": 11500.384598459847, "last_run_timestamp": "2026-08-29T19:19:45.463Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.8204619306930694, "accuracy_std_dev": 0.012993991694579071, "run_count": 23, "total_questions": 4545, "avg_cost_per_task_usd": 0.011459436963696373, "avg_duration_per_task_ms": 354916.7643564356, "avg_output_tokens_per_task": 11342.301650165016, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "Xiaomi", "accuracy": 0.8204264444444443, "accuracy_std_dev": 0.015265342891014489, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0033981470258136926, "avg_duration_per_task_ms": 86880.46913580247, "avg_output_tokens_per_task": 11274.21492704826, "last_run_timestamp": "2026-06-13T08:05:57.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.8203543500992719, "accuracy_std_dev": 0.047076899102709124, "run_count": 23, "total_questions": 4533, "avg_cost_per_task_usd": 0.07291643834105449, "avg_duration_per_task_ms": 483587.120891242, "avg_output_tokens_per_task": 28482.810500772117, "last_run_timestamp": "2026-08-29T19:11:37.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.8203043478260869, "accuracy_std_dev": 0.021973884023913672, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.004347063241106719, "avg_duration_per_task_ms": 319883.9997804128, "avg_output_tokens_per_task": 11111.253184014053, "last_run_timestamp": "2026-08-29T19:12:36.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.8201456666666667, "accuracy_std_dev": 0.021829497258424348, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03759047053872054, "avg_duration_per_task_ms": 252471.02188552188, "avg_output_tokens_per_task": 14715.675084175084, "last_run_timestamp": "2026-09-19T08:47:29.837Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.8201054571428571, "accuracy_std_dev": 0.026122159766927572, "run_count": 3, "total_questions": 315, "avg_cost_per_task_usd": 0.055772555555555554, "avg_duration_per_task_ms": 137698.42222222223, "avg_output_tokens_per_task": 23066.479365079365, "last_run_timestamp": "2026-09-12T16:00:25.695Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.8198026983870969, "accuracy_std_dev": 0.05250346254129803, "run_count": 19, "total_questions": 3720, "avg_cost_per_task_usd": 0.10939475376344085, "avg_duration_per_task_ms": 699399.8586021506, "avg_output_tokens_per_task": 34933.458333333336, "last_run_timestamp": "2026-08-29T18:49:03.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.8195846666666666, "accuracy_std_dev": 0.016130608974520736, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11832865488215488, "avg_duration_per_task_ms": 298591.92424242425, "avg_output_tokens_per_task": 48880.583333333336, "last_run_timestamp": "2026-09-19T08:45:26.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.819234, "accuracy_std_dev": 0.018593792606673883, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.011218165404040404, "avg_duration_per_task_ms": 587595.9393939395, "avg_output_tokens_per_task": 33133.1470959596, "last_run_timestamp": "2026-09-19T09:14:17.409Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.8192089661016949, "accuracy_std_dev": 0.16613254756324589, "run_count": 12, "total_questions": 2301, "avg_cost_per_task_usd": 0.08146611951325512, "avg_duration_per_task_ms": 457624.593220339, "avg_output_tokens_per_task": 22201.0308561495, "last_run_timestamp": "2026-09-19T09:05:59.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.8188576490610326, "accuracy_std_dev": 0.0725401109287967, "run_count": 13, "total_questions": 1704, "avg_cost_per_task_usd": 0.020530700117370893, "avg_duration_per_task_ms": 66110.05575117371, "avg_output_tokens_per_task": 20662.859154929578, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.8187381511299434, "accuracy_std_dev": 0.020219033179017607, "run_count": 11, "total_questions": 2124, "avg_cost_per_task_usd": 0.052499902542372884, "avg_duration_per_task_ms": 261839.05320150658, "avg_output_tokens_per_task": 21842.697740112995, "last_run_timestamp": "2026-08-01T17:52:55.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.818182, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14423544949494949, "avg_duration_per_task_ms": 273234.71717171714, "avg_output_tokens_per_task": 28813.959595959597, "last_run_timestamp": "2026-09-19T08:47:57.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8181816666666667, "accuracy_std_dev": 0.013255807590972678, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.17135338047138046, "avg_duration_per_task_ms": 45814.95117845118, "avg_output_tokens_per_task": 3234.355218855219, "last_run_timestamp": "2026-08-15T09:15:35.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.8181816666666667, "accuracy_std_dev": 0.08468970304522794, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.007531994107744108, "avg_duration_per_task_ms": 771784.1237373737, "avg_output_tokens_per_task": 30879.069865319867, "last_run_timestamp": "2026-09-19T13:19:37.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.8180828300653594, "accuracy_std_dev": 0.1016204485105719, "run_count": 14, "total_questions": 2754, "avg_cost_per_task_usd": 0.04520774546114742, "avg_duration_per_task_ms": 188140.4183006536, "avg_output_tokens_per_task": 21641.156136528687, "last_run_timestamp": "2026-09-19T08:53:56.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8180825764931331, "accuracy_std_dev": 0.05580919908720086, "run_count": 33, "total_questions": 6262, "avg_cost_per_task_usd": 0.005215132864899393, "avg_duration_per_task_ms": 358805.9053018205, "avg_output_tokens_per_task": 12457.650590865538, "last_run_timestamp": "2026-08-29T20:12:46.798Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.818035862576065, "accuracy_std_dev": 0.01278056208505712, "run_count": 10, "total_questions": 1972, "avg_cost_per_task_usd": 0.0909173985801217, "avg_duration_per_task_ms": 165913.36257606492, "avg_output_tokens_per_task": 22308.299188640973, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.8172269616858236, "accuracy_std_dev": 0.059023830851696865, "run_count": 12, "total_questions": 2349, "avg_cost_per_task_usd": 0.1289344618986803, "avg_duration_per_task_ms": 699975.4461472967, "avg_output_tokens_per_task": 43383.50404427416, "last_run_timestamp": "2026-08-29T19:22:50.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8169413157894736, "accuracy_std_dev": 0.012465325718516053, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.3349828732057416, "avg_duration_per_task_ms": 301917.04891015415, "avg_output_tokens_per_task": 21250.364167995747, "last_run_timestamp": "2026-09-19T08:51:02.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.8168373000310271, "accuracy_std_dev": 0.052236706041768964, "run_count": 19, "total_questions": 3223, "avg_cost_per_task_usd": 0.0374196242631089, "avg_duration_per_task_ms": 545799.8814768849, "avg_output_tokens_per_task": 31060.19795221843, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Ambient", "accuracy": 0.8164983333333333, "accuracy_std_dev": 0.010378007269649053, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002570441077441077, "avg_duration_per_task_ms": 557971.7020202021, "avg_output_tokens_per_task": 4954.074074074074, "last_run_timestamp": "2026-06-13T08:04:34.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.8161555499021526, "accuracy_std_dev": 0.033245258304521796, "run_count": 8, "total_questions": 1533, "avg_cost_per_task_usd": 0.01330266210045662, "avg_duration_per_task_ms": 93162.18590998043, "avg_output_tokens_per_task": 17543.737769080235, "last_run_timestamp": "2026-08-29T18:36:28.592Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.8161240555555556, "accuracy_std_dev": 0.03828528953567215, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.018823198372615035, "avg_duration_per_task_ms": 237443.63439955105, "avg_output_tokens_per_task": 17323.3285634119, "last_run_timestamp": "2026-08-29T19:18:38.408Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.8159271462140993, "accuracy_std_dev": 0.011945687451440362, "run_count": 4, "total_questions": 766, "avg_cost_per_task_usd": 0.10126304177545693, "avg_duration_per_task_ms": 221714.96997389034, "avg_output_tokens_per_task": 28424.819843342037, "last_run_timestamp": "2026-08-04T03:11:59.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.8157936203059803, "accuracy_std_dev": 0.02454612802005358, "run_count": 11, "total_questions": 2157, "avg_cost_per_task_usd": 0.00473010709318498, "avg_duration_per_task_ms": 443979.5808993973, "avg_output_tokens_per_task": 11275.669448307835, "last_run_timestamp": "2026-08-01T18:25:37.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.8157768571428571, "accuracy_std_dev": 0.021818009359429742, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04211656493506493, "avg_duration_per_task_ms": 262574.5, "avg_output_tokens_per_task": 14107.66883116883, "last_run_timestamp": "2026-09-19T08:55:42.602Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8157223200612557, "accuracy_std_dev": 0.0849169018041167, "run_count": 10, "total_questions": 1959, "avg_cost_per_task_usd": 0.10062858907605922, "avg_duration_per_task_ms": 941154.5890760592, "avg_output_tokens_per_task": 42235.08728943339, "last_run_timestamp": "2026-09-19T08:57:20.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.8148881320132014, "accuracy_std_dev": 0.01824134523693167, "run_count": 23, "total_questions": 4545, "avg_cost_per_task_usd": 0.06443867590759077, "avg_duration_per_task_ms": 437849.9201320132, "avg_output_tokens_per_task": 32018.359075907592, "last_run_timestamp": "2026-08-29T19:17:49.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8146043749999999, "accuracy_std_dev": 0.010793946798292766, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.36479117045454545, "avg_duration_per_task_ms": 359805.747790404, "avg_output_tokens_per_task": 22977.79356060606, "last_run_timestamp": "2026-09-19T08:53:08.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Fireworks", "accuracy": 0.8144500956822106, "accuracy_std_dev": 0.08783913519448809, "run_count": 15, "total_questions": 2895, "avg_cost_per_task_usd": 0.018499234196891194, "avg_duration_per_task_ms": 92763.99827288429, "avg_output_tokens_per_task": 15429.552677029362, "last_run_timestamp": "2026-07-18T08:31:24.271Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8143338571428571, "accuracy_std_dev": 0.014248444761532711, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026212479076479073, "avg_duration_per_task_ms": 329590.34992784995, "avg_output_tokens_per_task": 17511.52380952381, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8141788521373509, "accuracy_std_dev": 0.09118468393428555, "run_count": 15, "total_questions": 2854, "avg_cost_per_task_usd": 0.05950021793973371, "avg_duration_per_task_ms": 877552.4621583743, "avg_output_tokens_per_task": 27412.963910301332, "last_run_timestamp": "2026-08-29T19:12:33.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.813853, "accuracy_std_dev": 0.0071343829445860135, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.025065300144300147, "avg_duration_per_task_ms": 187376.2683982684, "avg_output_tokens_per_task": 16614.633477633477, "last_run_timestamp": "2026-09-19T08:53:39.948Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "AtlasCloud", "accuracy": 0.813825179389313, "accuracy_std_dev": 0.015856752133850578, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.02502054643765903, "avg_duration_per_task_ms": 242441.94974554708, "avg_output_tokens_per_task": 10837.982824427481, "last_run_timestamp": "2026-07-01T04:42:33.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.8137254984162897, "accuracy_std_dev": 0.08405639786637797, "run_count": 24, "total_questions": 4420, "avg_cost_per_task_usd": 0.010863166515837101, "avg_duration_per_task_ms": 478036.79049773753, "avg_output_tokens_per_task": 11973.086425339367, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.8136272825651302, "accuracy_std_dev": 0.008773473238499465, "run_count": 3, "total_questions": 499, "avg_cost_per_task_usd": 0.0068851422845691386, "avg_duration_per_task_ms": 1090136, "avg_output_tokens_per_task": 17404.21643286573, "last_run_timestamp": "2026-08-15T10:05:32.709Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.813131, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.037260742424242425, "avg_duration_per_task_ms": 35156.99494949495, "avg_output_tokens_per_task": 2566.8030303030305, "last_run_timestamp": "2026-08-01T08:32:23.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.812989225241824, "accuracy_std_dev": 0.04159005714835348, "run_count": 11, "total_questions": 2171, "avg_cost_per_task_usd": 0.05707970474435743, "avg_duration_per_task_ms": 931231.9829571625, "avg_output_tokens_per_task": 29294.633348687243, "last_run_timestamp": "2026-09-19T17:02:33.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.8127893802083332, "accuracy_std_dev": 0.02323940439151189, "run_count": 12, "total_questions": 2304, "avg_cost_per_task_usd": 0.014347588107638889, "avg_duration_per_task_ms": 360100.9900173611, "avg_output_tokens_per_task": 13752.019097222223, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.8116520828488373, "accuracy_std_dev": 0.02238695596611053, "run_count": 7, "total_questions": 1376, "avg_cost_per_task_usd": 0.021895043604651165, "avg_duration_per_task_ms": 295867.60247093026, "avg_output_tokens_per_task": 18000.89898255814, "last_run_timestamp": "2026-09-19T08:44:55.432Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.8111882789927104, "accuracy_std_dev": 0.14234185480055578, "run_count": 16, "total_questions": 3018, "avg_cost_per_task_usd": 0.003854944665341285, "avg_duration_per_task_ms": 404801.41351888666, "avg_output_tokens_per_task": 9504.889993373095, "last_run_timestamp": "2026-08-29T18:15:38.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AkashML", "accuracy": 0.8109296153846153, "accuracy_std_dev": 0.14891345125817648, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.008492456099456099, "avg_duration_per_task_ms": 161496.31857031857, "avg_output_tokens_per_task": 7371.756021756022, "last_run_timestamp": "2026-07-01T03:19:17.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "WandB", "accuracy": 0.8106060000000002, "accuracy_std_dev": 0.016940078320362043, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.000941689393939394, "avg_duration_per_task_ms": 123178.65404040404, "avg_output_tokens_per_task": 9148.02398989899, "last_run_timestamp": "2026-06-20T16:49:31.612Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.8100816932907348, "accuracy_std_dev": 0.08830772854780453, "run_count": 11, "total_questions": 1878, "avg_cost_per_task_usd": 0.08196660915867944, "avg_duration_per_task_ms": 525045.6922257721, "avg_output_tokens_per_task": 25551.769968051118, "last_run_timestamp": "2026-09-19T16:07:59.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openrouter/auto-beta", "model_slug": "openrouter/auto-beta", "model_name": "Auto Router (Beta)", "provider_name": "auto-routing", "accuracy": 0.8097645, "accuracy_std_dev": 0.0622895, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.034729030303030296, "avg_duration_per_task_ms": 253034.54292929292, "avg_output_tokens_per_task": 31317.588383838385, "last_run_timestamp": "2026-09-15T01:07:48.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.8093432500000001, "accuracy_std_dev": 0.008279566757234316, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03340588888888889, "avg_duration_per_task_ms": 671635.9166666666, "avg_output_tokens_per_task": 15467.433080808081, "last_run_timestamp": "2026-09-19T09:35:13.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.8091227713684209, "accuracy_std_dev": 0.08205896778163711, "run_count": 14, "total_questions": 2375, "avg_cost_per_task_usd": 0.0624333372631579, "avg_duration_per_task_ms": 697771.9258947369, "avg_output_tokens_per_task": 19861.26063157895, "last_run_timestamp": "2026-08-01T18:16:39.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.8089819438325991, "accuracy_std_dev": 0.06057756732336092, "run_count": 14, "total_questions": 2724, "avg_cost_per_task_usd": 0.15691310939794417, "avg_duration_per_task_ms": 677800.3755506608, "avg_output_tokens_per_task": 35807.34140969163, "last_run_timestamp": "2026-08-29T18:07:06.751Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.808541063084112, "accuracy_std_dev": 0.015819760309078213, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.011975492601246106, "avg_duration_per_task_ms": 326659.34890965733, "avg_output_tokens_per_task": 12473.692757009347, "last_run_timestamp": "2026-08-30T00:01:09.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.8079965500000003, "accuracy_std_dev": 0.018621334421772776, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.01976742373737374, "avg_duration_per_task_ms": 154039.40732323233, "avg_output_tokens_per_task": 15688.315909090908, "last_run_timestamp": "2026-08-29T19:24:09.844Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.8079122642559109, "accuracy_std_dev": 0.15281497299086352, "run_count": 11, "total_questions": 2157, "avg_cost_per_task_usd": 0.062700123783032, "avg_duration_per_task_ms": 394470.919796013, "avg_output_tokens_per_task": 17556.69772832638, "last_run_timestamp": "2026-09-19T09:15:56.025Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.8076977522123893, "accuracy_std_dev": 0.09983296230241988, "run_count": 12, "total_questions": 2373, "avg_cost_per_task_usd": 0.08567701222081754, "avg_duration_per_task_ms": 492014.607248209, "avg_output_tokens_per_task": 31610.124736620313, "last_run_timestamp": "2026-09-19T16:24:52.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8075196666666667, "accuracy_std_dev": 0.04008333166735965, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.16043005976430974, "avg_duration_per_task_ms": 159846.56397306398, "avg_output_tokens_per_task": 15912.45622895623, "last_run_timestamp": "2026-09-19T08:44:53.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.8073155454545455, "accuracy_std_dev": 0.016232793247361044, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.3389105385674931, "avg_duration_per_task_ms": 354028.78879706154, "avg_output_tokens_per_task": 22430.448576675848, "last_run_timestamp": "2026-09-19T08:54:14.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.80698645, "accuracy_std_dev": 0.018297436846386447, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.026827777777777784, "avg_duration_per_task_ms": 285701.8515151515, "avg_output_tokens_per_task": 15377.552777777777, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.8069215714285713, "accuracy_std_dev": 0.01495832495843468, "run_count": 13, "total_questions": 2562, "avg_cost_per_task_usd": 0.011023931303669009, "avg_duration_per_task_ms": 293511.45472287276, "avg_output_tokens_per_task": 11473.932084309134, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Together", "accuracy": 0.806818, "accuracy_std_dev": 0.027977996970834047, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.04117596338383839, "avg_duration_per_task_ms": 339680.07196969696, "avg_output_tokens_per_task": 15286.914141414141, "last_run_timestamp": "2026-05-16T10:24:57.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Fireworks", "accuracy": 0.8063973333333334, "accuracy_std_dev": 0.07629808423472303, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0736860505050505, "avg_duration_per_task_ms": 283613.16077441076, "avg_output_tokens_per_task": 18366.69107744108, "last_run_timestamp": "2026-05-23T11:07:21.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.8063971999999998, "accuracy_std_dev": 0.01659741351415936, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00614619090909091, "avg_duration_per_task_ms": 535780.8212121212, "avg_output_tokens_per_task": 18359.183838383837, "last_run_timestamp": "2026-08-29T19:12:20.528Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "auto-routing", "accuracy": 0.8060605000000001, "accuracy_std_dev": 0.034859352280987646, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.003273143939393939, "avg_duration_per_task_ms": 84028.83181818182, "avg_output_tokens_per_task": 10856.877272727274, "last_run_timestamp": "2026-06-13T08:37:07.102Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.8058781554959786, "accuracy_std_dev": 0.01700166220410292, "run_count": 17, "total_questions": 3357, "avg_cost_per_task_usd": 0.014998186773905272, "avg_duration_per_task_ms": 223628.16324098897, "avg_output_tokens_per_task": 13349.141197497765, "last_run_timestamp": "2026-08-29T18:59:02.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.805836, "accuracy_std_dev": 0.0155108889386349, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022988649831649832, "avg_duration_per_task_ms": 268127.5387205387, "avg_output_tokens_per_task": 17683.01936026936, "last_run_timestamp": "2026-09-19T08:47:58.850Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.805556, "accuracy_std_dev": null, "run_count": 1, "total_questions": 192, "avg_cost_per_task_usd": 0.16585933333333333, "avg_duration_per_task_ms": 688743.8177083334, "avg_output_tokens_per_task": 42426.447916666664, "last_run_timestamp": "2026-07-31T00:21:35.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.805555546875, "accuracy_std_dev": 0.01288499999999998, "run_count": 2, "total_questions": 384, "avg_cost_per_task_usd": 0.0362872109375, "avg_duration_per_task_ms": 729621.7864583334, "avg_output_tokens_per_task": 36271.864583333336, "last_run_timestamp": "2026-09-12T16:45:54.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "SiliconFlow", "accuracy": 0.8055555000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01909540909090909, "avg_duration_per_task_ms": 512069.0252525252, "avg_output_tokens_per_task": 9996.752525252525, "last_run_timestamp": "2026-05-09T11:21:48.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DekaLLM", "accuracy": 0.8055555, "accuracy_std_dev": 0.012626499999999985, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0015357878787878787, "avg_duration_per_task_ms": 249503.77525252526, "avg_output_tokens_per_task": 825.6868686868687, "last_run_timestamp": "2026-05-02T20:26:38.249Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.8052697686375321, "accuracy_std_dev": 0.05350092420872278, "run_count": 4, "total_questions": 778, "avg_cost_per_task_usd": 0.08564073007712082, "avg_duration_per_task_ms": 699962.8868894601, "avg_output_tokens_per_task": 25474.38560411311, "last_run_timestamp": "2026-09-05T09:27:38.360Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.8052215698062792, "accuracy_std_dev": 0.07724919018062586, "run_count": 16, "total_questions": 2994, "avg_cost_per_task_usd": 0.1436293523714095, "avg_duration_per_task_ms": 319160.0223780895, "avg_output_tokens_per_task": 29533.40046760187, "last_run_timestamp": "2026-09-19T09:11:52.131Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.804714, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.024081237373737373, "avg_duration_per_task_ms": 24354.015151515152, "avg_output_tokens_per_task": 1625.5555555555557, "last_run_timestamp": "2026-08-01T08:38:11.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.8045346501901139, "accuracy_std_dev": 0.045705742477164614, "run_count": 12, "total_questions": 2367, "avg_cost_per_task_usd": 0.07353121546261089, "avg_duration_per_task_ms": 676893.81791297, "avg_output_tokens_per_task": 28532.3481199831, "last_run_timestamp": "2026-09-19T16:47:45.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.803999108108108, "accuracy_std_dev": 0.01725557388613565, "run_count": 23, "total_questions": 4551, "avg_cost_per_task_usd": 0.016347447374203468, "avg_duration_per_task_ms": 162155.09140848165, "avg_output_tokens_per_task": 10785.491100856954, "last_run_timestamp": "2026-08-29T18:07:47.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.8039509370988447, "accuracy_std_dev": 0.04606482842170186, "run_count": 12, "total_questions": 2337, "avg_cost_per_task_usd": 0.004562442875481386, "avg_duration_per_task_ms": 529537.7351305092, "avg_output_tokens_per_task": 12022.224646983312, "last_run_timestamp": "2026-08-29T19:15:43.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.8036616249999999, "accuracy_std_dev": 0.02419106182010981, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.338616759469697, "avg_duration_per_task_ms": 355080.81313131313, "avg_output_tokens_per_task": 22428.503156565657, "last_run_timestamp": "2026-09-19T08:51:20.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8027495, "accuracy_std_dev": 0.02711787599075562, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.3324898122895622, "avg_duration_per_task_ms": 351546.73316498316, "avg_output_tokens_per_task": 22079.203703703704, "last_run_timestamp": "2026-09-19T08:49:06.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.8025378463620981, "accuracy_std_dev": 0.137748568405128, "run_count": 15, "total_questions": 2955, "avg_cost_per_task_usd": 0.004375704230118444, "avg_duration_per_task_ms": 379301.62267343485, "avg_output_tokens_per_task": 10823.406091370558, "last_run_timestamp": "2026-08-29T19:18:32.746Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.8025315620253164, "accuracy_std_dev": 0.041046181356000994, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.0376030194092827, "avg_duration_per_task_ms": 384444.73713080166, "avg_output_tokens_per_task": 37258.89915611815, "last_run_timestamp": "2026-09-19T16:23:22.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8023188347826088, "accuracy_std_dev": 0.008868734307792898, "run_count": 3, "total_questions": 575, "avg_cost_per_task_usd": 0.05968593391304348, "avg_duration_per_task_ms": 239074.65391304347, "avg_output_tokens_per_task": 15240.28, "last_run_timestamp": "2026-09-19T08:48:26.677Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.8022664747948416, "accuracy_std_dev": 0.03416052727843685, "run_count": 13, "total_questions": 2559, "avg_cost_per_task_usd": 0.10685159788980071, "avg_duration_per_task_ms": 697410.7983587339, "avg_output_tokens_per_task": 34433.65064478312, "last_run_timestamp": "2026-08-29T18:40:24.764Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8021885000000001, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04024864141414142, "avg_duration_per_task_ms": 73206.09595959596, "avg_output_tokens_per_task": 26709.396464646463, "last_run_timestamp": "2026-09-01T17:53:21.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.8020385792266869, "accuracy_std_dev": 0.0530296415355365, "run_count": 20, "total_questions": 3957, "avg_cost_per_task_usd": 0.003907585039171089, "avg_duration_per_task_ms": 215449.7402072277, "avg_output_tokens_per_task": 9523.579732120294, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DeepInfra", "accuracy": 0.8019233052236256, "accuracy_std_dev": 0.09230759113302772, "run_count": 15, "total_questions": 2929, "avg_cost_per_task_usd": 0.010292674291567086, "avg_duration_per_task_ms": 247238.68316831684, "avg_output_tokens_per_task": 8923.839877091157, "last_run_timestamp": "2026-07-11T15:51:15.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.8016302663043479, "accuracy_std_dev": 0.016866645256903415, "run_count": 6, "total_questions": 1104, "avg_cost_per_task_usd": 0.053515479166666664, "avg_duration_per_task_ms": 272256.60507246375, "avg_output_tokens_per_task": 12863.78713768116, "last_run_timestamp": "2026-07-18T08:30:38.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.8015763636363635, "accuracy_std_dev": 0.05357946338302784, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.143211629476584, "avg_duration_per_task_ms": 721714.1303948576, "avg_output_tokens_per_task": 46550.70293847567, "last_run_timestamp": "2026-09-19T09:25:42.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.80138718727491, "accuracy_std_dev": 0.013694984002920067, "run_count": 13, "total_questions": 2499, "avg_cost_per_task_usd": 0.009386521408563426, "avg_duration_per_task_ms": 310251.2785114046, "avg_output_tokens_per_task": 9778.357743097238, "last_run_timestamp": "2026-08-15T14:56:55.269Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8013465, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.041531338383838386, "avg_duration_per_task_ms": 80828.55555555556, "avg_output_tokens_per_task": 27572.47474747475, "last_run_timestamp": "2026-09-01T17:50:00.305Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.8011413739967898, "accuracy_std_dev": 0.026788015723950006, "run_count": 19, "total_questions": 3738, "avg_cost_per_task_usd": 0.010181774745853397, "avg_duration_per_task_ms": 304238.04440877476, "avg_output_tokens_per_task": 10343.559390048154, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.80042085, "accuracy_std_dev": 0.09910293924565254, "run_count": 20, "total_questions": 3960, "avg_cost_per_task_usd": 0.02063269419191919, "avg_duration_per_task_ms": 99441.73484848485, "avg_output_tokens_per_task": 15808.599242424243, "last_run_timestamp": "2026-08-29T19:13:10.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.799896012993763, "accuracy_std_dev": 0.11770242543873084, "run_count": 10, "total_questions": 1924, "avg_cost_per_task_usd": 0.006078587837837837, "avg_duration_per_task_ms": 630170.6242203743, "avg_output_tokens_per_task": 26380.861226611225, "last_run_timestamp": "2026-09-19T09:04:04.808Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7993420703124999, "accuracy_std_dev": 0.19162084340952992, "run_count": 14, "total_questions": 2432, "avg_cost_per_task_usd": 0.16367626192434215, "avg_duration_per_task_ms": 295251.3548519737, "avg_output_tokens_per_task": 28199.261924342107, "last_run_timestamp": "2026-09-20T10:34:11.013Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.7988215, "accuracy_std_dev": 0.010942500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.25381196717171717, "avg_duration_per_task_ms": 225693.02525252526, "avg_output_tokens_per_task": 25403.512626262625, "last_run_timestamp": "2026-09-01T13:22:15.928Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.7984771862955031, "accuracy_std_dev": 0.15755601228064434, "run_count": 17, "total_questions": 2802, "avg_cost_per_task_usd": 0.07837114346895076, "avg_duration_per_task_ms": 375817.0189150607, "avg_output_tokens_per_task": 18242.825124910778, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.79798, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2411195303030303, "avg_duration_per_task_ms": 184685.6616161616, "avg_output_tokens_per_task": 24018.91919191919, "last_run_timestamp": "2026-09-01T13:34:01.175Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.79798, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03602157070707071, "avg_duration_per_task_ms": 32858.843434343435, "avg_output_tokens_per_task": 2479.818181818182, "last_run_timestamp": "2026-08-01T08:32:36.352Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.7979798, "accuracy_std_dev": 0.09258320044241286, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010301554545454545, "avg_duration_per_task_ms": 568986.8757575757, "avg_output_tokens_per_task": 24756.19292929293, "last_run_timestamp": "2026-08-29T19:51:27.312Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7979040119760479, "accuracy_std_dev": 0.02389207889003172, "run_count": 4, "total_questions": 668, "avg_cost_per_task_usd": 0.15626441916167666, "avg_duration_per_task_ms": 1788980.868263473, "avg_output_tokens_per_task": 46933.58532934132, "last_run_timestamp": "2026-08-22T16:44:47.658Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.7976528739352641, "accuracy_std_dev": 0.01137849398787464, "run_count": 18, "total_questions": 3522, "avg_cost_per_task_usd": 0.03818711385576377, "avg_duration_per_task_ms": 869626.9395229983, "avg_output_tokens_per_task": 17421.532935831914, "last_run_timestamp": "2026-08-29T20:02:23.660Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.79727825, "accuracy_std_dev": 0.08339832626730288, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.08855118055555555, "avg_duration_per_task_ms": 505679.5707070707, "avg_output_tokens_per_task": 28090.66245791246, "last_run_timestamp": "2026-09-19T16:34:54.602Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.7971869670886074, "accuracy_std_dev": 0.03513963815261782, "run_count": 18, "total_questions": 3555, "avg_cost_per_task_usd": 0.031226099296765116, "avg_duration_per_task_ms": 382925.899859353, "avg_output_tokens_per_task": 15698.530239099859, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.7969371849642005, "accuracy_std_dev": 0.11399068476884858, "run_count": 13, "total_questions": 2514, "avg_cost_per_task_usd": 0.1032379212410501, "avg_duration_per_task_ms": 717491.7549721559, "avg_output_tokens_per_task": 30670.05369928401, "last_run_timestamp": "2026-08-29T19:03:21.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7964375923664123, "accuracy_std_dev": 0.11378331267012533, "run_count": 4, "total_questions": 655, "avg_cost_per_task_usd": 0.011028116030534352, "avg_duration_per_task_ms": 270893.28854961833, "avg_output_tokens_per_task": 23218.567938931297, "last_run_timestamp": "2026-09-12T08:53:48.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7962965, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.044405648989898995, "avg_duration_per_task_ms": 249218.0202020202, "avg_output_tokens_per_task": 22108.909090909092, "last_run_timestamp": "2026-09-01T18:00:27.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.796296, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04404542424242425, "avg_duration_per_task_ms": 221111.49494949495, "avg_output_tokens_per_task": 21928.032828282827, "last_run_timestamp": "2026-09-01T18:07:51.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.796213666369313, "accuracy_std_dev": 0.06626010818847784, "run_count": 17, "total_questions": 3363, "avg_cost_per_task_usd": 0.07408058043413619, "avg_duration_per_task_ms": 465867.30954504904, "avg_output_tokens_per_task": 23193.086827237585, "last_run_timestamp": "2026-08-22T17:17:31.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7956016344772546, "accuracy_std_dev": 0.026298589487996727, "run_count": 19, "total_questions": 3759, "avg_cost_per_task_usd": 0.009070496408619311, "avg_duration_per_task_ms": 398032.09071561584, "avg_output_tokens_per_task": 22891.514232508645, "last_run_timestamp": "2026-08-29T18:49:24.022Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.7955027719182596, "accuracy_std_dev": 0.03582202847933504, "run_count": 23, "total_questions": 4551, "avg_cost_per_task_usd": 0.010505668424522085, "avg_duration_per_task_ms": 330984.9740716326, "avg_output_tokens_per_task": 10159.019116677653, "last_run_timestamp": "2026-08-30T00:18:59.365Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.795170489361702, "accuracy_std_dev": 0.03970564182833467, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.03361843617021277, "avg_duration_per_task_ms": 752189.3389057751, "avg_output_tokens_per_task": 19508.68237082067, "last_run_timestamp": "2026-09-19T08:51:21.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.794507625, "accuracy_std_dev": 0.01694477015879457, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.010556673295454547, "avg_duration_per_task_ms": 162269.5918560606, "avg_output_tokens_per_task": 6275.243371212121, "last_run_timestamp": "2026-07-18T16:16:42.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7936028, "accuracy_std_dev": 0.016273367695716804, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.017573431313131313, "avg_duration_per_task_ms": 59390.138383838384, "avg_output_tokens_per_task": 5706.428282828283, "last_run_timestamp": "2026-08-29T18:05:48.084Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.7929361286060933, "accuracy_std_dev": 0.09695873156687938, "run_count": 19, "total_questions": 3709, "avg_cost_per_task_usd": 0.015247492315988135, "avg_duration_per_task_ms": 206557.6931787544, "avg_output_tokens_per_task": 15414.885144243732, "last_run_timestamp": "2026-08-29T20:00:43.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7929289999999999, "accuracy_std_dev": 0.016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03961820454545454, "avg_duration_per_task_ms": 540887.0883838384, "avg_output_tokens_per_task": 13782.598484848484, "last_run_timestamp": "2026-08-08T09:30:55.143Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7928263074433656, "accuracy_std_dev": 0.023434100042370905, "run_count": 17, "total_questions": 3090, "avg_cost_per_task_usd": 0.03394864174757281, "avg_duration_per_task_ms": 579008.496763754, "avg_output_tokens_per_task": 18524.763754045307, "last_run_timestamp": "2026-08-29T18:31:11.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.7910053138664271, "accuracy_std_dev": 0.15895006299418277, "run_count": 17, "total_questions": 3339, "avg_cost_per_task_usd": 0.011569411799940102, "avg_duration_per_task_ms": 139037.51362683438, "avg_output_tokens_per_task": 9589.577118897874, "last_run_timestamp": "2026-08-01T18:21:26.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.7909605118644066, "accuracy_std_dev": 0.012465345763974308, "run_count": 18, "total_questions": 3540, "avg_cost_per_task_usd": 0.03681927146892655, "avg_duration_per_task_ms": 671335.5409604519, "avg_output_tokens_per_task": 16802.62627118644, "last_run_timestamp": "2026-08-29T19:47:52.368Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7906814256756757, "accuracy_std_dev": 0.02348428081895062, "run_count": 6, "total_questions": 1184, "avg_cost_per_task_usd": 0.059051968749999996, "avg_duration_per_task_ms": 165937.2972972973, "avg_output_tokens_per_task": 18641.25929054054, "last_run_timestamp": "2026-09-19T09:21:23.718Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Fireworks", "accuracy": 0.7904040000000001, "accuracy_std_dev": 0.013598748159297637, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006979520202020202, "avg_duration_per_task_ms": 73170.09974747474, "avg_output_tokens_per_task": 4078.4949494949497, "last_run_timestamp": "2026-05-09T16:48:20.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.790404, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0189197702020202, "avg_duration_per_task_ms": 459328.30303030304, "avg_output_tokens_per_task": 25138.35606060606, "last_run_timestamp": "2026-08-29T19:17:42.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.790014, "accuracy_std_dev": 0.07589864878664765, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.016462866666666666, "avg_duration_per_task_ms": 474335.2464135021, "avg_output_tokens_per_task": 17867.735021097047, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7887197076142133, "accuracy_std_dev": 0.015079397638131601, "run_count": 15, "total_questions": 2955, "avg_cost_per_task_usd": 0.12096143113367173, "avg_duration_per_task_ms": 644827.7421319797, "avg_output_tokens_per_task": 30231.451776649745, "last_run_timestamp": "2026-08-29T20:08:40.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7884401666666667, "accuracy_std_dev": 0.023761946338748348, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.04315188552188553, "avg_duration_per_task_ms": 84212.74747474748, "avg_output_tokens_per_task": 26885.046296296296, "last_run_timestamp": "2026-09-01T17:55:54.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Morph", "accuracy": 0.7883901041069723, "accuracy_std_dev": 0.19387690688581088, "run_count": 16, "total_questions": 3141, "avg_cost_per_task_usd": 0.02842944317096465, "avg_duration_per_task_ms": 121865.52976758993, "avg_output_tokens_per_task": 8107.963387456224, "last_run_timestamp": "2026-07-04T18:14:56.425Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7882558207964601, "accuracy_std_dev": 0.04689700002782695, "run_count": 10, "total_questions": 1808, "avg_cost_per_task_usd": 0.00581368860619469, "avg_duration_per_task_ms": 499550.75, "avg_output_tokens_per_task": 13199.183628318584, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "AtlasCloud", "accuracy": 0.7879646298811546, "accuracy_std_dev": 0.023042252221345448, "run_count": 9, "total_questions": 1767, "avg_cost_per_task_usd": 0.023100724391624222, "avg_duration_per_task_ms": 400432.5291454443, "avg_output_tokens_per_task": 10710.642331635541, "last_run_timestamp": "2026-07-01T04:44:07.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7879190221533008, "accuracy_std_dev": 0.05144838052109178, "run_count": 12, "total_questions": 2257, "avg_cost_per_task_usd": 0.04170270757642889, "avg_duration_per_task_ms": 437887.0739920248, "avg_output_tokens_per_task": 20993.67213114754, "last_run_timestamp": "2026-09-19T10:02:08.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Chutes", "accuracy": 0.787879, "accuracy_std_dev": 0.010101000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004192080808080808, "avg_duration_per_task_ms": 367628.1565656566, "avg_output_tokens_per_task": 4131.532828282829, "last_run_timestamp": "2026-05-02T19:00:01.558Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7873578066075747, "accuracy_std_dev": 0.03703259018651595, "run_count": 19, "total_questions": 3723, "avg_cost_per_task_usd": 0.06184803760408272, "avg_duration_per_task_ms": 348744.5423045931, "avg_output_tokens_per_task": 18125.925597636317, "last_run_timestamp": "2026-08-29T18:54:50.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.7867736181619257, "accuracy_std_dev": 0.03349025786215252, "run_count": 14, "total_questions": 2742, "avg_cost_per_task_usd": 0.004660726112326768, "avg_duration_per_task_ms": 483380.77534646244, "avg_output_tokens_per_task": 11108.198395331874, "last_run_timestamp": "2026-08-01T18:07:34.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.7863751692015208, "accuracy_std_dev": 0.12285667045366319, "run_count": 15, "total_questions": 2630, "avg_cost_per_task_usd": 0.0199307855513308, "avg_duration_per_task_ms": 423150.044486692, "avg_output_tokens_per_task": 12234.871102661597, "last_run_timestamp": "2026-08-01T23:27:37.975Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.7862293764980027, "accuracy_std_dev": 0.1666251748521213, "run_count": 16, "total_questions": 3004, "avg_cost_per_task_usd": 0.00942010652463382, "avg_duration_per_task_ms": 248834.9277629827, "avg_output_tokens_per_task": 7858.000332889481, "last_run_timestamp": "2026-07-18T16:53:38.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "WandB", "accuracy": 0.7861595487316422, "accuracy_std_dev": 0.02054339937029649, "run_count": 4, "total_questions": 749, "avg_cost_per_task_usd": 0.023644552736982648, "avg_duration_per_task_ms": 116214.79706275034, "avg_output_tokens_per_task": 18836.48865153538, "last_run_timestamp": "2026-07-18T08:04:57.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7859751167002013, "accuracy_std_dev": 0.06046117739911878, "run_count": 23, "total_questions": 4473, "avg_cost_per_task_usd": 0.07098338184663536, "avg_duration_per_task_ms": 540645.976302258, "avg_output_tokens_per_task": 31720.033310976974, "last_run_timestamp": "2026-08-29T18:23:09.250Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.785920719858156, "accuracy_std_dev": 0.021394714621875175, "run_count": 13, "total_questions": 2538, "avg_cost_per_task_usd": 0.04721227226162333, "avg_duration_per_task_ms": 569592.5594956658, "avg_output_tokens_per_task": 20532.268321513002, "last_run_timestamp": "2026-08-29T16:48:15.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7856621441048034, "accuracy_std_dev": 0.00664559449898186, "run_count": 7, "total_questions": 1374, "avg_cost_per_task_usd": 0.018259436681222704, "avg_duration_per_task_ms": 150221.00873362445, "avg_output_tokens_per_task": 15062.550218340612, "last_run_timestamp": "2026-09-19T09:05:04.503Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.7853535, "accuracy_std_dev": 0.02272750000000001, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.002243229797979798, "avg_duration_per_task_ms": 474421.00757575757, "avg_output_tokens_per_task": 14645.75, "last_run_timestamp": "2026-06-20T12:44:34.391Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.7829818899297424, "accuracy_std_dev": 0.05788395901278767, "run_count": 13, "total_questions": 2562, "avg_cost_per_task_usd": 0.012435911787665888, "avg_duration_per_task_ms": 294718.2923497268, "avg_output_tokens_per_task": 12376.93169398907, "last_run_timestamp": "2026-08-29T16:52:23.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.7824072499999999, "accuracy_std_dev": 0.0034452088742919643, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.04613476767676768, "avg_duration_per_task_ms": 199616.12626262626, "avg_output_tokens_per_task": 21878.295454545456, "last_run_timestamp": "2026-09-01T19:10:24.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Friendli", "accuracy": 0.7820426666666666, "accuracy_std_dev": 0.18849682087457664, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.056621786195286204, "avg_duration_per_task_ms": 177410.45185185186, "avg_output_tokens_per_task": 17616.97272727273, "last_run_timestamp": "2026-07-01T03:02:14.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7820227306590256, "accuracy_std_dev": 0.04532488089752691, "run_count": 16, "total_questions": 3141, "avg_cost_per_task_usd": 0.03846658134352117, "avg_duration_per_task_ms": 642155.4489016237, "avg_output_tokens_per_task": 20074.63387456224, "last_run_timestamp": "2026-08-29T18:19:18.341Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.7819865, "accuracy_std_dev": 0.016169511173295674, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.252381888047138, "avg_duration_per_task_ms": 227878.92592592593, "avg_output_tokens_per_task": 25272.329124579126, "last_run_timestamp": "2026-09-01T13:35:35.064Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.7810580866666665, "accuracy_std_dev": 0.02055413818726277, "run_count": 16, "total_questions": 3150, "avg_cost_per_task_usd": 0.04090850158730158, "avg_duration_per_task_ms": 425804.21428571426, "avg_output_tokens_per_task": 18703.222222222223, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.7808641666666666, "accuracy_std_dev": 0.010351374971739527, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.021081101010101008, "avg_duration_per_task_ms": 286187.4873737374, "avg_output_tokens_per_task": 12540.748316498317, "last_run_timestamp": "2026-08-30T00:44:42.602Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.7805645299890948, "accuracy_std_dev": 0.02929019713259585, "run_count": 14, "total_questions": 2751, "avg_cost_per_task_usd": 0.004841514721919302, "avg_duration_per_task_ms": 633261.4282079244, "avg_output_tokens_per_task": 11578.931297709923, "last_run_timestamp": "2026-08-01T22:57:01.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.7794614999999999, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013140674242424242, "avg_duration_per_task_ms": 69975.75757575757, "avg_output_tokens_per_task": 10390.04797979798, "last_run_timestamp": "2026-09-01T17:51:03.252Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7783505180412369, "accuracy_std_dev": 0.01799990454372838, "run_count": 12, "total_questions": 2328, "avg_cost_per_task_usd": 0.012542361254295534, "avg_duration_per_task_ms": 614270.4780927835, "avg_output_tokens_per_task": 31292.053694158076, "last_run_timestamp": "2026-08-29T16:45:34.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7781144000000001, "accuracy_std_dev": 0.02296971973359712, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.20166651515151518, "avg_duration_per_task_ms": 657386.6868686868, "avg_output_tokens_per_task": 48003.93838383839, "last_run_timestamp": "2026-08-29T18:10:32.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7756354601542417, "accuracy_std_dev": 0.02561398969421896, "run_count": 12, "total_questions": 2334, "avg_cost_per_task_usd": 0.005398404884318766, "avg_duration_per_task_ms": 660748.7202227935, "avg_output_tokens_per_task": 33855.62853470437, "last_run_timestamp": "2026-09-19T17:12:41.248Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7753727142857143, "accuracy_std_dev": 0.16596390492747365, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01666139538239538, "avg_duration_per_task_ms": 580879.6435786436, "avg_output_tokens_per_task": 26434.363636363636, "last_run_timestamp": "2026-09-19T08:12:48.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.774832191940615, "accuracy_std_dev": 0.02630010557108088, "run_count": 10, "total_questions": 1886, "avg_cost_per_task_usd": 0.009038662778366914, "avg_duration_per_task_ms": 449340.6447507953, "avg_output_tokens_per_task": 36069.23966065748, "last_run_timestamp": "2026-09-19T17:14:53.367Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.7733209195128408, "accuracy_std_dev": 0.08870509019372362, "run_count": 21, "total_questions": 3777, "avg_cost_per_task_usd": 0.01586187847498014, "avg_duration_per_task_ms": 345375.779719354, "avg_output_tokens_per_task": 16140.08895949166, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7727855930680358, "accuracy_std_dev": 0.05706427864424032, "run_count": 12, "total_questions": 2337, "avg_cost_per_task_usd": 0.0018226653829696191, "avg_duration_per_task_ms": 478170.8801882756, "avg_output_tokens_per_task": 11197.934103551563, "last_run_timestamp": "2026-08-22T17:47:50.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Chutes", "accuracy": 0.7721462095719381, "accuracy_std_dev": 0.16044205147254612, "run_count": 18, "total_questions": 3364, "avg_cost_per_task_usd": 0.04585099791914388, "avg_duration_per_task_ms": 421607.292508918, "avg_output_tokens_per_task": 18189.730677764564, "last_run_timestamp": "2026-07-18T16:43:10.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7721407260997067, "accuracy_std_dev": 0.028224906684324803, "run_count": 11, "total_questions": 1705, "avg_cost_per_task_usd": 0.009176458064516129, "avg_duration_per_task_ms": 41666.574780058654, "avg_output_tokens_per_task": 12170.292082111437, "last_run_timestamp": "2026-08-29T19:07:43.372Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7715098846153846, "accuracy_std_dev": 0.09651313170593676, "run_count": 12, "total_questions": 2340, "avg_cost_per_task_usd": 0.09523880213675213, "avg_duration_per_task_ms": 1068926.4589743589, "avg_output_tokens_per_task": 33358.00341880342, "last_run_timestamp": "2026-08-29T18:41:58.319Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7714939999999999, "accuracy_std_dev": 0.029762842455652656, "run_count": 5, "total_questions": 946, "avg_cost_per_task_usd": 0.04608473361522198, "avg_duration_per_task_ms": 257712.6194503171, "avg_output_tokens_per_task": 17355.710359408033, "last_run_timestamp": "2026-09-19T09:19:09.687Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7714368913043479, "accuracy_std_dev": 0.014261246953428234, "run_count": 3, "total_questions": 552, "avg_cost_per_task_usd": 0.23206160869565215, "avg_duration_per_task_ms": 164099.60869565216, "avg_output_tokens_per_task": 15307.03079710145, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.7704213310344826, "accuracy_std_dev": 0.045856057411631505, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.0051286381609195394, "avg_duration_per_task_ms": 279309.07172413793, "avg_output_tokens_per_task": 33671.028045977015, "last_run_timestamp": "2026-09-19T16:37:35.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.7701633959390862, "accuracy_std_dev": 0.05197684190953593, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.007365169204737733, "avg_duration_per_task_ms": 438945.5972927242, "avg_output_tokens_per_task": 14498.280879864637, "last_run_timestamp": "2026-09-19T08:57:55.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7698924683870967, "accuracy_std_dev": 0.033487219191307406, "run_count": 12, "total_questions": 2325, "avg_cost_per_task_usd": 0.011860219354838712, "avg_duration_per_task_ms": 404374.8103225806, "avg_output_tokens_per_task": 29903.957849462364, "last_run_timestamp": "2026-08-29T16:43:11.936Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.7693605, "accuracy_std_dev": 0.13636349999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0882362904040404, "avg_duration_per_task_ms": 1341629.558080808, "avg_output_tokens_per_task": 22000.722222222223, "last_run_timestamp": "2026-08-30T15:00:10.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Phala", "accuracy": 0.7688491071428571, "accuracy_std_dev": 0.036143499999999995, "run_count": 2, "total_questions": 168, "avg_cost_per_task_usd": 0.013457690476190477, "avg_duration_per_task_ms": 239500.45238095237, "avg_output_tokens_per_task": 24312.886904761905, "last_run_timestamp": "2026-07-18T09:01:08.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.7685185, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013512386363636364, "avg_duration_per_task_ms": 71792.87121212122, "avg_output_tokens_per_task": 10688.070707070707, "last_run_timestamp": "2026-09-01T17:55:24.769Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.7680775714285716, "accuracy_std_dev": 0.014666364043105512, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.011148957431457432, "avg_duration_per_task_ms": 51087.674122174125, "avg_output_tokens_per_task": 18441.45286195286, "last_run_timestamp": "2026-08-29T18:36:51.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7679573333333333, "accuracy_std_dev": 0.010259553071270784, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.016145424242424242, "avg_duration_per_task_ms": 98341.22727272728, "avg_output_tokens_per_task": 6033.04797979798, "last_run_timestamp": "2026-09-19T08:57:22.040Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.76767675, "accuracy_std_dev": 0.03241562151667774, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.012590233585858583, "avg_duration_per_task_ms": 407671.7121212121, "avg_output_tokens_per_task": 21713.55460858586, "last_run_timestamp": "2026-08-29T18:36:10.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.767378624023886, "accuracy_std_dev": 0.043028318846837645, "run_count": 22, "total_questions": 4354, "avg_cost_per_task_usd": 0.012152718649517686, "avg_duration_per_task_ms": 79257.83968764354, "avg_output_tokens_per_task": 7012.955672944419, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.7672560000000002, "accuracy_std_dev": 0.0089379450938121, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013739436868686868, "avg_duration_per_task_ms": 117371.45833333333, "avg_output_tokens_per_task": 10352.156565656565, "last_run_timestamp": "2026-09-01T17:58:15.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7672559999999999, "accuracy_std_dev": 0.008469894233892938, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.27037940151515155, "avg_duration_per_task_ms": 263755.6675084175, "avg_output_tokens_per_task": 17837.74579124579, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.01935999999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2670539015151515, "avg_duration_per_task_ms": 177095.6616161616, "avg_output_tokens_per_task": 17616.689393939392, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.7668169148936171, "accuracy_std_dev": 0.053587367069901595, "run_count": 8, "total_questions": 1551, "avg_cost_per_task_usd": 0.010489795615731785, "avg_duration_per_task_ms": 700478.4300451322, "avg_output_tokens_per_task": 31633.707930367505, "last_run_timestamp": "2026-09-12T09:07:23.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05139547999999999, "avg_duration_per_task_ms": 172458.62, "avg_output_tokens_per_task": 21216.74, "last_run_timestamp": "2026-09-14T19:42:46.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.76638602247191, "accuracy_std_dev": 0.0119351260850762, "run_count": 3, "total_questions": 356, "avg_cost_per_task_usd": 0.002961258426966292, "avg_duration_per_task_ms": 69864.89325842696, "avg_output_tokens_per_task": 3595.182584269663, "last_run_timestamp": "2026-08-15T14:54:48.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.765993, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08016306060606061, "avg_duration_per_task_ms": 425226.22222222225, "avg_output_tokens_per_task": 69822.58585858585, "last_run_timestamp": "2026-07-18T08:29:23.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7659437959183671, "accuracy_std_dev": 0.0759970281637527, "run_count": 12, "total_questions": 2352, "avg_cost_per_task_usd": 0.004940951955782313, "avg_duration_per_task_ms": 720500.3579931973, "avg_output_tokens_per_task": 32852.24107142857, "last_run_timestamp": "2026-09-19T18:05:31.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.765512238095238, "accuracy_std_dev": 0.012007319862344141, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.011864828763828762, "avg_duration_per_task_ms": 112663.68951418951, "avg_output_tokens_per_task": 19644.012265512265, "last_run_timestamp": "2026-08-29T20:01:12.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "WandB", "accuracy": 0.7651513333333334, "accuracy_std_dev": 0.022539240875021092, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00031351936026936025, "avg_duration_per_task_ms": 33882.18939393939, "avg_output_tokens_per_task": 2865.460437710438, "last_run_timestamp": "2026-06-20T16:16:12.961Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.7642404121475055, "accuracy_std_dev": 0.012311945545143383, "run_count": 21, "total_questions": 4149, "avg_cost_per_task_usd": 0.009483342251144855, "avg_duration_per_task_ms": 98709.46348517716, "avg_output_tokens_per_task": 18943.602795854422, "last_run_timestamp": "2026-08-29T18:39:41.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.7638135731310943, "accuracy_std_dev": 0.022048436994096723, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.005248553990610328, "avg_duration_per_task_ms": 264357.4124232575, "avg_output_tokens_per_task": 16758.418201516793, "last_run_timestamp": "2026-07-18T16:28:34.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7636680000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.14155542328042328, "avg_duration_per_task_ms": 2288029.2486772486, "avg_output_tokens_per_task": 37245.06349206349, "last_run_timestamp": "2026-08-23T18:19:25.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "SiliconFlow", "accuracy": 0.7626265, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005752090909090909, "avg_duration_per_task_ms": 193402.83585858587, "avg_output_tokens_per_task": 2823.3358585858587, "last_run_timestamp": "2026-05-09T09:35:10.004Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "AtlasCloud", "accuracy": 0.7613916000000002, "accuracy_std_dev": 0.18676951595653934, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.005808141077441077, "avg_duration_per_task_ms": 42452.30505050505, "avg_output_tokens_per_task": 6068.625925925926, "last_run_timestamp": "2026-07-01T02:27:21.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.760943, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08179280303030302, "avg_duration_per_task_ms": 426340.89898989897, "avg_output_tokens_per_task": 70994.31818181818, "last_run_timestamp": "2026-07-18T08:26:55.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.7608648513980522, "accuracy_std_dev": 0.21997558889516092, "run_count": 19, "total_questions": 3183, "avg_cost_per_task_usd": 0.04201378196669809, "avg_duration_per_task_ms": 59751.10775997487, "avg_output_tokens_per_task": 14315.729814640277, "last_run_timestamp": "2026-08-15T14:54:40.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.760674964876033, "accuracy_std_dev": 0.01645419404771925, "run_count": 5, "total_questions": 968, "avg_cost_per_task_usd": 0.010708528925619836, "avg_duration_per_task_ms": 425619.78925619833, "avg_output_tokens_per_task": 25673.311983471074, "last_run_timestamp": "2026-08-29T18:56:40.728Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.7606652169197398, "accuracy_std_dev": 0.04591514075032818, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.004993729573391178, "avg_duration_per_task_ms": 317882.5820679682, "avg_output_tokens_per_task": 12242.755603759942, "last_run_timestamp": "2026-08-29T18:46:19.350Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.7606466373507059, "accuracy_std_dev": 0.023479472398289566, "run_count": 14, "total_questions": 2763, "avg_cost_per_task_usd": 0.010367346362649294, "avg_duration_per_task_ms": 144206.35830618892, "avg_output_tokens_per_task": 4038.1357220412597, "last_run_timestamp": "2026-08-01T17:44:11.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.7605543076923077, "accuracy_std_dev": 0.017867230930843605, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.010017935897435899, "avg_duration_per_task_ms": 107941.90792540793, "avg_output_tokens_per_task": 3732.981351981352, "last_run_timestamp": "2026-08-01T17:45:27.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.7605443265306122, "accuracy_std_dev": 0.0343317834601635, "run_count": 3, "total_questions": 490, "avg_cost_per_task_usd": 0.0037374408163265305, "avg_duration_per_task_ms": 668153.0571428571, "avg_output_tokens_per_task": 14206.583673469388, "last_run_timestamp": "2026-08-08T11:42:26.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7605394662379422, "accuracy_std_dev": 0.10958240222843778, "run_count": 19, "total_questions": 3732, "avg_cost_per_task_usd": 0.002762414523043944, "avg_duration_per_task_ms": 395715.09458735265, "avg_output_tokens_per_task": 14065.52706323687, "last_run_timestamp": "2026-08-29T19:07:25.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.7595521475625824, "accuracy_std_dev": 0.019571763499602556, "run_count": 8, "total_questions": 1518, "avg_cost_per_task_usd": 0.013668990118577072, "avg_duration_per_task_ms": 155525.03227931488, "avg_output_tokens_per_task": 18166.727272727272, "last_run_timestamp": "2026-08-29T18:11:51.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.759259, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018210444444444442, "avg_duration_per_task_ms": 254627.46717171717, "avg_output_tokens_per_task": 24079.401515151516, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7590111668383109, "accuracy_std_dev": 0.016611473377157127, "run_count": 5, "total_questions": 971, "avg_cost_per_task_usd": 0.004071515962924819, "avg_duration_per_task_ms": 4054.4541709577757, "avg_output_tokens_per_task": 2252.576725025747, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7587190035005834, "accuracy_std_dev": 0.01366534537530361, "run_count": 26, "total_questions": 5142, "avg_cost_per_task_usd": 0.015162635355892653, "avg_duration_per_task_ms": 166747.76215480358, "avg_output_tokens_per_task": 25091.703033838974, "last_run_timestamp": "2026-08-29T16:18:21.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.7585374615196078, "accuracy_std_dev": 0.038599957465404885, "run_count": 21, "total_questions": 4080, "avg_cost_per_task_usd": 0.01587172034313725, "avg_duration_per_task_ms": 44875.42181372549, "avg_output_tokens_per_task": 16664.9625, "last_run_timestamp": "2026-08-29T19:51:33.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7577766369409167, "accuracy_std_dev": 0.02573710862237637, "run_count": 20, "total_questions": 3622, "avg_cost_per_task_usd": 0.00409766344561016, "avg_duration_per_task_ms": 651759.315847598, "avg_output_tokens_per_task": 10571.302319160684, "last_run_timestamp": "2026-08-29T19:19:08.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.7569231911314984, "accuracy_std_dev": 0.07227409752980317, "run_count": 20, "total_questions": 3924, "avg_cost_per_task_usd": 0.006370367991845058, "avg_duration_per_task_ms": 612171.6885830784, "avg_output_tokens_per_task": 7236.809887869521, "last_run_timestamp": "2026-08-30T01:20:44.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.7565450000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.020916916230366494, "avg_duration_per_task_ms": 124793.11518324607, "avg_output_tokens_per_task": 18115.47120418848, "last_run_timestamp": "2026-07-18T08:26:49.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7559323627118644, "accuracy_std_dev": 0.007798453236949536, "run_count": 3, "total_questions": 590, "avg_cost_per_task_usd": 0.002668813559322034, "avg_duration_per_task_ms": 91653.82203389831, "avg_output_tokens_per_task": 31189.77966101695, "last_run_timestamp": "2026-09-01T13:23:13.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7559212403433475, "accuracy_std_dev": 0.12618386614116156, "run_count": 11, "total_questions": 2097, "avg_cost_per_task_usd": 0.07370895374344302, "avg_duration_per_task_ms": 833401.029089175, "avg_output_tokens_per_task": 27619.250357653793, "last_run_timestamp": "2026-09-19T17:25:48.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.755892, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0020983333333333336, "avg_duration_per_task_ms": 164183.51515151514, "avg_output_tokens_per_task": 7127.272727272727, "last_run_timestamp": "2026-08-21T21:34:38.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7552889455999998, "accuracy_std_dev": 0.020781831603270767, "run_count": 19, "total_questions": 3750, "avg_cost_per_task_usd": 0.010816569600000001, "avg_duration_per_task_ms": 208301.1416, "avg_output_tokens_per_task": 18668.501333333334, "last_run_timestamp": "2026-08-29T18:30:03.855Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.7551707142857145, "accuracy_std_dev": 0.008190904148841594, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02236280663780664, "avg_duration_per_task_ms": 98622.52741702742, "avg_output_tokens_per_task": 14763.670274170274, "last_run_timestamp": "2026-08-29T19:17:49.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.7551499775280899, "accuracy_std_dev": 0.03338976368963997, "run_count": 6, "total_questions": 1068, "avg_cost_per_task_usd": 0.006923780898876404, "avg_duration_per_task_ms": 301563.05337078654, "avg_output_tokens_per_task": 21238.912921348314, "last_run_timestamp": "2026-07-18T08:04:08.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.7533753845344554, "accuracy_std_dev": 0.11640605414937877, "run_count": 11, "total_questions": 1901, "avg_cost_per_task_usd": 0.04040272172540768, "avg_duration_per_task_ms": 597850.3240399789, "avg_output_tokens_per_task": 17550.840610205156, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7525250000000001, "accuracy_std_dev": 0.010101000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.056976058080808085, "avg_duration_per_task_ms": 98919.65151515152, "avg_output_tokens_per_task": 22701.11111111111, "last_run_timestamp": "2026-09-01T17:54:55.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Inceptron", "accuracy": 0.7518907958115183, "accuracy_std_dev": 0.056756617575006676, "run_count": 6, "total_questions": 1146, "avg_cost_per_task_usd": 0.0764649572425829, "avg_duration_per_task_ms": 465218.3132635253, "avg_output_tokens_per_task": 17503.08987783595, "last_run_timestamp": "2026-06-30T23:33:15.910Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.75178689122807, "accuracy_std_dev": 0.06483392638966569, "run_count": 13, "total_questions": 2565, "avg_cost_per_task_usd": 0.01753287212475633, "avg_duration_per_task_ms": 827358.4358674464, "avg_output_tokens_per_task": 10936.669005847953, "last_run_timestamp": "2026-08-22T17:28:27.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.7515384892307692, "accuracy_std_dev": 0.10704096328246257, "run_count": 8, "total_questions": 650, "avg_cost_per_task_usd": 0.002935932307692308, "avg_duration_per_task_ms": 200717.69076923077, "avg_output_tokens_per_task": 13572.203076923077, "last_run_timestamp": "2026-09-12T16:00:20.960Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.7510363518653986, "accuracy_std_dev": 0.11063154442009523, "run_count": 7, "total_questions": 1367, "avg_cost_per_task_usd": 0.05419231089978055, "avg_duration_per_task_ms": 442904.11777615215, "avg_output_tokens_per_task": 22733.470373079737, "last_run_timestamp": "2026-08-15T10:07:37.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.750842, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01581958585858586, "avg_duration_per_task_ms": 326624.7373737374, "avg_output_tokens_per_task": 13062.823232323231, "last_run_timestamp": "2026-09-12T09:42:09.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.7504692996781979, "accuracy_std_dev": 0.12884102665725183, "run_count": 13, "total_questions": 2486, "avg_cost_per_task_usd": 0.03235196661303299, "avg_duration_per_task_ms": 504543.08045052295, "avg_output_tokens_per_task": 14950.808125502816, "last_run_timestamp": "2026-08-29T16:22:26.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "AtlasCloud", "accuracy": 0.7496631999999999, "accuracy_std_dev": 0.012666643389627747, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.00575224191919192, "avg_duration_per_task_ms": 96461.17525252525, "avg_output_tokens_per_task": 2226.588888888889, "last_run_timestamp": "2026-07-01T05:04:09.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Cerebras", "accuracy": 0.7494948, "accuracy_std_dev": 0.018515427928082036, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0035561919191919194, "avg_duration_per_task_ms": 3605.560606060606, "avg_output_tokens_per_task": 2828.088888888889, "last_run_timestamp": "2026-05-23T08:06:03.679Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.7493782527075812, "accuracy_std_dev": 0.028699053688207896, "run_count": 21, "total_questions": 4155, "avg_cost_per_task_usd": 0.007139869554753309, "avg_duration_per_task_ms": 177604.2055354994, "avg_output_tokens_per_task": 14341.533814681106, "last_run_timestamp": "2026-08-29T19:17:18.768Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7485970000000001, "accuracy_std_dev": 0.07972918647312704, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006700525252525253, "avg_duration_per_task_ms": 93845.37542087543, "avg_output_tokens_per_task": 16482.319865319867, "last_run_timestamp": "2026-08-29T18:45:38.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7485421870794079, "accuracy_std_dev": 0.10163071553147703, "run_count": 5, "total_questions": 743, "avg_cost_per_task_usd": 0.014177663526244952, "avg_duration_per_task_ms": 490544.0363391655, "avg_output_tokens_per_task": 26085.409152086137, "last_run_timestamp": "2026-09-15T00:30:59.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.7479971828113618, "accuracy_std_dev": 0.07910913250219596, "run_count": 14, "total_questions": 2746, "avg_cost_per_task_usd": 0.006062376911871813, "avg_duration_per_task_ms": 285154.7793153678, "avg_output_tokens_per_task": 21130.692279679533, "last_run_timestamp": "2026-08-29T20:02:35.387Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Together", "accuracy": 0.7474749999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.004966555555555556, "avg_duration_per_task_ms": 39085.368686868685, "avg_output_tokens_per_task": 4025.959595959596, "last_run_timestamp": "2026-05-09T08:06:12.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.7474749999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08105034848484848, "avg_duration_per_task_ms": 422601, "avg_output_tokens_per_task": 70367.68181818182, "last_run_timestamp": "2026-07-18T08:27:39.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.7473080875303925, "accuracy_std_dev": 0.15339754617572723, "run_count": 16, "total_questions": 2879, "avg_cost_per_task_usd": 0.06448924591872178, "avg_duration_per_task_ms": 699576.7933310177, "avg_output_tokens_per_task": 22910.511983327546, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.7472975263157895, "accuracy_std_dev": 0.014827052609720271, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.0034353149920255174, "avg_duration_per_task_ms": 77071.67570441254, "avg_output_tokens_per_task": 5605.975544922913, "last_run_timestamp": "2026-08-29T18:15:48.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.7467733333333335, "accuracy_std_dev": 0.012587939799224065, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.011146377104377103, "avg_duration_per_task_ms": 81852.9377104377, "avg_output_tokens_per_task": 12261.164983164983, "last_run_timestamp": "2026-08-29T18:17:28.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7465941264367815, "accuracy_std_dev": 0.10532480903970348, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.04816395977011494, "avg_duration_per_task_ms": 468882.4540229885, "avg_output_tokens_per_task": 36470.09642401022, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7464074370245138, "accuracy_std_dev": 0.026169100495032988, "run_count": 18, "total_questions": 3549, "avg_cost_per_task_usd": 0.10053423443223442, "avg_duration_per_task_ms": 535768.9887292195, "avg_output_tokens_per_task": 31443.99887292195, "last_run_timestamp": "2026-08-29T18:15:24.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.7463525, "accuracy_std_dev": 0.013479678966050276, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.010016893658810323, "avg_duration_per_task_ms": 77371.40852974186, "avg_output_tokens_per_task": 9879.324354657689, "last_run_timestamp": "2026-08-29T18:11:58.511Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7462122499999999, "accuracy_std_dev": 0.013222542538691302, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0095508547979798, "avg_duration_per_task_ms": 202963.16287878787, "avg_output_tokens_per_task": 8137.35101010101, "last_run_timestamp": "2026-08-29T18:16:24.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7460692307032589, "accuracy_std_dev": 0.11382303442613109, "run_count": 12, "total_questions": 2332, "avg_cost_per_task_usd": 0.03359197898799315, "avg_duration_per_task_ms": 1194136.795883362, "avg_output_tokens_per_task": 13708.225986277874, "last_run_timestamp": "2026-08-29T20:16:12.181Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.7455108333333333, "accuracy_std_dev": 0.020304992673532824, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.08322730471380473, "avg_duration_per_task_ms": 907303.9099326599, "avg_output_tokens_per_task": 23781.781144781144, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.7450696428571427, "accuracy_std_dev": 0.014631101665420764, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.00583946608946609, "avg_duration_per_task_ms": 147663.13888888888, "avg_output_tokens_per_task": 6788.261904761905, "last_run_timestamp": "2026-08-29T18:27:42.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.7449942438162545, "accuracy_std_dev": 0.01819967836712137, "run_count": 3, "total_questions": 283, "avg_cost_per_task_usd": 0.01679327561837456, "avg_duration_per_task_ms": 97120.16254416961, "avg_output_tokens_per_task": 9655.678445229682, "last_run_timestamp": "2026-08-30T00:01:09.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.744804408618582, "accuracy_std_dev": 0.052815519367177405, "run_count": 18, "total_questions": 3272, "avg_cost_per_task_usd": 0.07897924847188266, "avg_duration_per_task_ms": 486054.8694987775, "avg_output_tokens_per_task": 22570.938569682152, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.7427059912472648, "accuracy_std_dev": 0.04677682194604984, "run_count": 14, "total_questions": 2742, "avg_cost_per_task_usd": 0.008213210430342814, "avg_duration_per_task_ms": 681410.7862873814, "avg_output_tokens_per_task": 20787.104668125456, "last_run_timestamp": "2026-08-22T10:02:15.507Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.7426647142857141, "accuracy_std_dev": 0.007278926328092368, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0053309747474747475, "avg_duration_per_task_ms": 143246.7821067821, "avg_output_tokens_per_task": 8614.1443001443, "last_run_timestamp": "2026-08-29T18:49:21.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Wafer", "accuracy": 0.7424240000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.048281424242424244, "avg_duration_per_task_ms": 987580.0707070708, "avg_output_tokens_per_task": 21326.60606060606, "last_run_timestamp": "2026-06-13T08:04:32.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.7399168897876645, "accuracy_std_dev": 0.01871797550235247, "run_count": 15, "total_questions": 2967, "avg_cost_per_task_usd": 0.003766104819683181, "avg_duration_per_task_ms": 178216.75362318842, "avg_output_tokens_per_task": 5955.366363329963, "last_run_timestamp": "2026-08-29T18:12:05.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.7398376524390244, "accuracy_std_dev": 0.030475562908008763, "run_count": 5, "total_questions": 984, "avg_cost_per_task_usd": 0.004495964430894309, "avg_duration_per_task_ms": 781618.3089430894, "avg_output_tokens_per_task": 27377.12906504065, "last_run_timestamp": "2026-08-29T18:16:54.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Groq", "accuracy": 0.7397306666666669, "accuracy_std_dev": 0.023056894603470685, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.009337984175084173, "avg_duration_per_task_ms": 13785.80404040404, "avg_output_tokens_per_task": 2990.9208754208753, "last_run_timestamp": "2026-08-01T17:45:49.025Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.7393956351351351, "accuracy_std_dev": 0.03984266574715486, "run_count": 18, "total_questions": 3552, "avg_cost_per_task_usd": 0.00809140793918919, "avg_duration_per_task_ms": 458526.92088963964, "avg_output_tokens_per_task": 21780.53462837838, "last_run_timestamp": "2026-08-29T19:32:52.576Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.7388981686340642, "accuracy_std_dev": 0.02112011938163336, "run_count": 18, "total_questions": 3558, "avg_cost_per_task_usd": 0.003153482855536819, "avg_duration_per_task_ms": 127292.86734120292, "avg_output_tokens_per_task": 5353.168071950534, "last_run_timestamp": "2026-08-29T18:16:39.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7388467499999999, "accuracy_std_dev": 0.07747406520692911, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01745328409090909, "avg_duration_per_task_ms": 2275674.0732323234, "avg_output_tokens_per_task": 17131.02398989899, "last_run_timestamp": "2026-08-22T18:24:57.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.738764951772013, "accuracy_std_dev": 0.25012599709265915, "run_count": 14, "total_questions": 2737, "avg_cost_per_task_usd": 0.060706641213006944, "avg_duration_per_task_ms": 511177.83814395324, "avg_output_tokens_per_task": 17354.046401169162, "last_run_timestamp": "2026-08-29T18:36:08.021Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.7384959999999999, "accuracy_std_dev": 0.0076941959727230055, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.058646202861952865, "avg_duration_per_task_ms": 123198.74579124579, "avg_output_tokens_per_task": 23489.968855218856, "last_run_timestamp": "2026-09-01T17:53:51.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.7384040983827492, "accuracy_std_dev": 0.01718365966906416, "run_count": 30, "total_questions": 5936, "avg_cost_per_task_usd": 0.005306772237196765, "avg_duration_per_task_ms": 67835.73685983827, "avg_output_tokens_per_task": 5569.043800539083, "last_run_timestamp": "2026-08-29T19:15:44.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.7381102500000001, "accuracy_std_dev": 0.012944321677380398, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.015707310606060607, "avg_duration_per_task_ms": 67722.01136363637, "avg_output_tokens_per_task": 10394.812184343435, "last_run_timestamp": "2026-08-29T18:28:10.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Together", "accuracy": 0.7380050000000001, "accuracy_std_dev": 0.01915680847636162, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0026956635101010105, "avg_duration_per_task_ms": 123729.69255050505, "avg_output_tokens_per_task": 4372.988005050505, "last_run_timestamp": "2026-07-04T17:56:44.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.7373739999999999, "accuracy_std_dev": 0.010101000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001737270202020202, "avg_duration_per_task_ms": 127098.16161616161, "avg_output_tokens_per_task": 2854.8282828282827, "last_run_timestamp": "2026-05-09T09:51:37.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Nebius", "accuracy": 0.7373736666666667, "accuracy_std_dev": 0.18223849985542448, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04463949831649832, "avg_duration_per_task_ms": 228965.81313131313, "avg_output_tokens_per_task": 14731.36026936027, "last_run_timestamp": "2026-06-30T21:13:44.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.736111, "accuracy_std_dev": null, "run_count": 1, "total_questions": 168, "avg_cost_per_task_usd": 0.05987852380952381, "avg_duration_per_task_ms": 1235915.3511904762, "avg_output_tokens_per_task": 62102.07738095238, "last_run_timestamp": "2026-07-18T08:29:05.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7358002289156627, "accuracy_std_dev": 0.04992948859920135, "run_count": 9, "total_questions": 1743, "avg_cost_per_task_usd": 0.0068313775100401625, "avg_duration_per_task_ms": 1022403.1669535284, "avg_output_tokens_per_task": 18090.09982788296, "last_run_timestamp": "2026-08-29T20:11:32.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.7357838333333334, "accuracy_std_dev": 0.03274764050745574, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.004765584175084174, "avg_duration_per_task_ms": 89248.37345679013, "avg_output_tokens_per_task": 5293.844837261504, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.735337781737194, "accuracy_std_dev": 0.017059479919011102, "run_count": 14, "total_questions": 2694, "avg_cost_per_task_usd": 0.0013270267260579063, "avg_duration_per_task_ms": 313987.4647364514, "avg_output_tokens_per_task": 7917.988864142539, "last_run_timestamp": "2026-07-18T16:27:55.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Fireworks", "accuracy": 0.7349234689468948, "accuracy_std_dev": 0.22406624102384445, "run_count": 6, "total_questions": 1111, "avg_cost_per_task_usd": 0.07603830513051305, "avg_duration_per_task_ms": 255861.0288028803, "avg_output_tokens_per_task": 17676.750675067506, "last_run_timestamp": "2026-07-18T08:32:17.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.7338664379562044, "accuracy_std_dev": 0.09881106660279618, "run_count": 15, "total_questions": 2877, "avg_cost_per_task_usd": 0.12776717935349324, "avg_duration_per_task_ms": 902541.7400069517, "avg_output_tokens_per_task": 30238.822036843936, "last_run_timestamp": "2026-08-29T18:42:16.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7330631323430115, "accuracy_std_dev": 0.07952342617106217, "run_count": 8, "total_questions": 1481, "avg_cost_per_task_usd": 0.03091582241728561, "avg_duration_per_task_ms": 150438.8203916273, "avg_output_tokens_per_task": 27968.933153274815, "last_run_timestamp": "2026-08-29T19:16:48.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7321035652173912, "accuracy_std_dev": 0.012294195363546603, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.20428436868686872, "avg_duration_per_task_ms": 241980.82740447958, "avg_output_tokens_per_task": 38170.10320597277, "last_run_timestamp": "2026-09-19T08:49:06.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.7315366610778443, "accuracy_std_dev": 0.23240685150709303, "run_count": 5, "total_questions": 835, "avg_cost_per_task_usd": 0.05666666227544911, "avg_duration_per_task_ms": 351949.8934131737, "avg_output_tokens_per_task": 18214.095808383234, "last_run_timestamp": "2026-08-15T08:47:51.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.731313, "accuracy_std_dev": 0.03591182134618071, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00554679898989899, "avg_duration_per_task_ms": 199026.8303030303, "avg_output_tokens_per_task": 11014.675757575758, "last_run_timestamp": "2026-05-23T09:41:28.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "AtlasCloud", "accuracy": 0.7310988181818181, "accuracy_std_dev": 0.012035232012857314, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.006938019742883379, "avg_duration_per_task_ms": 43950.472910927456, "avg_output_tokens_per_task": 5102.792011019284, "last_run_timestamp": "2026-07-01T02:32:36.197Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "WandB", "accuracy": 0.7296560582226762, "accuracy_std_dev": 0.15560230205957754, "run_count": 5, "total_questions": 979, "avg_cost_per_task_usd": 0.01869288764044944, "avg_duration_per_task_ms": 96519.51685393258, "avg_output_tokens_per_task": 14860.048008171603, "last_run_timestamp": "2026-07-18T16:00:08.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.729592, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.0030441326530612246, "avg_duration_per_task_ms": 200954.75, "avg_output_tokens_per_task": 31361.45408163265, "last_run_timestamp": "2026-09-01T13:20:55.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.729377, "accuracy_std_dev": 0.01822436147029575, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.18529492234848485, "avg_duration_per_task_ms": 288734.0814393939, "avg_output_tokens_per_task": 36901.592803030304, "last_run_timestamp": "2026-09-19T08:54:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7272725, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.058774820707070706, "avg_duration_per_task_ms": 121424.5, "avg_output_tokens_per_task": 23508.14898989899, "last_run_timestamp": "2026-09-01T17:49:53.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.726431125, "accuracy_std_dev": 0.010170955675814077, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.19067559027777775, "avg_duration_per_task_ms": 240794.80744949495, "avg_output_tokens_per_task": 37944.43686868687, "last_run_timestamp": "2026-09-19T08:48:18.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7263065279007972, "accuracy_std_dev": 0.1217642983358514, "run_count": 6, "total_questions": 1129, "avg_cost_per_task_usd": 0.2174638077945084, "avg_duration_per_task_ms": 1661844.5757307352, "avg_output_tokens_per_task": 59283.423383525245, "last_run_timestamp": "2026-08-22T09:29:12.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.7261392373678025, "accuracy_std_dev": 0.007451069289848932, "run_count": 13, "total_questions": 2553, "avg_cost_per_task_usd": 0.0023729573051312187, "avg_duration_per_task_ms": 400351.9271445358, "avg_output_tokens_per_task": 6460.609479044261, "last_run_timestamp": "2026-08-29T18:39:40.767Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.7260814529667148, "accuracy_std_dev": 0.020878527218346984, "run_count": 21, "total_questions": 4146, "avg_cost_per_task_usd": 0.004243377713458756, "avg_duration_per_task_ms": 227026.06222865413, "avg_output_tokens_per_task": 16857.46454413893, "last_run_timestamp": "2026-08-29T18:36:32.803Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7249342617387262, "accuracy_std_dev": 0.20867571941350305, "run_count": 12, "total_questions": 2151, "avg_cost_per_task_usd": 0.004015295676429567, "avg_duration_per_task_ms": 907421.850302185, "avg_output_tokens_per_task": 26470.74058577406, "last_run_timestamp": "2026-09-19T20:39:06.442Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7244675919165579, "accuracy_std_dev": 0.009807494652528479, "run_count": 8, "total_questions": 1534, "avg_cost_per_task_usd": 0.18618542242503258, "avg_duration_per_task_ms": 349648.9543676662, "avg_output_tokens_per_task": 37047.215775749675, "last_run_timestamp": "2026-09-19T08:49:55.182Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.7230691676829271, "accuracy_std_dev": 0.01328152332415223, "run_count": 10, "total_questions": 1968, "avg_cost_per_task_usd": 0.005270103150406503, "avg_duration_per_task_ms": 358290.0543699187, "avg_output_tokens_per_task": 7038.59349593496, "last_run_timestamp": "2026-08-22T16:50:17.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7223346, "accuracy_std_dev": 0.07253641034202524, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.1090764063973064, "avg_duration_per_task_ms": 826522.4053872054, "avg_output_tokens_per_task": 27681.187205387207, "last_run_timestamp": "2026-08-29T19:12:26.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.7219408075949366, "accuracy_std_dev": 0.025598999999999983, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.27939689620253166, "avg_duration_per_task_ms": 450249.90379746835, "avg_output_tokens_per_task": 45941.00253164557, "last_run_timestamp": "2026-08-15T08:16:15.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Phala", "accuracy": 0.7198153664772725, "accuracy_std_dev": 0.16422235834886253, "run_count": 15, "total_questions": 2816, "avg_cost_per_task_usd": 0.008276323508522728, "avg_duration_per_task_ms": 230801.09623579544, "avg_output_tokens_per_task": 6590.78515625, "last_run_timestamp": "2026-07-18T16:47:35.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7197963406113538, "accuracy_std_dev": 0.10733758179611058, "run_count": 7, "total_questions": 1374, "avg_cost_per_task_usd": 0.00507840538573508, "avg_duration_per_task_ms": 429791.4192139738, "avg_output_tokens_per_task": 22080.005094614266, "last_run_timestamp": "2026-09-19T09:16:40.511Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.7197671666666666, "accuracy_std_dev": 0.010411737109574402, "run_count": 24, "total_questions": 4752, "avg_cost_per_task_usd": 0.1969247567340067, "avg_duration_per_task_ms": 258757.54566498316, "avg_output_tokens_per_task": 36726.52988215488, "last_run_timestamp": "2026-09-19T08:56:57.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "DeepInfra", "accuracy": 0.7196582307692306, "accuracy_std_dev": 0.022147329696075465, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.03072831965811966, "avg_duration_per_task_ms": 160744.22905982906, "avg_output_tokens_per_task": 13887.176068376068, "last_run_timestamp": "2026-06-30T21:06:56.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7183463229974161, "accuracy_std_dev": 0.029264511720208542, "run_count": 6, "total_questions": 1161, "avg_cost_per_task_usd": 0.17074779758828595, "avg_duration_per_task_ms": 1524939.6339362618, "avg_output_tokens_per_task": 41593.74849267872, "last_run_timestamp": "2026-08-22T16:49:30.177Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.7181841912777354, "accuracy_std_dev": 0.06555394037846128, "run_count": 7, "total_questions": 1307, "avg_cost_per_task_usd": 0.060895005355776584, "avg_duration_per_task_ms": 455243.765876052, "avg_output_tokens_per_task": 23693.538638102524, "last_run_timestamp": "2026-08-15T09:29:41.209Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7170990662824207, "accuracy_std_dev": 0.10692249999999998, "run_count": 2, "total_questions": 347, "avg_cost_per_task_usd": 0.005610605187319885, "avg_duration_per_task_ms": 269295.4322766571, "avg_output_tokens_per_task": 27309.671469740635, "last_run_timestamp": "2026-09-01T13:46:27.106Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7166106666666667, "accuracy_std_dev": 0.0055553002519115975, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.006570582491582491, "avg_duration_per_task_ms": 1280458.1801346801, "avg_output_tokens_per_task": 54335.32491582492, "last_run_timestamp": "2026-09-19T16:57:17.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.715615295838926, "accuracy_std_dev": 0.03314273943226165, "run_count": 19, "total_questions": 3725, "avg_cost_per_task_usd": 0.003583049395973154, "avg_duration_per_task_ms": 213584.61583892617, "avg_output_tokens_per_task": 9282.251812080536, "last_run_timestamp": "2026-08-29T19:25:16.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.7151083124999998, "accuracy_std_dev": 0.01761938659339479, "run_count": 20, "total_questions": 3936, "avg_cost_per_task_usd": 0.005333751270325202, "avg_duration_per_task_ms": 47488.129827235774, "avg_output_tokens_per_task": 6775.859756097561, "last_run_timestamp": "2026-08-29T18:12:18.859Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7146379609952925, "accuracy_std_dev": 0.12348955006372564, "run_count": 23, "total_questions": 4461, "avg_cost_per_task_usd": 0.0694516646491818, "avg_duration_per_task_ms": 609642.5270118808, "avg_output_tokens_per_task": 37654.42793095719, "last_run_timestamp": "2026-08-29T19:10:33.460Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.7145656108440172, "accuracy_std_dev": 0.017584165899374688, "run_count": 19, "total_questions": 3744, "avg_cost_per_task_usd": 0.006881101762820511, "avg_duration_per_task_ms": 63918.18936965812, "avg_output_tokens_per_task": 6296.184561965812, "last_run_timestamp": "2026-08-29T19:10:03.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7144979411764706, "accuracy_std_dev": 0.017256114730285467, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.007874328282828283, "avg_duration_per_task_ms": 136479.90077243018, "avg_output_tokens_per_task": 4099.4212715389185, "last_run_timestamp": "2026-08-29T18:30:44.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.713069165234002, "accuracy_std_dev": 0.0550740987322616, "run_count": 32, "total_questions": 6282, "avg_cost_per_task_usd": 0.005810328876154091, "avg_duration_per_task_ms": 328744.6372174467, "avg_output_tokens_per_task": 17988.781916587075, "last_run_timestamp": "2026-08-29T19:12:34.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.7115813800243607, "accuracy_std_dev": 0.0647214547058296, "run_count": 17, "total_questions": 3284, "avg_cost_per_task_usd": 0.004965959805115713, "avg_duration_per_task_ms": 87058.84226552983, "avg_output_tokens_per_task": 6155.969853836785, "last_run_timestamp": "2026-08-08T16:46:11.805Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.7114771734605377, "accuracy_std_dev": 0.08659653364462867, "run_count": 7, "total_questions": 1153, "avg_cost_per_task_usd": 0.007400922810060712, "avg_duration_per_task_ms": 492060.8178664354, "avg_output_tokens_per_task": 32850.75888985256, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.7105995910791368, "accuracy_std_dev": 0.04851107627346435, "run_count": 18, "total_questions": 3475, "avg_cost_per_task_usd": 0.0055933033093525176, "avg_duration_per_task_ms": 116405.79597122302, "avg_output_tokens_per_task": 7306.719136690647, "last_run_timestamp": "2026-08-29T18:11:53.163Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.7105179047619047, "accuracy_std_dev": 0.016159891819253525, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.00377856685906686, "avg_duration_per_task_ms": 69215.27994227994, "avg_output_tokens_per_task": 3508.088023088023, "last_run_timestamp": "2026-08-29T19:09:55.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.7104377647058823, "accuracy_std_dev": 0.016291381852087287, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.006191515745692216, "avg_duration_per_task_ms": 101266.32917409387, "avg_output_tokens_per_task": 4016.169043374926, "last_run_timestamp": "2026-08-29T18:10:22.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.709930669222344, "accuracy_std_dev": 0.17059505917300186, "run_count": 5, "total_questions": 913, "avg_cost_per_task_usd": 0.05822560240963856, "avg_duration_per_task_ms": 183232.56188389924, "avg_output_tokens_per_task": 16113.98247535597, "last_run_timestamp": "2026-07-18T08:37:00.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.70928025, "accuracy_std_dev": 0.2309020441067759, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.015585212121212121, "avg_duration_per_task_ms": 184205.19981060605, "avg_output_tokens_per_task": 14971.618055555555, "last_run_timestamp": "2026-08-15T15:24:35.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.708754, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01329097474747475, "avg_duration_per_task_ms": 223430.93434343435, "avg_output_tokens_per_task": 33132.76262626263, "last_run_timestamp": "2026-09-01T18:00:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.7083626384778011, "accuracy_std_dev": 0.1958727186126973, "run_count": 15, "total_questions": 2838, "avg_cost_per_task_usd": 0.040317193798449606, "avg_duration_per_task_ms": 587592.7716701903, "avg_output_tokens_per_task": 13532.868217054263, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7073907882352942, "accuracy_std_dev": 0.022165627566230677, "run_count": 17, "total_questions": 3315, "avg_cost_per_task_usd": 0.011149934238310712, "avg_duration_per_task_ms": 62511.60935143288, "avg_output_tokens_per_task": 7378.323076923077, "last_run_timestamp": "2026-08-29T19:12:26.066Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.7072108333333335, "accuracy_std_dev": 0.016372077555161485, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.006544036195286195, "avg_duration_per_task_ms": 37236.955808080806, "avg_output_tokens_per_task": 4331.103535353535, "last_run_timestamp": "2026-06-30T19:34:58.612Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Ambient", "accuracy": 0.707071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0038913585858585857, "avg_duration_per_task_ms": 62517.207070707074, "avg_output_tokens_per_task": 3515.439393939394, "last_run_timestamp": "2026-06-13T08:06:00.319Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.7067875794392524, "accuracy_std_dev": 0.019523788595938254, "run_count": 18, "total_questions": 3531, "avg_cost_per_task_usd": 0.007194955536675164, "avg_duration_per_task_ms": 73519.27867459643, "avg_output_tokens_per_task": 6882.512036250354, "last_run_timestamp": "2026-08-29T18:42:28.455Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.7052954681504223, "accuracy_std_dev": 0.015732334162799875, "run_count": 20, "total_questions": 3909, "avg_cost_per_task_usd": 0.006693433870555131, "avg_duration_per_task_ms": 93563.2755180353, "avg_output_tokens_per_task": 6043.323612177028, "last_run_timestamp": "2026-08-29T18:09:55.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7052560216881594, "accuracy_std_dev": 0.14220511739766134, "run_count": 11, "total_questions": 1706, "avg_cost_per_task_usd": 0.04422351465416178, "avg_duration_per_task_ms": 329497.4015240328, "avg_output_tokens_per_task": 12749.240914419695, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.7049911764705882, "accuracy_std_dev": 0.01828311467941453, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.006265460487225194, "avg_duration_per_task_ms": 163546.93166963756, "avg_output_tokens_per_task": 4319.140819964349, "last_run_timestamp": "2026-08-29T18:14:24.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.7047773804347826, "accuracy_std_dev": 0.10309159308121915, "run_count": 13, "total_questions": 2484, "avg_cost_per_task_usd": 0.10197293719806763, "avg_duration_per_task_ms": 616168.0583735909, "avg_output_tokens_per_task": 39185.06280193237, "last_run_timestamp": "2026-08-29T16:33:36.680Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7040404, "accuracy_std_dev": 0.045813599669093895, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.009978467676767678, "avg_duration_per_task_ms": 82731.23535353535, "avg_output_tokens_per_task": 3750.608080808081, "last_run_timestamp": "2026-09-19T09:43:03.159Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.7037366126080326, "accuracy_std_dev": 0.08290095218562599, "run_count": 21, "total_questions": 3934, "avg_cost_per_task_usd": 0.006630117946110829, "avg_duration_per_task_ms": 234021.50737163192, "avg_output_tokens_per_task": 12083.909506863243, "last_run_timestamp": "2026-08-29T19:16:51.883Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.7036724033898304, "accuracy_std_dev": 0.03265004974302829, "run_count": 3, "total_questions": 590, "avg_cost_per_task_usd": 0.011227418644067795, "avg_duration_per_task_ms": 599401.1288135593, "avg_output_tokens_per_task": 55874.53050847458, "last_run_timestamp": "2026-09-19T08:50:20.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.70328275, "accuracy_std_dev": 0.02153875190644758, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0019388005050505052, "avg_duration_per_task_ms": 636092.8409090909, "avg_output_tokens_per_task": 14000.35101010101, "last_run_timestamp": "2026-09-05T08:28:07.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.7030109514661275, "accuracy_std_dev": 0.03752877227029357, "run_count": 15, "total_questions": 2967, "avg_cost_per_task_usd": 0.009539730704415232, "avg_duration_per_task_ms": 166708.53791708796, "avg_output_tokens_per_task": 5363.80316818335, "last_run_timestamp": "2026-08-29T19:24:48.378Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.6998774823529411, "accuracy_std_dev": 0.036475041139420926, "run_count": 7, "total_questions": 1360, "avg_cost_per_task_usd": 0.010993920588235295, "avg_duration_per_task_ms": 289576.1580882353, "avg_output_tokens_per_task": 46600.06911764706, "last_run_timestamp": "2026-09-19T08:58:43.694Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.6989537142857144, "accuracy_std_dev": 0.013756650596444363, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.0019099855699855697, "avg_duration_per_task_ms": 141976.2987012987, "avg_output_tokens_per_task": 4153.015151515152, "last_run_timestamp": "2026-08-29T16:33:36.891Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Nebius", "accuracy": 0.698653, "accuracy_std_dev": 0.25478202686610374, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.07541588215488215, "avg_duration_per_task_ms": 218316.69360269362, "avg_output_tokens_per_task": 18749.255892255893, "last_run_timestamp": "2026-07-11T08:07:17.820Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6980896447963801, "accuracy_std_dev": 0.028298860510700467, "run_count": 7, "total_questions": 1326, "avg_cost_per_task_usd": 0.014896262443438914, "avg_duration_per_task_ms": 719785.2081447964, "avg_output_tokens_per_task": 74296.09426847662, "last_run_timestamp": "2026-09-19T16:29:00.211Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Io Net", "accuracy": 0.6969694999999999, "accuracy_std_dev": 0.02525250000000001, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008116111111111111, "avg_duration_per_task_ms": 29037.578282828283, "avg_output_tokens_per_task": 1510.050505050505, "last_run_timestamp": "2026-05-09T16:14:50.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.6961280000000001, "accuracy_std_dev": 0.01094299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004348992424242424, "avg_duration_per_task_ms": 356365.1767676768, "avg_output_tokens_per_task": 12056.626262626263, "last_run_timestamp": "2026-07-18T08:44:31.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Friendli", "accuracy": 0.6948655000000001, "accuracy_std_dev": 0.03859079866820586, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0038991833333333332, "avg_duration_per_task_ms": 161549.07575757575, "avg_output_tokens_per_task": 4745.70202020202, "last_run_timestamp": "2026-08-01T17:52:16.799Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.6925813109756097, "accuracy_std_dev": 0.24072562924993626, "run_count": 9, "total_questions": 1312, "avg_cost_per_task_usd": 0.009318611280487804, "avg_duration_per_task_ms": 54896.76600609756, "avg_output_tokens_per_task": 9351.999237804877, "last_run_timestamp": "2026-08-29T18:12:40.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.6905726, "accuracy_std_dev": 0.11069382289468553, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.012376834343434341, "avg_duration_per_task_ms": 236253.0919191919, "avg_output_tokens_per_task": 10174.568686868686, "last_run_timestamp": "2026-08-15T15:26:03.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Z.AI", "accuracy": 0.6897547142857142, "accuracy_std_dev": 0.016389379799250536, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0065042720057720054, "avg_duration_per_task_ms": 263620.07575757575, "avg_output_tokens_per_task": 16221.229437229436, "last_run_timestamp": "2026-05-23T17:26:25.014Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.6896746666666668, "accuracy_std_dev": 0.04960523518294776, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.04329939057239058, "avg_duration_per_task_ms": 1345214.5555555555, "avg_output_tokens_per_task": 31899.07744107744, "last_run_timestamp": "2026-08-22T16:20:35.762Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.6885525, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013761295454545455, "avg_duration_per_task_ms": 259532.94696969696, "avg_output_tokens_per_task": 32681.065656565657, "last_run_timestamp": "2026-09-01T18:15:18.821Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6876781730769231, "accuracy_std_dev": 0.02499090104818154, "run_count": 5, "total_questions": 936, "avg_cost_per_task_usd": 0.010550641025641025, "avg_duration_per_task_ms": 352806.86645299144, "avg_output_tokens_per_task": 45914.32692307692, "last_run_timestamp": "2026-09-19T08:48:39.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403:free", "model_slug": null, "model_name": "Gemma 4 26B A4B IT (free)", "provider_name": "Darkbloom", "accuracy": 0.684722, "accuracy_std_dev": null, "run_count": 1, "total_questions": 120, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 382535.98333333334, "avg_output_tokens_per_task": 6994.291666666667, "last_run_timestamp": "2026-08-04T05:36:23.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Ambient", "accuracy": 0.6843435, "accuracy_std_dev": 0.017676499999999984, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0009698787878787878, "avg_duration_per_task_ms": 30045.00505050505, "avg_output_tokens_per_task": 1560.3434343434344, "last_run_timestamp": "2026-05-23T16:14:51.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.6819954736842105, "accuracy_std_dev": 0.056917293716749924, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.0068329181286549715, "avg_duration_per_task_ms": 33886.56246677299, "avg_output_tokens_per_task": 5649.480595427964, "last_run_timestamp": "2026-08-29T18:10:09.843Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6808069808587688, "accuracy_std_dev": 0.21007685166978776, "run_count": 10, "total_questions": 1933, "avg_cost_per_task_usd": 0.015859241076047595, "avg_duration_per_task_ms": 373979.4381789964, "avg_output_tokens_per_task": 19354.842214174856, "last_run_timestamp": "2026-09-19T08:52:30.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.6804601535703615, "accuracy_std_dev": 0.12307264388535231, "run_count": 19, "total_questions": 3347, "avg_cost_per_task_usd": 0.025374532118314908, "avg_duration_per_task_ms": 194166.67403645054, "avg_output_tokens_per_task": 9087.962354347177, "last_run_timestamp": "2026-08-29T19:20:11.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.6803968704149127, "accuracy_std_dev": 0.16973745683387653, "run_count": 18, "total_questions": 3326, "avg_cost_per_task_usd": 0.0034803749248346364, "avg_duration_per_task_ms": 122097.64552014432, "avg_output_tokens_per_task": 4482.349368610944, "last_run_timestamp": "2026-08-29T19:47:54.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.6790924761904762, "accuracy_std_dev": 0.13080806951758023, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.04236039634439633, "avg_duration_per_task_ms": 53557.522607022605, "avg_output_tokens_per_task": 15382.483164983165, "last_run_timestamp": "2026-08-15T14:46:10.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.678451, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0025699040404040404, "avg_duration_per_task_ms": 57532.25252525252, "avg_output_tokens_per_task": 2314.656565656566, "last_run_timestamp": "2026-09-01T13:44:51.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.678451, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01359871212121212, "avg_duration_per_task_ms": 222801.71212121213, "avg_output_tokens_per_task": 33900.580808080806, "last_run_timestamp": "2026-09-01T18:09:36.992Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.6767873430232558, "accuracy_std_dev": 0.041497327246575476, "run_count": 16, "total_questions": 3096, "avg_cost_per_task_usd": 0.014108918604651159, "avg_duration_per_task_ms": 334124.46253229975, "avg_output_tokens_per_task": 16872.68766149871, "last_run_timestamp": "2026-08-01T18:07:49.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.676548715835141, "accuracy_std_dev": 0.020437457580933318, "run_count": 14, "total_questions": 2766, "avg_cost_per_task_usd": 0.005609137382501808, "avg_duration_per_task_ms": 246410.7856109906, "avg_output_tokens_per_task": 35921.823571945046, "last_run_timestamp": "2026-07-18T17:04:25.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.6752246559691913, "accuracy_std_dev": 0.03602352977366553, "run_count": 12, "total_questions": 2337, "avg_cost_per_task_usd": 0.013390630723149338, "avg_duration_per_task_ms": 424291.90629011556, "avg_output_tokens_per_task": 15722.762088147198, "last_run_timestamp": "2026-08-01T17:59:38.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6750842, "accuracy_std_dev": 0.023641210809939493, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005652539393939394, "avg_duration_per_task_ms": 500294.62828282826, "avg_output_tokens_per_task": 16033.422222222222, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "Novita", "accuracy": 0.6738817142857142, "accuracy_std_dev": 0.012906576382541092, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0017926522366522367, "avg_duration_per_task_ms": 45642.906204906205, "avg_output_tokens_per_task": 5814.865800865801, "last_run_timestamp": "2026-05-30T08:06:07.683Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.6731349473684209, "accuracy_std_dev": 0.085633655090479, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.009196745614035088, "avg_duration_per_task_ms": 10451.350345560872, "avg_output_tokens_per_task": 11969.038277511961, "last_run_timestamp": "2026-08-29T18:50:56.950Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.6729346307692308, "accuracy_std_dev": 0.008729468916008341, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.002463391452991453, "avg_duration_per_task_ms": 1210800.4, "avg_output_tokens_per_task": 14547.558974358974, "last_run_timestamp": "2026-07-18T09:58:41.490Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.6716106314888761, "accuracy_std_dev": 0.014749164820987887, "run_count": 18, "total_questions": 3506, "avg_cost_per_task_usd": 0.005318032515687393, "avg_duration_per_task_ms": 413726.3981745579, "avg_output_tokens_per_task": 37904.13605248146, "last_run_timestamp": "2026-08-29T21:19:16.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.671246097613883, "accuracy_std_dev": 0.027379954626458713, "run_count": 14, "total_questions": 2766, "avg_cost_per_task_usd": 0.0018295003615328993, "avg_duration_per_task_ms": 160145.0853217643, "avg_output_tokens_per_task": 3977.9674620390456, "last_run_timestamp": "2026-08-29T17:07:45.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6708755, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014901020202020204, "avg_duration_per_task_ms": 14005.674242424242, "avg_output_tokens_per_task": 1670.229797979798, "last_run_timestamp": "2026-09-01T13:21:57.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Cloudflare", "accuracy": 0.6707074, "accuracy_std_dev": 0.056996856052242015, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.028509575757575758, "avg_duration_per_task_ms": 178293.2676767677, "avg_output_tokens_per_task": 9460.838383838383, "last_run_timestamp": "2026-05-23T08:06:11.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.6703578219761501, "accuracy_std_dev": 0.0697645260135041, "run_count": 18, "total_questions": 3522, "avg_cost_per_task_usd": 0.09066784469051674, "avg_duration_per_task_ms": 715646.2507098239, "avg_output_tokens_per_task": 22289.18341851221, "last_run_timestamp": "2026-08-22T17:24:19.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.6696696261261261, "accuracy_std_dev": 0.08367726914995827, "run_count": 21, "total_questions": 3774, "avg_cost_per_task_usd": 0.006798153153153154, "avg_duration_per_task_ms": 35321.55060943296, "avg_output_tokens_per_task": 4497.988076311606, "last_run_timestamp": "2026-08-22T17:34:08.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.6658203026315791, "accuracy_std_dev": 0.02735758829038327, "run_count": 22, "total_questions": 4332, "avg_cost_per_task_usd": 0.0065892525392428425, "avg_duration_per_task_ms": 404554.2984764543, "avg_output_tokens_per_task": 35346.29085872576, "last_run_timestamp": "2026-08-29T21:14:10.017Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6643096000000001, "accuracy_std_dev": 0.049762880827379743, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00878879595959596, "avg_duration_per_task_ms": 266170.2818181818, "avg_output_tokens_per_task": 43649.65555555555, "last_run_timestamp": "2026-09-19T09:51:50.936Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.6640851398698884, "accuracy_std_dev": 0.02270387404538052, "run_count": 33, "total_questions": 6456, "avg_cost_per_task_usd": 0.006331274473358116, "avg_duration_per_task_ms": 262668.35517348204, "avg_output_tokens_per_task": 41968.7614622057, "last_run_timestamp": "2026-08-29T22:02:09.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6634949143621766, "accuracy_std_dev": 0.09211341535949208, "run_count": 17, "total_questions": 3363, "avg_cost_per_task_usd": 0.0035216934284864705, "avg_duration_per_task_ms": 207363.49955396968, "avg_output_tokens_per_task": 3883.495391019923, "last_run_timestamp": "2026-08-29T19:11:19.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.662458, "accuracy_std_dev": 0.15968229836929324, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.012940323232323234, "avg_duration_per_task_ms": 748366.1805555555, "avg_output_tokens_per_task": 27034.502525252527, "last_run_timestamp": "2026-09-19T09:48:41.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.661616, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003362974747474748, "avg_duration_per_task_ms": 65871.15656565657, "avg_output_tokens_per_task": 2362.121212121212, "last_run_timestamp": "2026-08-01T09:30:53.884Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.6615472218430034, "accuracy_std_dev": 0.011144726307502387, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.002312742320819112, "avg_duration_per_task_ms": 194982.48521046643, "avg_output_tokens_per_task": 3098.169510807736, "last_run_timestamp": "2026-08-29T18:47:21.578Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.6588440238663485, "accuracy_std_dev": 0.045738584476405955, "run_count": 13, "total_questions": 2514, "avg_cost_per_task_usd": 0.0178249661893397, "avg_duration_per_task_ms": 380429.29435163084, "avg_output_tokens_per_task": 16824.745425616547, "last_run_timestamp": "2026-08-01T18:05:37.080Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "NextBit", "accuracy": 0.6578283333333335, "accuracy_std_dev": 0.02342969883265444, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0015064617003367007, "avg_duration_per_task_ms": 42020.857323232325, "avg_output_tokens_per_task": 3276.22095959596, "last_run_timestamp": "2026-06-20T16:19:52.738Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.6575277142857143, "accuracy_std_dev": 0.021158523643569998, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005534338383838384, "avg_duration_per_task_ms": 453292.816017316, "avg_output_tokens_per_task": 34128.61327561327, "last_run_timestamp": "2026-08-29T22:11:02.681Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.6573137777777778, "accuracy_std_dev": 0.15283980178661852, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0030762968574635243, "avg_duration_per_task_ms": 54932.89898989899, "avg_output_tokens_per_task": 1996.0347923681256, "last_run_timestamp": "2026-07-01T02:34:47.176Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.6566732189349114, "accuracy_std_dev": 0.013404939721861693, "run_count": 13, "total_questions": 2535, "avg_cost_per_task_usd": 0.002109493885601578, "avg_duration_per_task_ms": 61133.307297830375, "avg_output_tokens_per_task": 2302.72899408284, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.6566194539007094, "accuracy_std_dev": 0.021003345172555248, "run_count": 16, "total_questions": 3102, "avg_cost_per_task_usd": 0.007993988072211475, "avg_duration_per_task_ms": 399361.4635718891, "avg_output_tokens_per_task": 27495.45293359123, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Phala", "accuracy": 0.656566, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0025412272727272727, "avg_duration_per_task_ms": 58356.79797979798, "avg_output_tokens_per_task": 3959.469696969697, "last_run_timestamp": "2026-06-20T08:04:16.013Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6560645175572519, "accuracy_std_dev": 0.017260755028097704, "run_count": 20, "total_questions": 3930, "avg_cost_per_task_usd": 0.0233054806615776, "avg_duration_per_task_ms": 505245.40814249363, "avg_output_tokens_per_task": 12144.778880407124, "last_run_timestamp": "2026-08-29T18:31:43.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.6551628333333336, "accuracy_std_dev": 0.01778329004758867, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.001573050785634119, "avg_duration_per_task_ms": 60087.49382716049, "avg_output_tokens_per_task": 1458.057519640853, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6544860441176471, "accuracy_std_dev": 0.16388817243977186, "run_count": 23, "total_questions": 4488, "avg_cost_per_task_usd": 0.013230713458110515, "avg_duration_per_task_ms": 237553.63123885918, "avg_output_tokens_per_task": 8754.879679144386, "last_run_timestamp": "2026-08-29T19:03:14.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.653757604130809, "accuracy_std_dev": 0.015316614138466363, "run_count": 3, "total_questions": 581, "avg_cost_per_task_usd": 0.01179434079173838, "avg_duration_per_task_ms": 1792911.3149741825, "avg_output_tokens_per_task": 50132.19449225473, "last_run_timestamp": "2026-08-29T20:13:41.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Ambient", "accuracy": 0.6523570000000001, "accuracy_std_dev": 0.08558150047177253, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.025248665824915827, "avg_duration_per_task_ms": 247158.74831649833, "avg_output_tokens_per_task": 6019.595959595959, "last_run_timestamp": "2026-06-13T09:16:40.692Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6521176887892377, "accuracy_std_dev": 0.0771054939636864, "run_count": 17, "total_questions": 3345, "avg_cost_per_task_usd": 0.006340421225710015, "avg_duration_per_task_ms": 602311.3940209268, "avg_output_tokens_per_task": 18765.147683109117, "last_run_timestamp": "2026-08-29T19:15:54.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Io Net", "accuracy": 0.6519742727272728, "accuracy_std_dev": 0.04715498277361752, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.04022706014692378, "avg_duration_per_task_ms": 252622.85904499542, "avg_output_tokens_per_task": 12756.500459136823, "last_run_timestamp": "2026-06-20T09:54:07.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6517467489082969, "accuracy_std_dev": 0.01932244128386387, "run_count": 14, "total_questions": 2748, "avg_cost_per_task_usd": 0.004762270378457059, "avg_duration_per_task_ms": 99071.56186317322, "avg_output_tokens_per_task": 8974.740902474527, "last_run_timestamp": "2026-09-19T09:11:59.179Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.650363823580786, "accuracy_std_dev": 0.007103203700131033, "run_count": 6, "total_questions": 1145, "avg_cost_per_task_usd": 0.0036315545851528385, "avg_duration_per_task_ms": 58503.03493449782, "avg_output_tokens_per_task": 2202.661135371179, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.6500593861209965, "accuracy_std_dev": 0.05876659054376805, "run_count": 6, "total_questions": 1124, "avg_cost_per_task_usd": 0.04040380338078292, "avg_duration_per_task_ms": 414098.88790035585, "avg_output_tokens_per_task": 20106.83896797153, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.6498317272727273, "accuracy_std_dev": 0.020163709926908996, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0017987033976124885, "avg_duration_per_task_ms": 59537.23829201102, "avg_output_tokens_per_task": 2292.5165289256197, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.6494412710059173, "accuracy_std_dev": 0.019678828707867992, "run_count": 13, "total_questions": 2535, "avg_cost_per_task_usd": 0.005908273767258381, "avg_duration_per_task_ms": 187327.3996055227, "avg_output_tokens_per_task": 9793.276134122289, "last_run_timestamp": "2026-09-19T09:14:07.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6489900000000001, "accuracy_std_dev": 0.01599299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.015694313131313132, "avg_duration_per_task_ms": 13831.43686868687, "avg_output_tokens_per_task": 1769.391414141414, "last_run_timestamp": "2026-09-01T13:20:53.165Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "AtlasCloud", "accuracy": 0.648008, "accuracy_std_dev": 0.021443451704424816, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0008452306397306398, "avg_duration_per_task_ms": 27333.61153198653, "avg_output_tokens_per_task": 890.7693602693603, "last_run_timestamp": "2026-07-01T02:41:54.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.6477275, "accuracy_std_dev": 0.015583769481418798, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01589866666666667, "avg_duration_per_task_ms": 19920.939393939392, "avg_output_tokens_per_task": 1699.0972222222222, "last_run_timestamp": "2026-09-01T13:27:17.676Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.6475160921501707, "accuracy_std_dev": 0.010423132030888464, "run_count": 9, "total_questions": 1758, "avg_cost_per_task_usd": 0.002509189419795222, "avg_duration_per_task_ms": 70158.00284414107, "avg_output_tokens_per_task": 2711.532423208191, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.646465, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0029047154882154884, "avg_duration_per_task_ms": 53981.06734006734, "avg_output_tokens_per_task": 2363.939393939394, "last_run_timestamp": "2026-08-29T16:33:11.860Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Venice", "accuracy": 0.6464646666666666, "accuracy_std_dev": 0.02295965891635927, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0018580218855218853, "avg_duration_per_task_ms": 38024.51683501683, "avg_output_tokens_per_task": 2618.2138047138046, "last_run_timestamp": "2026-05-16T08:06:13.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.6459677395788569, "accuracy_std_dev": 0.1480303961068657, "run_count": 12, "total_questions": 2327, "avg_cost_per_task_usd": 0.028217204555221313, "avg_duration_per_task_ms": 1621170.644177052, "avg_output_tokens_per_task": 20034.05242801891, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.6447810909090911, "accuracy_std_dev": 0.013751933189420343, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.00262662580348944, "avg_duration_per_task_ms": 31615.61432506887, "avg_output_tokens_per_task": 2488.2291092745636, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.6440172000000001, "accuracy_std_dev": 0.016796480873153823, "run_count": 12, "total_questions": 2340, "avg_cost_per_task_usd": 0.0037662337606837607, "avg_duration_per_task_ms": 61315.614102564105, "avg_output_tokens_per_task": 7079.364957264957, "last_run_timestamp": "2026-08-29T18:43:21.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Ambient", "accuracy": 0.6439395, "accuracy_std_dev": 0.007575499999999957, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00040658080808080813, "avg_duration_per_task_ms": 74334.95202020202, "avg_output_tokens_per_task": 2837.530303030303, "last_run_timestamp": "2026-06-13T16:33:02.717Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.6437363975720788, "accuracy_std_dev": 0.021818161687227465, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.001327898330804249, "avg_duration_per_task_ms": 54999.67981790592, "avg_output_tokens_per_task": 4282.004046535154, "last_run_timestamp": "2026-07-18T08:33:31.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.6420172700135687, "accuracy_std_dev": 0.04047369989098828, "run_count": 23, "total_questions": 4422, "avg_cost_per_task_usd": 0.012410424694708279, "avg_duration_per_task_ms": 541845.6732247851, "avg_output_tokens_per_task": 31847.307553143375, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.6414145, "accuracy_std_dev": 0.001683499999999949, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004025537878787879, "avg_duration_per_task_ms": 31194.16414141414, "avg_output_tokens_per_task": 2323.563131313131, "last_run_timestamp": "2026-09-01T17:49:44.421Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.641414, "accuracy_std_dev": 0.003367000000000009, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004276166666666667, "avg_duration_per_task_ms": 30535.468434343435, "avg_output_tokens_per_task": 2347.165404040404, "last_run_timestamp": "2026-09-01T17:50:18.482Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.6413600199386503, "accuracy_std_dev": 0.016520034383968443, "run_count": 20, "total_questions": 3912, "avg_cost_per_task_usd": 0.0015176196319018407, "avg_duration_per_task_ms": 81604.62193251534, "avg_output_tokens_per_task": 5287.028885480572, "last_run_timestamp": "2026-08-29T19:32:59.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.6411883358705995, "accuracy_std_dev": 0.07008215552556424, "run_count": 16, "total_questions": 3153, "avg_cost_per_task_usd": 0.012979633047890899, "avg_duration_per_task_ms": 350488.820805582, "avg_output_tokens_per_task": 15445.433872502379, "last_run_timestamp": "2026-08-01T18:25:56.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.6408530000000001, "accuracy_std_dev": 0.00731669260435796, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.00720101430976431, "avg_duration_per_task_ms": 742062.9217171717, "avg_output_tokens_per_task": 55987.864478114476, "last_run_timestamp": "2026-08-29T20:02:26.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.6408341596638656, "accuracy_std_dev": 0.030146989624529465, "run_count": 11, "total_questions": 2142, "avg_cost_per_task_usd": 0.0023272763772175537, "avg_duration_per_task_ms": 40174.9827264239, "avg_output_tokens_per_task": 1243.1489262371615, "last_run_timestamp": "2026-07-18T08:26:39.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.639886023931624, "accuracy_std_dev": 0.03350303592049533, "run_count": 10, "total_questions": 1755, "avg_cost_per_task_usd": 0.016610918518518522, "avg_duration_per_task_ms": 866168.4626780627, "avg_output_tokens_per_task": 69483.85982905983, "last_run_timestamp": "2026-09-19T16:49:53.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6397305, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004182681818181818, "avg_duration_per_task_ms": 33306.957070707074, "avg_output_tokens_per_task": 2424.3232323232323, "last_run_timestamp": "2026-09-01T17:50:13.815Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.637803, "accuracy_std_dev": null, "run_count": 1, "total_questions": 179, "avg_cost_per_task_usd": 0.002030703910614525, "avg_duration_per_task_ms": 92260.82681564246, "avg_output_tokens_per_task": 6534.692737430168, "last_run_timestamp": "2026-07-18T09:17:11.788Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.6374862222222222, "accuracy_std_dev": 0.015120312553925432, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.001489671717171717, "avg_duration_per_task_ms": 106578.17789001123, "avg_output_tokens_per_task": 7528.87822671156, "last_run_timestamp": "2026-08-29T19:02:06.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6374368988195617, "accuracy_std_dev": 0.013998977609052932, "run_count": 18, "total_questions": 3558, "avg_cost_per_task_usd": 0.0016592459246767845, "avg_duration_per_task_ms": 92130.81225407532, "avg_output_tokens_per_task": 5361.156548622822, "last_run_timestamp": "2026-08-29T18:23:33.062Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.6374213227665708, "accuracy_std_dev": 0.019782319798481038, "run_count": 16, "total_questions": 3123, "avg_cost_per_task_usd": 0.0016995072046109506, "avg_duration_per_task_ms": 146595.44732628882, "avg_output_tokens_per_task": 5504.1364073006725, "last_run_timestamp": "2026-08-29T18:40:25.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.6370851428571429, "accuracy_std_dev": 0.022815823531228588, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.0012857987012987013, "avg_duration_per_task_ms": 143754.60606060605, "avg_output_tokens_per_task": 7026.642857142857, "last_run_timestamp": "2026-08-29T17:04:59.562Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.6342237881448958, "accuracy_std_dev": 0.029265826681420387, "run_count": 14, "total_questions": 2733, "avg_cost_per_task_usd": 0.006398584705451885, "avg_duration_per_task_ms": 104759.4526161727, "avg_output_tokens_per_task": 8931.070252469814, "last_run_timestamp": "2026-09-19T08:52:51.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.6341992857142857, "accuracy_std_dev": 0.012328930526139207, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0010588167388167389, "avg_duration_per_task_ms": 49239.52308802309, "avg_output_tokens_per_task": 3418.099567099567, "last_run_timestamp": "2026-06-30T19:57:58.812Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.6334175, "accuracy_std_dev": 0.007154500000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001171128787878788, "avg_duration_per_task_ms": 257331.09848484848, "avg_output_tokens_per_task": 11528.555555555555, "last_run_timestamp": "2026-08-29T19:38:59.849Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.6332902229423202, "accuracy_std_dev": 0.011149999831137883, "run_count": 8, "total_questions": 1543, "avg_cost_per_task_usd": 0.016406834737524304, "avg_duration_per_task_ms": 444530.66429034347, "avg_output_tokens_per_task": 82062.96305897602, "last_run_timestamp": "2026-09-19T16:24:34.829Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.6327843526405451, "accuracy_std_dev": 0.01928203380236522, "run_count": 9, "total_questions": 1761, "avg_cost_per_task_usd": 0.010741952299829643, "avg_duration_per_task_ms": 472791.5741056218, "avg_output_tokens_per_task": 11222.989210675752, "last_run_timestamp": "2026-09-19T10:52:13.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6315639897260275, "accuracy_std_dev": 0.025646576254064718, "run_count": 3, "total_questions": 584, "avg_cost_per_task_usd": 0.006708763698630138, "avg_duration_per_task_ms": 935572.4657534247, "avg_output_tokens_per_task": 37027.727739726026, "last_run_timestamp": "2026-09-19T08:08:47.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.6313515294117649, "accuracy_std_dev": 0.02320569910065199, "run_count": 16, "total_questions": 3162, "avg_cost_per_task_usd": 0.0037360388994307405, "avg_duration_per_task_ms": 386733.3523086654, "avg_output_tokens_per_task": 12360.750158127767, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6308680607902736, "accuracy_std_dev": 0.011456259937693471, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.0034108257345491387, "avg_duration_per_task_ms": 140057.4062816616, "avg_output_tokens_per_task": 11472.941236068895, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "AtlasCloud", "accuracy": 0.6296296666666669, "accuracy_std_dev": 0.012371329390525862, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0035772985409652075, "avg_duration_per_task_ms": 14957.81088664422, "avg_output_tokens_per_task": 873.4545454545455, "last_run_timestamp": "2026-06-30T19:49:03.931Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Venice", "accuracy": 0.6292411856287424, "accuracy_std_dev": 0.0117815, "run_count": 2, "total_questions": 334, "avg_cost_per_task_usd": 0.0071094161676646716, "avg_duration_per_task_ms": 112258.81736526947, "avg_output_tokens_per_task": 28232.077844311378, "last_run_timestamp": "2026-08-15T08:02:44.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "AtlasCloud", "accuracy": 0.6272402163187856, "accuracy_std_dev": 0.023564479969529893, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.007974122707147374, "avg_duration_per_task_ms": 212675.86464263123, "avg_output_tokens_per_task": 6702.41302972802, "last_run_timestamp": "2026-07-01T03:52:27.932Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6240829792835564, "accuracy_std_dev": 0.024714634180733906, "run_count": 12, "total_questions": 2317, "avg_cost_per_task_usd": 0.007086491152352179, "avg_duration_per_task_ms": 448985.8131204143, "avg_output_tokens_per_task": 11630.75960293483, "last_run_timestamp": "2026-09-19T09:21:13.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "NextBit", "accuracy": 0.6235919324055667, "accuracy_std_dev": 0.017558878241772012, "run_count": 8, "total_questions": 1509, "avg_cost_per_task_usd": 0.005936661365142478, "avg_duration_per_task_ms": 567045.3445990722, "avg_output_tokens_per_task": 10721.506295559973, "last_run_timestamp": "2026-07-18T08:03:56.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.6233549090909093, "accuracy_std_dev": 0.012558972760935413, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.007362255739210285, "avg_duration_per_task_ms": 220161.05739210284, "avg_output_tokens_per_task": 17663.295224977042, "last_run_timestamp": "2026-07-18T16:20:00.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.620681432286024, "accuracy_std_dev": 0.013051064268315992, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.004365833875045143, "avg_duration_per_task_ms": 90965.72264355363, "avg_output_tokens_per_task": 8292.383893102204, "last_run_timestamp": "2026-08-01T17:51:29.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6202935541022593, "accuracy_std_dev": 0.014314030660974362, "run_count": 13, "total_questions": 2523, "avg_cost_per_task_usd": 0.004428355925485533, "avg_duration_per_task_ms": 174619.32778438367, "avg_output_tokens_per_task": 8993.805390408244, "last_run_timestamp": "2026-08-01T18:16:00.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.6193183750000001, "accuracy_std_dev": 0.02645072511736446, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008760742424242424, "avg_duration_per_task_ms": 607268.2133838384, "avg_output_tokens_per_task": 15504.423611111111, "last_run_timestamp": "2026-07-18T17:46:02.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.6191422040816327, "accuracy_std_dev": 0.01950844855599051, "run_count": 18, "total_questions": 3528, "avg_cost_per_task_usd": 0.002386935657596371, "avg_duration_per_task_ms": 23158.938775510203, "avg_output_tokens_per_task": 1422.096655328798, "last_run_timestamp": "2026-08-29T18:49:22.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Together", "accuracy": 0.618687, "accuracy_std_dev": 0.01041181185000961, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.001993618686868687, "avg_duration_per_task_ms": 11609.738636363636, "avg_output_tokens_per_task": 726.0012626262626, "last_run_timestamp": "2026-05-23T08:09:18.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6176683697888469, "accuracy_std_dev": 0.033957228972488325, "run_count": 10, "total_questions": 1847, "avg_cost_per_task_usd": 0.013724128857606929, "avg_duration_per_task_ms": 551868.6426637791, "avg_output_tokens_per_task": 67292.0590146183, "last_run_timestamp": "2026-09-19T16:33:17.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.6139522500000001, "accuracy_std_dev": 0.01942167556977256, "run_count": 16, "total_questions": 3168, "avg_cost_per_task_usd": 0.007408927083333333, "avg_duration_per_task_ms": 22613.55808080808, "avg_output_tokens_per_task": 1412.095643939394, "last_run_timestamp": "2026-08-29T18:32:24.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.6138280549120549, "accuracy_std_dev": 0.022194557320808772, "run_count": 12, "total_questions": 2331, "avg_cost_per_task_usd": 0.0015813187473187472, "avg_duration_per_task_ms": 35072.91076791077, "avg_output_tokens_per_task": 1478.07722007722, "last_run_timestamp": "2026-08-29T18:37:17.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6129581867007673, "accuracy_std_dev": 0.0015394999999999714, "run_count": 2, "total_questions": 391, "avg_cost_per_task_usd": 0.0032793452685421997, "avg_duration_per_task_ms": 346994.51406649617, "avg_output_tokens_per_task": 13256.76726342711, "last_run_timestamp": "2026-09-01T13:34:17.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.6129154305135952, "accuracy_std_dev": 0.04617983854978681, "run_count": 8, "total_questions": 1324, "avg_cost_per_task_usd": 0.002775106495468278, "avg_duration_per_task_ms": 1032830.075528701, "avg_output_tokens_per_task": 25108.58836858006, "last_run_timestamp": "2026-07-18T16:00:09.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.6124774159836067, "accuracy_std_dev": 0.020744809856282916, "run_count": 15, "total_questions": 2928, "avg_cost_per_task_usd": 0.0023938254781420767, "avg_duration_per_task_ms": 43103.5293715847, "avg_output_tokens_per_task": 1524.1318306010928, "last_run_timestamp": "2026-08-29T19:00:24.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6122354822934233, "accuracy_std_dev": 0.01895311219280912, "run_count": 18, "total_questions": 3558, "avg_cost_per_task_usd": 0.00473105171444632, "avg_duration_per_task_ms": 207884.38617200675, "avg_output_tokens_per_task": 11432.557335581787, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6118327857142859, "accuracy_std_dev": 0.022771416930562546, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.004391077561327561, "avg_duration_per_task_ms": 183806.7113997114, "avg_output_tokens_per_task": 8779.912337662337, "last_run_timestamp": "2026-08-01T18:08:49.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.6099262895816242, "accuracy_std_dev": 0.02527746961067045, "run_count": 7, "total_questions": 1219, "avg_cost_per_task_usd": 0.02817470713699754, "avg_duration_per_task_ms": 563426.3125512715, "avg_output_tokens_per_task": 29571.532403609515, "last_run_timestamp": "2026-09-19T16:13:01.086Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.609428, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0023919898989898988, "avg_duration_per_task_ms": 73719.80808080808, "avg_output_tokens_per_task": 2205.409090909091, "last_run_timestamp": "2026-09-01T13:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.6088711387096775, "accuracy_std_dev": 0.030645019138335016, "run_count": 19, "total_questions": 3720, "avg_cost_per_task_usd": 0.002026076612903226, "avg_duration_per_task_ms": 34227.742204301074, "avg_output_tokens_per_task": 1358.1854838709678, "last_run_timestamp": "2026-08-29T18:47:46.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Wafer", "accuracy": 0.6085067890625, "accuracy_std_dev": 0.10062656908435016, "run_count": 4, "total_questions": 768, "avg_cost_per_task_usd": 0.010816417968749999, "avg_duration_per_task_ms": 300613.0859375, "avg_output_tokens_per_task": 26862.471354166668, "last_run_timestamp": "2026-07-18T08:04:42.524Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.607744, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003953611111111112, "avg_duration_per_task_ms": 57224.242424242424, "avg_output_tokens_per_task": 2094.1161616161617, "last_run_timestamp": "2026-08-01T17:44:33.510Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.606531074232691, "accuracy_std_dev": 0.0392900303150812, "run_count": 15, "total_questions": 2802, "avg_cost_per_task_usd": 0.002877521056388294, "avg_duration_per_task_ms": 32091.98608137045, "avg_output_tokens_per_task": 1530.401142041399, "last_run_timestamp": "2026-08-29T18:43:09.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.6062243505658437, "accuracy_std_dev": 0.14728578802372175, "run_count": 21, "total_questions": 3888, "avg_cost_per_task_usd": 0.008590261574074075, "avg_duration_per_task_ms": 53306.474022633745, "avg_output_tokens_per_task": 28544.59079218107, "last_run_timestamp": "2026-08-29T19:48:18.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6057348861480076, "accuracy_std_dev": 0.07360041928198763, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.004612943074003796, "avg_duration_per_task_ms": 545569.0094876661, "avg_output_tokens_per_task": 18056.067678684376, "last_run_timestamp": "2026-08-22T22:18:10.299Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.6035355000000001, "accuracy_std_dev": 0.004208499999999948, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.002990320707070707, "avg_duration_per_task_ms": 555382.6085858586, "avg_output_tokens_per_task": 13221.70707070707, "last_run_timestamp": "2026-09-01T13:04:03.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.6009468851294903, "accuracy_std_dev": 0.08092483969818855, "run_count": 13, "total_questions": 2394, "avg_cost_per_task_usd": 0.006364984126984126, "avg_duration_per_task_ms": 31444.639097744363, "avg_output_tokens_per_task": 10538.471177944863, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.5992354149569821, "accuracy_std_dev": 0.019006860146473396, "run_count": 23, "total_questions": 4533, "avg_cost_per_task_usd": 0.04292028435914405, "avg_duration_per_task_ms": 513498.9199205824, "avg_output_tokens_per_task": 16315.828369733068, "last_run_timestamp": "2026-08-30T00:22:07.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.594276, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.002775611111111111, "avg_duration_per_task_ms": 509237.26767676766, "avg_output_tokens_per_task": 10972.9898989899, "last_run_timestamp": "2026-09-01T13:31:26.819Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.5931538333333333, "accuracy_std_dev": 0.09808536433028917, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.5963892592592592, "avg_duration_per_task_ms": 620612.2373737374, "avg_output_tokens_per_task": 39751.62205387205, "last_run_timestamp": "2026-09-19T08:49:57.309Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5900675, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017035757575757576, "avg_duration_per_task_ms": 123031.02272727272, "avg_output_tokens_per_task": 42470.74747474748, "last_run_timestamp": "2026-09-01T18:10:49.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.5872781331360947, "accuracy_std_dev": 0.004687499999999956, "run_count": 2, "total_questions": 338, "avg_cost_per_task_usd": 0.09507673668639052, "avg_duration_per_task_ms": 191708.50591715975, "avg_output_tokens_per_task": 21455.86094674556, "last_run_timestamp": "2026-08-01T18:26:20.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.5870612380952381, "accuracy_std_dev": 0.09923890865789474, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.6539967813852815, "avg_duration_per_task_ms": 491814.81192881195, "avg_output_tokens_per_task": 40702.665704665706, "last_run_timestamp": "2026-09-19T09:08:58.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.5867004444444445, "accuracy_std_dev": 0.030454637959544256, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.002968678451178451, "avg_duration_per_task_ms": 201841.53367003368, "avg_output_tokens_per_task": 10450.568181818182, "last_run_timestamp": "2026-08-29T19:16:45.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.5824918571428572, "accuracy_std_dev": 0.09639527555617557, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.6154766688311687, "avg_duration_per_task_ms": 632957.1277056277, "avg_output_tokens_per_task": 40886.683982683986, "last_run_timestamp": "2026-09-19T09:11:28.823Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.5820909628506444, "accuracy_std_dev": 0.10256844962388531, "run_count": 20, "total_questions": 3957, "avg_cost_per_task_usd": 0.6594307323730098, "avg_duration_per_task_ms": 597798.0687389437, "avg_output_tokens_per_task": 41132.2928986606, "last_run_timestamp": "2026-09-19T09:07:29.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.5774372016509435, "accuracy_std_dev": 0.01609338000361728, "run_count": 13, "total_questions": 2544, "avg_cost_per_task_usd": 0.005421983490566038, "avg_duration_per_task_ms": 524101.0585691824, "avg_output_tokens_per_task": 35942.22051886792, "last_run_timestamp": "2026-08-29T16:53:28.602Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.5772384474734041, "accuracy_std_dev": 0.11649000927593507, "run_count": 11, "total_questions": 1504, "avg_cost_per_task_usd": 0.004710451462765957, "avg_duration_per_task_ms": 563810.3085106383, "avg_output_tokens_per_task": 32534.692819148935, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.5767835180722893, "accuracy_std_dev": 0.11488998007127417, "run_count": 18, "total_questions": 3486, "avg_cost_per_task_usd": 0.001997228341939185, "avg_duration_per_task_ms": 61275.66838783706, "avg_output_tokens_per_task": 1863.6497418244405, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.5764792857142857, "accuracy_std_dev": 0.10228809062044081, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.6180922987012987, "avg_duration_per_task_ms": 636696.5533910533, "avg_output_tokens_per_task": 41167.0367965368, "last_run_timestamp": "2026-09-19T08:49:03.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.5726323012863915, "accuracy_std_dev": 0.036706189578649295, "run_count": 23, "total_questions": 4431, "avg_cost_per_task_usd": 0.056247474159331984, "avg_duration_per_task_ms": 424509.57323403296, "avg_output_tokens_per_task": 17825.642744301513, "last_run_timestamp": "2026-08-29T19:11:06.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.5675273333333334, "accuracy_std_dev": 0.09565683495356374, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.6112196195286197, "avg_duration_per_task_ms": 627120.2558922559, "avg_output_tokens_per_task": 40643.95005611672, "last_run_timestamp": "2026-09-19T08:58:01.146Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.5565433914539719, "accuracy_std_dev": 0.11305693050580784, "run_count": 18, "total_questions": 3487, "avg_cost_per_task_usd": 0.06438679782047606, "avg_duration_per_task_ms": 679986.9673071408, "avg_output_tokens_per_task": 20386.74533983367, "last_run_timestamp": "2026-08-22T17:02:13.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.5547767757009346, "accuracy_std_dev": 0.0771476524984634, "run_count": 13, "total_questions": 2568, "avg_cost_per_task_usd": 0.007002486370716511, "avg_duration_per_task_ms": 230184.06152647975, "avg_output_tokens_per_task": 21060.570093457944, "last_run_timestamp": "2026-08-29T17:01:22.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.55303, "accuracy_std_dev": 0.061447999999999975, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006801247474747475, "avg_duration_per_task_ms": 137063.60858585857, "avg_output_tokens_per_task": 7268.926767676768, "last_run_timestamp": "2026-09-01T13:21:33.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.5514001070195628, "accuracy_std_dev": 0.20533572929527444, "run_count": 6, "total_questions": 869, "avg_cost_per_task_usd": 0.05092492520138089, "avg_duration_per_task_ms": 1396586.3498273878, "avg_output_tokens_per_task": 19293.715765247412, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5498265044082287, "accuracy_std_dev": 0.03556443036291385, "run_count": 21, "total_questions": 3743, "avg_cost_per_task_usd": 0.0300913881912904, "avg_duration_per_task_ms": 386568.3438418381, "avg_output_tokens_per_task": 14387.59177130644, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5496635000000001, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017014479797979796, "avg_duration_per_task_ms": 123100.59595959596, "avg_output_tokens_per_task": 42507.18939393939, "last_run_timestamp": "2026-09-01T17:51:18.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Phala", "accuracy": 0.5454545714285715, "accuracy_std_dev": 0.10688204941343671, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.004287935425685424, "avg_duration_per_task_ms": 239211.0404040404, "avg_output_tokens_per_task": 11229.227272727272, "last_run_timestamp": "2026-06-20T18:20:35.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.5433505, "accuracy_std_dev": 0.016294800251920854, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.017506126262626265, "avg_duration_per_task_ms": 128742.19065656565, "avg_output_tokens_per_task": 43717.62373737374, "last_run_timestamp": "2026-09-01T18:06:03.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.5429295000000001, "accuracy_std_dev": 0.010942500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001760580808080808, "avg_duration_per_task_ms": 80209.29040404041, "avg_output_tokens_per_task": 1718.6565656565656, "last_run_timestamp": "2026-09-01T13:31:30.699Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.5427914079754601, "accuracy_std_dev": 0.1644485316347865, "run_count": 17, "total_questions": 3260, "avg_cost_per_task_usd": 0.031211190490797545, "avg_duration_per_task_ms": 399121.22024539876, "avg_output_tokens_per_task": 11046.685889570552, "last_run_timestamp": "2026-07-18T16:15:33.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Novita", "accuracy": 0.5424244, "accuracy_std_dev": 0.020353179334934343, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004261171717171717, "avg_duration_per_task_ms": 201511.99797979798, "avg_output_tokens_per_task": 9415.110101010101, "last_run_timestamp": "2026-05-23T11:11:04.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.542088, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.023943484848484846, "avg_duration_per_task_ms": 10793.646464646465, "avg_output_tokens_per_task": 1334.611111111111, "last_run_timestamp": "2026-08-01T08:53:34.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.5412917072553431, "accuracy_std_dev": 0.15868845181824737, "run_count": 18, "total_questions": 3556, "avg_cost_per_task_usd": 0.0018783034308211474, "avg_duration_per_task_ms": 76872.3416760405, "avg_output_tokens_per_task": 2062.933914510686, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Phala", "accuracy": 0.5402590995515695, "accuracy_std_dev": 0.1064112241873893, "run_count": 17, "total_questions": 3345, "avg_cost_per_task_usd": 0.034116275635276534, "avg_duration_per_task_ms": 482865.85979073244, "avg_output_tokens_per_task": 12392.341405082212, "last_run_timestamp": "2026-07-11T16:02:27.787Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.5350947692307693, "accuracy_std_dev": 0.014328365504260027, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.0032667284382284375, "avg_duration_per_task_ms": 72101.14724164724, "avg_output_tokens_per_task": 7056.418803418804, "last_run_timestamp": "2026-08-01T17:48:17.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.53367, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01864569696969697, "avg_duration_per_task_ms": 17970.575757575756, "avg_output_tokens_per_task": 1670.050505050505, "last_run_timestamp": "2026-09-01T13:20:31.881Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5312429141104295, "accuracy_std_dev": 0.042893663569032354, "run_count": 15, "total_questions": 2934, "avg_cost_per_task_usd": 0.01798347137014315, "avg_duration_per_task_ms": 733666.4400136332, "avg_output_tokens_per_task": 45997.68916155419, "last_run_timestamp": "2026-08-30T00:49:28.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Novita", "accuracy": 0.5303032857142858, "accuracy_std_dev": 0.016421324661325395, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0034015209235209233, "avg_duration_per_task_ms": 115367.47474747474, "avg_output_tokens_per_task": 6747.353535353535, "last_run_timestamp": "2026-07-01T03:39:59.085Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "ModelRun", "accuracy": 0.5290405, "accuracy_std_dev": 0.029959029561219083, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0006086212121212121, "avg_duration_per_task_ms": 25756.060606060608, "avg_output_tokens_per_task": 699.3156565656566, "last_run_timestamp": "2026-06-20T16:15:10.268Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.5282828, "accuracy_std_dev": 0.025272813831467202, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004160109090909091, "avg_duration_per_task_ms": 148058.1797979798, "avg_output_tokens_per_task": 8276.889898989899, "last_run_timestamp": "2026-06-20T17:38:01.923Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.5223280439238653, "accuracy_std_dev": 0.19977793037975025, "run_count": 7, "total_questions": 1366, "avg_cost_per_task_usd": 0.02168075768667643, "avg_duration_per_task_ms": 693100.055636896, "avg_output_tokens_per_task": 30376.870424597364, "last_run_timestamp": "2026-09-19T16:22:29.649Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.521886, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.016360343434343436, "avg_duration_per_task_ms": 13511.040404040405, "avg_output_tokens_per_task": 1443.3737373737374, "last_run_timestamp": "2026-09-01T14:02:01.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.5202023333333333, "accuracy_std_dev": 0.0023810642345154617, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0035723956228956226, "avg_duration_per_task_ms": 90157.08922558922, "avg_output_tokens_per_task": 2721.4242424242425, "last_run_timestamp": "2026-08-15T08:46:42.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.5190796666666666, "accuracy_std_dev": 0.008837261503932561, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004016927609427609, "avg_duration_per_task_ms": 28916.838383838385, "avg_output_tokens_per_task": 2584.1296296296296, "last_run_timestamp": "2026-08-15T09:25:42.974Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Ambient", "accuracy": 0.5176769999999999, "accuracy_std_dev": 0.007575999999999972, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009877323232323232, "avg_duration_per_task_ms": 465993.38636363635, "avg_output_tokens_per_task": 18118.93181818182, "last_run_timestamp": "2026-06-13T11:17:16.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.517396, "accuracy_std_dev": 0.02012409515978297, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015192781144781144, "avg_duration_per_task_ms": 14836.205387205388, "avg_output_tokens_per_task": 1327.0218855218855, "last_run_timestamp": "2026-09-01T13:22:30.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.5171403267929634, "accuracy_std_dev": 0.04098216724042571, "run_count": 24, "total_questions": 4434, "avg_cost_per_task_usd": 0.009192668245376635, "avg_duration_per_task_ms": 398119.1844835363, "avg_output_tokens_per_task": 23257.760261614796, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.516835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018182969696969697, "avg_duration_per_task_ms": 17464.777777777777, "avg_output_tokens_per_task": 1627.171717171717, "last_run_timestamp": "2026-09-01T13:20:49.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.5157126666666666, "accuracy_std_dev": 0.02305568803763809, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004710382154882156, "avg_duration_per_task_ms": 211036.34006734006, "avg_output_tokens_per_task": 2694.722222222222, "last_run_timestamp": "2026-08-15T09:11:29.276Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.5154740415003991, "accuracy_std_dev": 0.0612012483160448, "run_count": 21, "total_questions": 3759, "avg_cost_per_task_usd": 0.0585535027932961, "avg_duration_per_task_ms": 505974.45304602286, "avg_output_tokens_per_task": 16676.524607608408, "last_run_timestamp": "2026-08-29T18:52:09.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.5134683333333333, "accuracy_std_dev": 0.00836132925370652, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008388316498316498, "avg_duration_per_task_ms": 89120.7138047138, "avg_output_tokens_per_task": 2792.7188552188554, "last_run_timestamp": "2026-08-15T08:49:03.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.511785, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01733130303030303, "avg_duration_per_task_ms": 13903.823232323231, "avg_output_tokens_per_task": 1536.9141414141413, "last_run_timestamp": "2026-09-01T13:47:27.116Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.5108945297941495, "accuracy_std_dev": 0.01301865738320295, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.004122480678945469, "avg_duration_per_task_ms": 117292.87612856626, "avg_output_tokens_per_task": 8300.09245214879, "last_run_timestamp": "2026-08-01T18:08:24.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Friendli", "accuracy": 0.510101, "accuracy_std_dev": 0.010101000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0006397070707070707, "avg_duration_per_task_ms": 7448.810606060606, "avg_output_tokens_per_task": 796.5252525252525, "last_run_timestamp": "2026-05-09T08:07:34.813Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.5092595, "accuracy_std_dev": 0.009259499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008195479797979797, "avg_duration_per_task_ms": 19948.4797979798, "avg_output_tokens_per_task": 1859.0176767676767, "last_run_timestamp": "2026-09-01T17:50:32.282Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.506734, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0237055303030303, "avg_duration_per_task_ms": 11478.515151515152, "avg_output_tokens_per_task": 1318.7474747474748, "last_run_timestamp": "2026-08-01T08:34:23.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.5053764679900744, "accuracy_std_dev": 0.20352934331044298, "run_count": 11, "total_questions": 2015, "avg_cost_per_task_usd": 0.01812188188585608, "avg_duration_per_task_ms": 1793360.2853598015, "avg_output_tokens_per_task": 13961.292803970224, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02292151515151515, "avg_duration_per_task_ms": 12797.353535353535, "avg_output_tokens_per_task": 1266.479797979798, "last_run_timestamp": "2026-08-01T08:47:47.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.5049259153605015, "accuracy_std_dev": 0.11693449444168029, "run_count": 13, "total_questions": 2233, "avg_cost_per_task_usd": 0.008191620241827138, "avg_duration_per_task_ms": 482531.36945812806, "avg_output_tokens_per_task": 40792.52754142409, "last_run_timestamp": "2026-09-19T16:17:10.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.5033673333333334, "accuracy_std_dev": 0.009622237242739144, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008440841750841751, "avg_duration_per_task_ms": 14338.025252525253, "avg_output_tokens_per_task": 1851.915824915825, "last_run_timestamp": "2026-09-01T17:51:18.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.503367, "accuracy_std_dev": 0.008418000000000037, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013204560606060607, "avg_duration_per_task_ms": 13084.876262626263, "avg_output_tokens_per_task": 1127.6338383838383, "last_run_timestamp": "2026-09-01T13:20:22.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Together", "accuracy": 0.5009998431876606, "accuracy_std_dev": 0.23720298614056273, "run_count": 6, "total_questions": 1167, "avg_cost_per_task_usd": 0.004498337617823479, "avg_duration_per_task_ms": 250896.1996572408, "avg_output_tokens_per_task": 23117.198800342758, "last_run_timestamp": "2026-07-04T16:00:23.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.4959782777777778, "accuracy_std_dev": 0.02300204246439761, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.0006973439955106623, "avg_duration_per_task_ms": 43601.93097643097, "avg_output_tokens_per_task": 1512.7345679012346, "last_run_timestamp": "2026-08-29T19:21:45.914Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.4956976666666667, "accuracy_std_dev": 0.01927874840624025, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.0016561282267115599, "avg_duration_per_task_ms": 7718.37682379349, "avg_output_tokens_per_task": 1641.1672278338945, "last_run_timestamp": "2026-08-29T19:15:06.404Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.4934997222222222, "accuracy_std_dev": 0.02101166846409543, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.0017448033108866444, "avg_duration_per_task_ms": 10151.609988776656, "avg_output_tokens_per_task": 1588.8221099887767, "last_run_timestamp": "2026-08-29T16:43:47.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.49248677460142937, "accuracy_std_dev": 0.1468475282129393, "run_count": 12, "total_questions": 1819, "avg_cost_per_task_usd": 0.02537227047828477, "avg_duration_per_task_ms": 43723.59538207806, "avg_output_tokens_per_task": 5430.2094557449145, "last_run_timestamp": "2026-08-29T16:54:06.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.4907005714285715, "accuracy_std_dev": 0.024199875069672482, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.0008166632996632996, "avg_duration_per_task_ms": 41913.33645983646, "avg_output_tokens_per_task": 1437.7967772967772, "last_run_timestamp": "2026-08-29T18:24:23.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.4900520065566357, "accuracy_std_dev": 0.037634349752895334, "run_count": 23, "total_questions": 4423, "avg_cost_per_task_usd": 0.007361529957042731, "avg_duration_per_task_ms": 281581.36536287586, "avg_output_tokens_per_task": 15918.30137915442, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.4897118333333333, "accuracy_std_dev": 0.027081268771052807, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.0022169312570145903, "avg_duration_per_task_ms": 39178.599326599324, "avg_output_tokens_per_task": 1582.158810325477, "last_run_timestamp": "2026-08-29T18:45:20.686Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.4882155000000001, "accuracy_std_dev": 0.016835500000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008046262626262627, "avg_duration_per_task_ms": 20667.484848484848, "avg_output_tokens_per_task": 1820.4444444444443, "last_run_timestamp": "2026-09-01T17:50:43.769Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.4874045119744545, "accuracy_std_dev": 0.019802710459293865, "run_count": 19, "total_questions": 3758, "avg_cost_per_task_usd": 0.0007857953698775944, "avg_duration_per_task_ms": 54493.003459286854, "avg_output_tokens_per_task": 1787.7637040979243, "last_run_timestamp": "2026-08-29T19:00:36.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.48552127324913896, "accuracy_std_dev": 0.10753376064866745, "run_count": 14, "total_questions": 2613, "avg_cost_per_task_usd": 0.010886646000765405, "avg_duration_per_task_ms": 309006.7183314198, "avg_output_tokens_per_task": 14771.444316877152, "last_run_timestamp": "2026-08-01T18:13:08.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.48484835714285707, "accuracy_std_dev": 0.019539890950884275, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.0012388686868686867, "avg_duration_per_task_ms": 18026.963203463205, "avg_output_tokens_per_task": 1322.105339105339, "last_run_timestamp": "2026-07-18T16:13:31.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Inceptron", "accuracy": 0.4848483333333333, "accuracy_std_dev": 0.01797495489717725, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00031208922558922556, "avg_duration_per_task_ms": 18182.459595959597, "avg_output_tokens_per_task": 730.1346801346801, "last_run_timestamp": "2026-05-16T08:12:45.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.4848048041942605, "accuracy_std_dev": 0.0329764460112243, "run_count": 23, "total_questions": 4530, "avg_cost_per_task_usd": 0.006697250110375276, "avg_duration_per_task_ms": 285389.3573951435, "avg_output_tokens_per_task": 16704.50375275938, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.4829639762922659, "accuracy_std_dev": 0.0222392355589962, "run_count": 13, "total_questions": 2573, "avg_cost_per_task_usd": 0.0010490423630003885, "avg_duration_per_task_ms": 37775.31752817723, "avg_output_tokens_per_task": 1882.54061406918, "last_run_timestamp": "2026-08-29T17:00:17.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.481481, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.001115070707070707, "avg_duration_per_task_ms": 120135.48484848485, "avg_output_tokens_per_task": 2056.4848484848485, "last_run_timestamp": "2026-09-01T14:05:19.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.47643079999999993, "accuracy_std_dev": 0.007896357575490103, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0023637252525252526, "avg_duration_per_task_ms": 39731.30202020202, "avg_output_tokens_per_task": 2520.320202020202, "last_run_timestamp": "2026-08-29T18:16:38.845Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4761502777777778, "accuracy_std_dev": 0.019858171124825148, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.0006140196408529742, "avg_duration_per_task_ms": 115870.00168350169, "avg_output_tokens_per_task": 1751.4741863075196, "last_run_timestamp": "2026-08-29T18:48:55.759Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.4751962666666667, "accuracy_std_dev": 0.01117488509988159, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.0007880821548821547, "avg_duration_per_task_ms": 116358.99057239057, "avg_output_tokens_per_task": 1780.7713804713806, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.46742483824768327, "accuracy_std_dev": 0.006948085005956674, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.002137747262005055, "avg_duration_per_task_ms": 34994.02274641955, "avg_output_tokens_per_task": 2587.217354675653, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.46464666666666665, "accuracy_std_dev": 0.24034588812283755, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0018368636363636363, "avg_duration_per_task_ms": 32878.122895622895, "avg_output_tokens_per_task": 3002.4276094276092, "last_run_timestamp": "2026-06-30T19:37:23.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.462542, "accuracy_std_dev": 0.005471000000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001783989898989899, "avg_duration_per_task_ms": 27644.777777777777, "avg_output_tokens_per_task": 1397.5252525252524, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.46128619685039374, "accuracy_std_dev": 0.075159, "run_count": 2, "total_questions": 254, "avg_cost_per_task_usd": 0.00731248031496063, "avg_duration_per_task_ms": 1016477.8543307086, "avg_output_tokens_per_task": 20586.740157480315, "last_run_timestamp": "2026-08-08T10:11:59.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Google", "accuracy": 0.4557567443438914, "accuracy_std_dev": 0.195794758968546, "run_count": 7, "total_questions": 1326, "avg_cost_per_task_usd": 0.002924125942684766, "avg_duration_per_task_ms": 340072.5972850679, "avg_output_tokens_per_task": 15733.245852187029, "last_run_timestamp": "2026-07-11T15:34:14.755Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.4461280000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0009776919191919191, "avg_duration_per_task_ms": 90556.41414141415, "avg_output_tokens_per_task": 1759.8131313131314, "last_run_timestamp": "2026-09-01T14:15:49.249Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.4444445, "accuracy_std_dev": 0.015151500000000012, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0015239469696969696, "avg_duration_per_task_ms": 59255.792929292926, "avg_output_tokens_per_task": 1592.9924242424242, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.44245509090909096, "accuracy_std_dev": 0.40072706253791307, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0037717125803489442, "avg_duration_per_task_ms": 180639.4880624426, "avg_output_tokens_per_task": 3668.438934802571, "last_run_timestamp": "2026-08-15T14:54:58.071Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.4402355, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0013545, "avg_duration_per_task_ms": 68001.7196969697, "avg_output_tokens_per_task": 1535.1843434343434, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.4335015, "accuracy_std_dev": 0.009259500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010084242424242425, "avg_duration_per_task_ms": 21177.93181818182, "avg_output_tokens_per_task": 1486.2676767676767, "last_run_timestamp": "2026-09-01T17:49:46.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4301345, "accuracy_std_dev": 0.009259500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010125732323232322, "avg_duration_per_task_ms": 17946.272727272728, "avg_output_tokens_per_task": 1495.5707070707072, "last_run_timestamp": "2026-09-01T17:55:31.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.4297135, "accuracy_std_dev": 0.007469839539775939, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010784545454545454, "avg_duration_per_task_ms": 21933.41792929293, "avg_output_tokens_per_task": 1515.1376262626263, "last_run_timestamp": "2026-09-01T17:31:07.952Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.41662231578947373, "accuracy_std_dev": 0.25537013940925096, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.0011135746943115364, "avg_duration_per_task_ms": 32676.62706007443, "avg_output_tokens_per_task": 946.1113769271664, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.3860826666666667, "accuracy_std_dev": 0.005722799567422292, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00015977104377104374, "avg_duration_per_task_ms": 6254.590909090909, "avg_output_tokens_per_task": 953.9612794612794, "last_run_timestamp": "2026-09-01T13:54:02.609Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.36363599999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00041437878787878784, "avg_duration_per_task_ms": 34981.92424242424, "avg_output_tokens_per_task": 1665.6666666666667, "last_run_timestamp": "2026-08-01T08:55:19.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Together", "accuracy": 0.3417510000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0007054343434343434, "avg_duration_per_task_ms": 14064.868686868687, "avg_output_tokens_per_task": 1539, "last_run_timestamp": "2026-08-01T08:06:23.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.328283, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005318181818181818, "avg_duration_per_task_ms": 36388.32323232323, "avg_output_tokens_per_task": 2041.050505050505, "last_run_timestamp": "2026-08-01T08:36:11.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.3156760930232559, "accuracy_std_dev": 0.020895107938223238, "run_count": 8, "total_questions": 1548, "avg_cost_per_task_usd": 0.00008417894056847546, "avg_duration_per_task_ms": 8191.765503875969, "avg_output_tokens_per_task": 362.62467700258395, "last_run_timestamp": "2026-07-18T09:08:52.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Nebius", "accuracy": 0.3131315, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00008114646464646465, "avg_duration_per_task_ms": 46871.881313131315, "avg_output_tokens_per_task": 1262.8939393939395, "last_run_timestamp": "2026-05-09T08:25:54.935Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.31154682352941176, "accuracy_std_dev": 0.04799129866994309, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.005848113784907903, "avg_duration_per_task_ms": 288872.33244206774, "avg_output_tokens_per_task": 13122.53505644682, "last_run_timestamp": "2026-08-29T19:48:22.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.30923066923076925, "accuracy_std_dev": 0.02667595582842347, "run_count": 10, "total_questions": 1950, "avg_cost_per_task_usd": 0.0000934446153846154, "avg_duration_per_task_ms": 5045.05282051282, "avg_output_tokens_per_task": 300.8317948717949, "last_run_timestamp": "2026-07-18T09:09:45.839Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.30896005292479106, "accuracy_std_dev": 0.05550927626372292, "run_count": 11, "total_questions": 2154, "avg_cost_per_task_usd": 0.00260005756731662, "avg_duration_per_task_ms": 28138.977251624885, "avg_output_tokens_per_task": 1086.3834726090993, "last_run_timestamp": "2026-08-29T17:41:08.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Novita", "accuracy": 0.3056295271111111, "accuracy_std_dev": 0.05792376099499717, "run_count": 6, "total_questions": 1125, "avg_cost_per_task_usd": 0.00008087733333333333, "avg_duration_per_task_ms": 36138.544, "avg_output_tokens_per_task": 344.34222222222223, "last_run_timestamp": "2026-07-18T08:00:15.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DeepInfra", "accuracy": 0.3026373112840467, "accuracy_std_dev": 0.026204171217336758, "run_count": 8, "total_questions": 1542, "avg_cost_per_task_usd": 0.00002046757457846952, "avg_duration_per_task_ms": 9360.384565499351, "avg_output_tokens_per_task": 275.0188067444877, "last_run_timestamp": "2026-07-18T08:00:15.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.29322554528061223, "accuracy_std_dev": 0.026492655735409588, "run_count": 24, "total_questions": 4704, "avg_cost_per_task_usd": 0.00015754315476190474, "avg_duration_per_task_ms": 151689.59715136053, "avg_output_tokens_per_task": 3673.2096088435374, "last_run_timestamp": "2026-08-29T16:57:41.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.29040400000000005, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.000060893939393939394, "avg_duration_per_task_ms": 9271.39393939394, "avg_output_tokens_per_task": 573.0479797979798, "last_run_timestamp": "2026-09-01T13:00:51.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "DekaLLM", "accuracy": 0.2874569642004773, "accuracy_std_dev": 0.027560829670429898, "run_count": 7, "total_questions": 1257, "avg_cost_per_task_usd": 0.000016271280827366746, "avg_duration_per_task_ms": 11521.354017501988, "avg_output_tokens_per_task": 197.71917263325378, "last_run_timestamp": "2026-07-18T08:04:03.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.28451170588235297, "accuracy_std_dev": 0.025856077905943135, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.0006791901366607251, "avg_duration_per_task_ms": 47282.23143196673, "avg_output_tokens_per_task": 8804.306892453951, "last_run_timestamp": "2026-08-29T18:21:00.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.2780711647307286, "accuracy_std_dev": 0.014628725522067875, "run_count": 5, "total_questions": 947, "avg_cost_per_task_usd": 0.0063300253431890175, "avg_duration_per_task_ms": 245873.9429778247, "avg_output_tokens_per_task": 28094.863780359028, "last_run_timestamp": "2026-08-29T18:18:26.937Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.275533, "accuracy_std_dev": 0.009152576176501717, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0007405521885521885, "avg_duration_per_task_ms": 48360.410774410775, "avg_output_tokens_per_task": 2532.1531986531986, "last_run_timestamp": "2026-08-15T09:28:07.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cerebras", "accuracy": 0.2727275, "accuracy_std_dev": 0.035353499999999996, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00008969696969696971, "avg_duration_per_task_ms": 633.3914141414141, "avg_output_tokens_per_task": 628.320707070707, "last_run_timestamp": "2026-05-23T08:04:59.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.27185566666666666, "accuracy_std_dev": 0.008747009330178078, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.00006098307952622674, "avg_duration_per_task_ms": 9307.97461928934, "avg_output_tokens_per_task": 612.7817258883249, "last_run_timestamp": "2026-09-01T12:41:31.862Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.26936, "accuracy_std_dev": 0.00673399999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0000632449494949495, "avg_duration_per_task_ms": 9455.388888888889, "avg_output_tokens_per_task": 592.1616161616162, "last_run_timestamp": "2026-09-01T13:21:46.100Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.268788525198939, "accuracy_std_dev": 0.004147499999999998, "run_count": 2, "total_questions": 377, "avg_cost_per_task_usd": 0.0000679681697612732, "avg_duration_per_task_ms": 9180.114058355437, "avg_output_tokens_per_task": 568.5623342175066, "last_run_timestamp": "2026-09-01T13:04:23.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.26757776470588235, "accuracy_std_dev": 0.021117749736764967, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.000780840760546643, "avg_duration_per_task_ms": 19816.014557338087, "avg_output_tokens_per_task": 9728.812834224598, "last_run_timestamp": "2026-08-29T18:45:59.793Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.256419686746988, "accuracy_std_dev": 0.07325243553772927, "run_count": 14, "total_questions": 2739, "avg_cost_per_task_usd": 0.001453557867834976, "avg_duration_per_task_ms": 454232.56005841546, "avg_output_tokens_per_task": 5530.41730558598, "last_run_timestamp": "2026-08-29T17:16:14.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "WandB", "accuracy": 0.2561763187294633, "accuracy_std_dev": 0.022252850067062038, "run_count": 14, "total_questions": 2739, "avg_cost_per_task_usd": 0.0006673464768163562, "avg_duration_per_task_ms": 19297.468054034318, "avg_output_tokens_per_task": 2614.9587440671776, "last_run_timestamp": "2026-07-18T08:26:34.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Ambient", "accuracy": 0.245614, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.03239533333333334, "avg_duration_per_task_ms": 323216.8245614035, "avg_output_tokens_per_task": 11659.74269005848, "last_run_timestamp": "2026-06-30T21:29:29.599Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Friendli", "accuracy": 0.242424, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00020296464646464647, "avg_duration_per_task_ms": 11928.080808080807, "avg_output_tokens_per_task": 1760.0050505050506, "last_run_timestamp": "2026-05-09T08:06:13.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.24149673724489795, "accuracy_std_dev": 0.024442688065230603, "run_count": 12, "total_questions": 2352, "avg_cost_per_task_usd": 0.00032777253401360546, "avg_duration_per_task_ms": 55241.83588435374, "avg_output_tokens_per_task": 6268.276785714285, "last_run_timestamp": "2026-08-29T16:55:14.828Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.2372174911504425, "accuracy_std_dev": 0.3611729272686932, "run_count": 7, "total_questions": 1356, "avg_cost_per_task_usd": 0.0035571969026548674, "avg_duration_per_task_ms": 170613.9918879056, "avg_output_tokens_per_task": 3415.4498525073745, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Parasail", "accuracy": 0.21734899999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 171, "avg_cost_per_task_usd": 0.0003523450292397661, "avg_duration_per_task_ms": 72727.70760233919, "avg_output_tokens_per_task": 4269.023391812865, "last_run_timestamp": "2026-08-01T08:05:56.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.19696999999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0190694696969697, "avg_duration_per_task_ms": 2493159.101010101, "avg_output_tokens_per_task": 105666.86363636363, "last_run_timestamp": "2026-09-03T21:14:17.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "auto-routing", "accuracy": 0.116162, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0001537979797979798, "avg_duration_per_task_ms": 29722.909090909092, "avg_output_tokens_per_task": 1836.7929292929293, "last_run_timestamp": "2026-08-01T09:03:02.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.114478, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.017837661616161616, "avg_duration_per_task_ms": 71618.32323232324, "avg_output_tokens_per_task": 11767.727272727272, "last_run_timestamp": "2026-09-01T12:56:59.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.10606100000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.017886560606060607, "avg_duration_per_task_ms": 21072.777777777777, "avg_output_tokens_per_task": 11801.328282828283, "last_run_timestamp": "2026-09-01T13:00:01.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018059262626262628, "avg_duration_per_task_ms": 21777.424242424244, "avg_output_tokens_per_task": 11808.9898989899, "last_run_timestamp": "2026-09-01T13:21:37.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.060606, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.000040161616161616165, "avg_duration_per_task_ms": 3671.1060606060605, "avg_output_tokens_per_task": 93.72222222222223, "last_run_timestamp": "2026-08-01T08:05:34.450Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.806667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20567228, "avg_duration_per_task_ms": 69455.26, "avg_output_tokens_per_task": 15737.28, "last_run_timestamp": "2026-08-15T00:15:11.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8057142857142857, "accuracy_std_dev": 0.02209041203595731, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.8699426371428572, "avg_duration_per_task_ms": 139011.88, "avg_output_tokens_per_task": 5785.405714285715, "last_run_timestamp": "2026-09-19T08:15:00.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.805556, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.07709047916666667, "avg_duration_per_task_ms": 121300.85416666667, "avg_output_tokens_per_task": 14694.6875, "last_run_timestamp": "2026-08-15T00:15:15.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8, "accuracy_std_dev": 0.02805880446490904, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.9499947457142858, "avg_duration_per_task_ms": 132744.98857142858, "avg_output_tokens_per_task": 5787.242857142857, "last_run_timestamp": "2026-09-19T08:01:53.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8, "accuracy_std_dev": 0.033110611481316166, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.7322859599999999, "avg_duration_per_task_ms": 146639.07333333333, "avg_output_tokens_per_task": 5472.34, "last_run_timestamp": "2026-08-15T08:09:14.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.7977776666666667, "accuracy_std_dev": 0.01911642045177097, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008363246666666668, "avg_duration_per_task_ms": 272879.14, "avg_output_tokens_per_task": 12625.026666666667, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7973334, "accuracy_std_dev": 0.014360451867542312, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.873586176, "avg_duration_per_task_ms": 155291.072, "avg_output_tokens_per_task": 5726.508, "last_run_timestamp": "2026-09-19T08:14:32.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Nebius", "accuracy": 0.7966666666666666, "accuracy_std_dev": 0.021343747458109477, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08247077333333334, "avg_duration_per_task_ms": 126277.50333333333, "avg_output_tokens_per_task": 8287.77, "last_run_timestamp": "2026-05-23T17:33:06.915Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.795, "accuracy_std_dev": 0.019649399202011232, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.104553105, "avg_duration_per_task_ms": 447536.345, "avg_output_tokens_per_task": 18774.07, "last_run_timestamp": "2026-09-19T08:12:56.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.7942857142857143, "accuracy_std_dev": 0.014443616234114186, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.8340979885714286, "avg_duration_per_task_ms": 141474.59714285715, "avg_output_tokens_per_task": 5747.588571428571, "last_run_timestamp": "2026-09-19T08:01:53.308Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.7933333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.8155115399999999, "avg_duration_per_task_ms": 189088.09333333332, "avg_output_tokens_per_task": 7070.38, "last_run_timestamp": "2026-09-19T08:02:29.967Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.792381, "accuracy_std_dev": 0.022375873792994085, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.49301859142857146, "avg_duration_per_task_ms": 131851.76571428572, "avg_output_tokens_per_task": 7199.7, "last_run_timestamp": "2026-09-19T08:01:53.318Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.79, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08253803, "avg_duration_per_task_ms": 85518.8, "avg_output_tokens_per_task": 4440.97, "last_run_timestamp": "2026-09-19T08:01:53.363Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7884616923076923, "accuracy_std_dev": 0.02158726126280039, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.9622519753846155, "avg_duration_per_task_ms": 100478.9923076923, "avg_output_tokens_per_task": 5605.658461538462, "last_run_timestamp": "2026-09-19T08:15:41.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.8471384599999998, "avg_duration_per_task_ms": 331793.46, "avg_output_tokens_per_task": 22741.94, "last_run_timestamp": "2026-08-01T08:06:01.602Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7866669999999999, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2795196199999999, "avg_duration_per_task_ms": 104647.27, "avg_output_tokens_per_task": 5686.75, "last_run_timestamp": "2026-09-01T13:03:28.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.7866666, "accuracy_std_dev": 0.005962698872155093, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.118893224, "avg_duration_per_task_ms": 130658.76, "avg_output_tokens_per_task": 11469.216, "last_run_timestamp": "2026-09-20T10:30:07.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.7866665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12536731, "avg_duration_per_task_ms": 118841.15, "avg_output_tokens_per_task": 19578.58, "last_run_timestamp": "2026-09-01T17:30:58.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.7859532642140468, "accuracy_std_dev": 0.022060806735148097, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.028781180602006688, "avg_duration_per_task_ms": 346700.6989966555, "avg_output_tokens_per_task": 11874.802675585284, "last_run_timestamp": "2026-08-29T17:06:21.366Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Morph", "accuracy": 0.7858823529411765, "accuracy_std_dev": 0.0414686149812562, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.08477611529411765, "avg_duration_per_task_ms": 198992.7294117647, "avg_output_tokens_per_task": 11058.981176470588, "last_run_timestamp": "2026-07-04T16:00:23.544Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7857143728222997, "accuracy_std_dev": 0.022636034639976627, "run_count": 6, "total_questions": 287, "avg_cost_per_task_usd": 0.11757857142857142, "avg_duration_per_task_ms": 73705.68641114983, "avg_output_tokens_per_task": 8241.627177700348, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.78533336, "accuracy_std_dev": 0.0372379576957491, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.10169507999999998, "avg_duration_per_task_ms": 377017.644, "avg_output_tokens_per_task": 16270.1992, "last_run_timestamp": "2026-08-29T16:14:58.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.785, "accuracy_std_dev": 0.010928810045014038, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08068941, "avg_duration_per_task_ms": 125123.275, "avg_output_tokens_per_task": 4412.62, "last_run_timestamp": "2026-09-19T08:07:46.498Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.7838095714285714, "accuracy_std_dev": 0.023599102563790993, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.46679962, "avg_duration_per_task_ms": 112990.85857142857, "avg_output_tokens_per_task": 7266.794285714286, "last_run_timestamp": "2026-09-19T08:08:46.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.7833334999999999, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.7261046600000001, "avg_duration_per_task_ms": 117322.17, "avg_output_tokens_per_task": 6966.33, "last_run_timestamp": "2026-09-19T08:08:24.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.7833334999999999, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08340285, "avg_duration_per_task_ms": 39163.99, "avg_output_tokens_per_task": 4681.96, "last_run_timestamp": "2026-09-19T08:11:03.394Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "ModelRun", "accuracy": 0.7822223333333334, "accuracy_std_dev": 0.013698571612479267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06831790666666666, "avg_duration_per_task_ms": 97592.92, "avg_output_tokens_per_task": 11907.126666666667, "last_run_timestamp": "2026-07-18T08:04:03.432Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.7822223333333334, "accuracy_std_dev": 0.031427125187575726, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10594911333333333, "avg_duration_per_task_ms": 72818.32666666666, "avg_output_tokens_per_task": 4800.32, "last_run_timestamp": "2026-09-12T08:12:03.011Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.7816665, "accuracy_std_dev": 0.028431350069069865, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.15782728499999998, "avg_duration_per_task_ms": 597754.365, "avg_output_tokens_per_task": 12437.015, "last_run_timestamp": "2026-09-19T08:01:36.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.7809090909090909, "accuracy_std_dev": 0.024968384418025142, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.5751214472727272, "avg_duration_per_task_ms": 120619.24545454545, "avg_output_tokens_per_task": 7275.691818181818, "last_run_timestamp": "2026-09-19T08:15:48.414Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7806349047619048, "accuracy_std_dev": 0.023267850671002527, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.48060664571428574, "avg_duration_per_task_ms": 128644.07238095238, "avg_output_tokens_per_task": 7184.6923809523805, "last_run_timestamp": "2026-09-19T08:12:50.307Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepSeek", "accuracy": 0.78, "accuracy_std_dev": 0.021081640250859665, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.016256343333333336, "avg_duration_per_task_ms": 172479.31, "avg_output_tokens_per_task": 11984.07, "last_run_timestamp": "2026-08-08T08:29:14.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.78, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03263777, "avg_duration_per_task_ms": 159319.31, "avg_output_tokens_per_task": 15620.74, "last_run_timestamp": "2026-09-19T08:12:48.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Phala", "accuracy": 0.78, "accuracy_std_dev": 1.1102230246251565e-16, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.14591014000000002, "avg_duration_per_task_ms": 245617.41333333333, "avg_output_tokens_per_task": 6015.373333333333, "last_run_timestamp": "2026-07-18T16:00:09.009Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Fireworks", "accuracy": 0.78, "accuracy_std_dev": 0.028284271247461888, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.031875935, "avg_duration_per_task_ms": 94447.85, "avg_output_tokens_per_task": 3704.84, "last_run_timestamp": "2026-04-25T14:38:47.693Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.77916675, "accuracy_std_dev": 0.022715135068660724, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.4318646125000001, "avg_duration_per_task_ms": 135663.39, "avg_output_tokens_per_task": 7317.8775, "last_run_timestamp": "2026-09-19T08:14:21.987Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.7789859999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.333418, "avg_duration_per_task_ms": 227429.76086956522, "avg_output_tokens_per_task": 5856.152173913043, "last_run_timestamp": "2026-08-08T08:09:44.831Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.777778, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.7786092066666667, "avg_duration_per_task_ms": 116605.5, "avg_output_tokens_per_task": 6845.246666666667, "last_run_timestamp": "2026-09-19T08:02:40.695Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ambient", "accuracy": 0.777778, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.012714583333333333, "avg_duration_per_task_ms": 516568.7291666667, "avg_output_tokens_per_task": 14230.208333333334, "last_run_timestamp": "2026-08-03T17:20:47.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.777777825958702, "accuracy_std_dev": 0.019674442666690654, "run_count": 7, "total_questions": 339, "avg_cost_per_task_usd": 0.06404329498525074, "avg_duration_per_task_ms": 85698.01474926254, "avg_output_tokens_per_task": 4107.47197640118, "last_run_timestamp": "2026-09-19T08:08:24.135Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Together", "accuracy": 0.7775, "accuracy_std_dev": 0.028173569173961615, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.12033262749999998, "avg_duration_per_task_ms": 104051.4975, "avg_output_tokens_per_task": 15655.525, "last_run_timestamp": "2026-06-20T17:08:55.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.7773333, "accuracy_std_dev": 0.014047595289230106, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.434012314, "avg_duration_per_task_ms": 97749.408, "avg_output_tokens_per_task": 5681.338, "last_run_timestamp": "2026-09-19T08:07:34.969Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.7771428571428571, "accuracy_std_dev": 0.030207219333078877, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.47215102000000003, "avg_duration_per_task_ms": 119873.53142857143, "avg_output_tokens_per_task": 7280.5142857142855, "last_run_timestamp": "2026-09-19T08:15:47.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7770272162162162, "accuracy_std_dev": 0.015058546012148703, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.17008879054054055, "avg_duration_per_task_ms": 358528.91891891893, "avg_output_tokens_per_task": 20610.358108108107, "last_run_timestamp": "2026-09-19T08:15:57.539Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepSeek", "accuracy": 0.7766665, "accuracy_std_dev": 0.019148513142887433, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.008114253333333333, "avg_duration_per_task_ms": 195047.99, "avg_output_tokens_per_task": 21425.716666666667, "last_run_timestamp": "2026-08-08T08:30:38.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7766665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06410135, "avg_duration_per_task_ms": 123064.78, "avg_output_tokens_per_task": 4532.5, "last_run_timestamp": "2026-09-19T08:14:24.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7760001999999999, "accuracy_std_dev": 0.041601285236877006, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08482736, "avg_duration_per_task_ms": 237369.612, "avg_output_tokens_per_task": 11416.556, "last_run_timestamp": "2026-09-19T08:07:09.158Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.775, "accuracy_std_dev": 0.025549537941027427, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.007826415, "avg_duration_per_task_ms": 211518.38, "avg_output_tokens_per_task": 4265.165, "last_run_timestamp": "2026-09-19T08:02:29.952Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7749997500000001, "accuracy_std_dev": 0.021794360891925678, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.21402777999999997, "avg_duration_per_task_ms": 289883.17, "avg_output_tokens_per_task": 10604.905, "last_run_timestamp": "2026-09-19T08:06:32.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.7747619285714286, "accuracy_std_dev": 0.03189058782655896, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.44453945142857143, "avg_duration_per_task_ms": 80162.03857142857, "avg_output_tokens_per_task": 5717.024285714286, "last_run_timestamp": "2026-09-19T08:12:06.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.7746666, "accuracy_std_dev": 0.025785425686616063, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.473358384, "avg_duration_per_task_ms": 128837.64, "avg_output_tokens_per_task": 7803.012, "last_run_timestamp": "2026-09-19T08:15:49.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.7744443333333334, "accuracy_std_dev": 0.035986907770589154, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06337854666666667, "avg_duration_per_task_ms": 170014.64, "avg_output_tokens_per_task": 7038.76, "last_run_timestamp": "2026-08-29T16:51:02.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.7740476428571429, "accuracy_std_dev": 0.027736848735950892, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.030251097857142858, "avg_duration_per_task_ms": 120961.04571428572, "avg_output_tokens_per_task": 5680.855, "last_run_timestamp": "2026-08-29T17:11:32.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08473178, "avg_duration_per_task_ms": 234930.66, "avg_output_tokens_per_task": 22523.52, "last_run_timestamp": "2026-09-20T10:30:07.955Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.7733333333333333, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02996874, "avg_duration_per_task_ms": 113373.20666666667, "avg_output_tokens_per_task": 12010.053333333333, "last_run_timestamp": "2026-08-29T16:46:02.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.7733333333333333, "accuracy_std_dev": 0.02177310607658502, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06427175333333333, "avg_duration_per_task_ms": 56109.4, "avg_output_tokens_per_task": 4673.253333333333, "last_run_timestamp": "2026-09-19T08:02:29.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepSeek", "accuracy": 0.7733331666666667, "accuracy_std_dev": 0.00384890557676971, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.006986003333333334, "avg_duration_per_task_ms": 162086.69, "avg_output_tokens_per_task": 16532.63, "last_run_timestamp": "2026-08-08T08:27:33.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.773333, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12487065999999998, "avg_duration_per_task_ms": 149585.85, "avg_output_tokens_per_task": 20477.48, "last_run_timestamp": "2026-09-01T17:31:27.257Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02012318, "avg_duration_per_task_ms": 229027.46, "avg_output_tokens_per_task": 10940.04, "last_run_timestamp": "2026-07-18T08:04:46.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.7728203846153846, "accuracy_std_dev": 0.029289100594270586, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.49779368153846165, "avg_duration_per_task_ms": 130806.25846153845, "avg_output_tokens_per_task": 7936.776923076923, "last_run_timestamp": "2026-09-19T08:16:40.718Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.772727090909091, "accuracy_std_dev": 0.03440164644700108, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.14892930181818181, "avg_duration_per_task_ms": 546664.8418181818, "avg_output_tokens_per_task": 35137.674545454545, "last_run_timestamp": "2026-09-19T08:12:02.486Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7726666, "accuracy_std_dev": 0.03641728405359191, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.113975304, "avg_duration_per_task_ms": 177276.2, "avg_output_tokens_per_task": 9278.056, "last_run_timestamp": "2026-08-29T16:15:12.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.7723808571428572, "accuracy_std_dev": 0.018662706092468933, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.4297011400000001, "avg_duration_per_task_ms": 103448.23714285715, "avg_output_tokens_per_task": 5585.251428571429, "last_run_timestamp": "2026-09-19T08:14:49.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.772, "accuracy_std_dev": 0.01808611499465822, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.137215892, "avg_duration_per_task_ms": 326138.736, "avg_output_tokens_per_task": 19302.048, "last_run_timestamp": "2026-09-19T08:05:35.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.772, "accuracy_std_dev": 0.01359732000064717, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.099907088, "avg_duration_per_task_ms": 347432.108, "avg_output_tokens_per_task": 19671.476, "last_run_timestamp": "2026-09-19T08:02:29.955Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.7714287142857142, "accuracy_std_dev": 0.024615140153480954, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.53567294, "avg_duration_per_task_ms": 174941.87714285715, "avg_output_tokens_per_task": 10495.22, "last_run_timestamp": "2026-09-19T08:07:29.514Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.7713334, "accuracy_std_dev": 0.0331394434343729, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.122264977, "avg_duration_per_task_ms": 407781.64, "avg_output_tokens_per_task": 17384.362, "last_run_timestamp": "2026-08-29T17:01:22.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.7713333, "accuracy_std_dev": 0.0188680293992245, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.4928351919999999, "avg_duration_per_task_ms": 131035.412, "avg_output_tokens_per_task": 8091.076, "last_run_timestamp": "2026-09-19T08:08:44.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7712177472324724, "accuracy_std_dev": 0.052442826700422794, "run_count": 11, "total_questions": 542, "avg_cost_per_task_usd": 0.09838513653136532, "avg_duration_per_task_ms": 1843075.8265682657, "avg_output_tokens_per_task": 19082.356088560886, "last_run_timestamp": "2026-08-29T16:45:34.216Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.7711113333333333, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08392440666666666, "avg_duration_per_task_ms": 191039.60666666666, "avg_output_tokens_per_task": 7574.006666666667, "last_run_timestamp": "2026-09-19T08:01:53.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.7706666, "accuracy_std_dev": 0.024073989653565938, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.008760292, "avg_duration_per_task_ms": 280566.352, "avg_output_tokens_per_task": 6313.424, "last_run_timestamp": "2026-09-20T10:30:07.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.77, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07565745, "avg_duration_per_task_ms": 364682.91, "avg_output_tokens_per_task": 21233.7, "last_run_timestamp": "2026-09-19T08:01:53.308Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7699997500000001, "accuracy_std_dev": 0.015275197762631398, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.100920285, "avg_duration_per_task_ms": 129822.61, "avg_output_tokens_per_task": 4551.79, "last_run_timestamp": "2026-09-19T08:15:16.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "WandB", "accuracy": 0.7695238571428571, "accuracy_std_dev": 0.017036849485986643, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10181366857142858, "avg_duration_per_task_ms": 251127.43142857144, "avg_output_tokens_per_task": 7309.714285714285, "last_run_timestamp": "2026-07-18T08:04:04.764Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Parasail", "accuracy": 0.7694109317269077, "accuracy_std_dev": 0.035180887422576496, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.05336053413654618, "avg_duration_per_task_ms": 321533.1927710843, "avg_output_tokens_per_task": 6818.439759036145, "last_run_timestamp": "2026-08-01T17:45:43.989Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7693334, "accuracy_std_dev": 0.029992482022000132, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.554491632, "avg_duration_per_task_ms": 184901.904, "avg_output_tokens_per_task": 10536.404, "last_run_timestamp": "2026-09-19T08:02:29.873Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "auto-routing", "accuracy": 0.7688889999999999, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10034226, "avg_duration_per_task_ms": 157612.40666666668, "avg_output_tokens_per_task": 8568.786666666667, "last_run_timestamp": "2026-06-30T19:14:03.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.7688889999999999, "accuracy_std_dev": 0.003142382535593046, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07201002000000001, "avg_duration_per_task_ms": 47850.82666666667, "avg_output_tokens_per_task": 4678.52, "last_run_timestamp": "2026-09-19T08:15:44.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.7687072040816327, "accuracy_std_dev": 0.0156945, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 1.7723489081632653, "avg_duration_per_task_ms": 313060.2448979592, "avg_output_tokens_per_task": 23077.85714285714, "last_run_timestamp": "2026-08-01T08:06:07.759Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.7686666, "accuracy_std_dev": 0.023438065906554666, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.058330564, "avg_duration_per_task_ms": 133631.792, "avg_output_tokens_per_task": 4186.012, "last_run_timestamp": "2026-09-19T08:16:41.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "CoreWeave", "accuracy": 0.7684563154362417, "accuracy_std_dev": 0.006017520216464235, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1581116510067114, "avg_duration_per_task_ms": 537705.5637583892, "avg_output_tokens_per_task": 10979.71812080537, "last_run_timestamp": "2026-08-08T08:27:49.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.7683858338108882, "accuracy_std_dev": 0.03056258771966879, "run_count": 14, "total_questions": 698, "avg_cost_per_task_usd": 0.028291176217765036, "avg_duration_per_task_ms": 202040.56017191976, "avg_output_tokens_per_task": 8723.312320916906, "last_run_timestamp": "2026-08-29T16:33:11.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.7683335, "accuracy_std_dev": 0.025980986639271425, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.074955525, "avg_duration_per_task_ms": 242542.055, "avg_output_tokens_per_task": 12922.96, "last_run_timestamp": "2026-09-19T08:01:36.428Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.767573537414966, "accuracy_std_dev": 0.02493140611535758, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.062283115646258505, "avg_duration_per_task_ms": 166794.86394557822, "avg_output_tokens_per_task": 4126.721088435374, "last_run_timestamp": "2026-09-19T08:15:43.811Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.7673684210526316, "accuracy_std_dev": 0.027435852827077042, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.06853673368421052, "avg_duration_per_task_ms": 181102.96, "avg_output_tokens_per_task": 7826.743157894737, "last_run_timestamp": "2026-08-29T16:59:11.598Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7671691306532663, "accuracy_std_dev": 0.03712240837960737, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.11847029773869346, "avg_duration_per_task_ms": 600977.8140703518, "avg_output_tokens_per_task": 7982.221105527638, "last_run_timestamp": "2026-08-29T16:54:14.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.7670723286004056, "accuracy_std_dev": 0.034808554659020254, "run_count": 10, "total_questions": 493, "avg_cost_per_task_usd": 0.0794524705882353, "avg_duration_per_task_ms": 974078.9594320487, "avg_output_tokens_per_task": 25171.47261663286, "last_run_timestamp": "2026-09-19T16:00:24.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.7669025283018868, "accuracy_std_dev": 0.01965390363525289, "run_count": 17, "total_questions": 848, "avg_cost_per_task_usd": 0.06553669221698114, "avg_duration_per_task_ms": 138580.51533018867, "avg_output_tokens_per_task": 8122.042452830188, "last_run_timestamp": "2026-08-29T16:41:08.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.025223939999999997, "avg_duration_per_task_ms": 165137.1, "avg_output_tokens_per_task": 14240.82, "last_run_timestamp": "2026-09-15T00:00:05.846Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.86423468, "avg_duration_per_task_ms": 335268.18, "avg_output_tokens_per_task": 22931.04, "last_run_timestamp": "2026-08-01T08:05:35.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.045774379999999996, "avg_duration_per_task_ms": 165566.44, "avg_output_tokens_per_task": 12426.8, "last_run_timestamp": "2026-09-20T21:00:04.179Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.76666675, "accuracy_std_dev": 0.028284389098714877, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.080202585, "avg_duration_per_task_ms": 46082.84, "avg_output_tokens_per_task": 4823.05, "last_run_timestamp": "2026-09-19T08:16:29.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.019625871060640555, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.8067599599999999, "avg_duration_per_task_ms": 188419.76, "avg_output_tokens_per_task": 6949.133333333333, "last_run_timestamp": "2026-09-19T08:01:53.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12935825333333334, "avg_duration_per_task_ms": 58122.44, "avg_output_tokens_per_task": 4752.84, "last_run_timestamp": "2026-09-19T08:15:57.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08184152, "avg_duration_per_task_ms": 115682.96666666666, "avg_output_tokens_per_task": 4471.64, "last_run_timestamp": "2026-09-19T08:13:25.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.7666666, "accuracy_std_dev": 0.03910112246266086, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07291049200000001, "avg_duration_per_task_ms": 332506.232, "avg_output_tokens_per_task": 12583.292, "last_run_timestamp": "2026-09-19T08:01:53.355Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7666665454545455, "accuracy_std_dev": 0.014494959069612862, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.017875983636363633, "avg_duration_per_task_ms": 241228.09454545454, "avg_output_tokens_per_task": 17448.958181818183, "last_run_timestamp": "2026-09-19T08:08:46.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7666665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.21096532, "avg_duration_per_task_ms": 124417.44, "avg_output_tokens_per_task": 5140.16, "last_run_timestamp": "2026-09-01T12:59:37.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.7661904285714286, "accuracy_std_dev": 0.01872337959311179, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.019808859999999998, "avg_duration_per_task_ms": 510174.85142857145, "avg_output_tokens_per_task": 13353.934285714286, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AkashML", "accuracy": 0.766, "accuracy_std_dev": 0.0200997512422418, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.073923902, "avg_duration_per_task_ms": 82060.868, "avg_output_tokens_per_task": 4072.032, "last_run_timestamp": "2026-06-13T11:18:01.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.76599995, "accuracy_std_dev": 0.022201089564422297, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.5305423589999999, "avg_duration_per_task_ms": 122447.576, "avg_output_tokens_per_task": 8112.119, "last_run_timestamp": "2026-09-19T08:14:26.593Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.7658824117647058, "accuracy_std_dev": 0.041720086264163854, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.010050117647058824, "avg_duration_per_task_ms": 111496.44235294117, "avg_output_tokens_per_task": 10075.014117647059, "last_run_timestamp": "2026-08-29T16:53:19.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7655556666666666, "accuracy_std_dev": 0.0355206634353708, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.23852161333333333, "avg_duration_per_task_ms": 442810.72, "avg_output_tokens_per_task": 14679.013333333334, "last_run_timestamp": "2026-09-19T08:02:40.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.7653968095238095, "accuracy_std_dev": 0.03808201053443309, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.051203074285714294, "avg_duration_per_task_ms": 167458.42666666667, "avg_output_tokens_per_task": 7624.500952380952, "last_run_timestamp": "2026-08-29T17:02:01.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Io Net", "accuracy": 0.7653333999999999, "accuracy_std_dev": 0.01995535461574161, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.233724332, "avg_duration_per_task_ms": 181237.788, "avg_output_tokens_per_task": 6561.384, "last_run_timestamp": "2026-07-18T08:04:09.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7652380714285715, "accuracy_std_dev": 0.03138900937194484, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.5665733142857142, "avg_duration_per_task_ms": 182477.46285714285, "avg_output_tokens_per_task": 10583.794285714286, "last_run_timestamp": "2026-09-19T08:14:01.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.7651851481481481, "accuracy_std_dev": 0.03854701154165998, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.08648212370370369, "avg_duration_per_task_ms": 139440.87037037036, "avg_output_tokens_per_task": 5742.8592592592595, "last_run_timestamp": "2026-08-29T17:16:15.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.7650793809523809, "accuracy_std_dev": 0.034969675378037926, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.010378357142857146, "avg_duration_per_task_ms": 113551.27047619048, "avg_output_tokens_per_task": 9004.36761904762, "last_run_timestamp": "2026-08-29T17:17:18.830Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7644446666666667, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13677892666666666, "avg_duration_per_task_ms": 241869.39333333334, "avg_output_tokens_per_task": 11667.466666666667, "last_run_timestamp": "2026-08-08T08:29:31.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7644444761904762, "accuracy_std_dev": 0.03577317233509074, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.12052889047619045, "avg_duration_per_task_ms": 163976.96857142856, "avg_output_tokens_per_task": 5957.86, "last_run_timestamp": "2026-08-29T16:12:45.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.7644444444444445, "accuracy_std_dev": 0.020608041101101583, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.027644917777777778, "avg_duration_per_task_ms": 219174.32, "avg_output_tokens_per_task": 5612.551111111111, "last_run_timestamp": "2026-06-20T10:37:11.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.7642424545454546, "accuracy_std_dev": 0.014290618654231216, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.016105059999999997, "avg_duration_per_task_ms": 208696.1781818182, "avg_output_tokens_per_task": 19948.01090909091, "last_run_timestamp": "2026-09-19T08:21:12.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.7641560048076923, "accuracy_std_dev": 0.054154465237873047, "run_count": 25, "total_questions": 1248, "avg_cost_per_task_usd": 0.1535889431089744, "avg_duration_per_task_ms": 302284.3413461539, "avg_output_tokens_per_task": 17197.23798076923, "last_run_timestamp": "2026-08-29T16:13:54.412Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.7641026153846153, "accuracy_std_dev": 0.022423789261676347, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.042344887692307695, "avg_duration_per_task_ms": 152484.15384615384, "avg_output_tokens_per_task": 9627.38153846154, "last_run_timestamp": "2026-08-29T17:07:19.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.7639931779448622, "accuracy_std_dev": 0.027369852484074517, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.0707108245614035, "avg_duration_per_task_ms": 77146.8395989975, "avg_output_tokens_per_task": 13251.315789473685, "last_run_timestamp": "2026-08-29T17:41:16.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.7633335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.34846716, "avg_duration_per_task_ms": 136967.1, "avg_output_tokens_per_task": 12613.18, "last_run_timestamp": "2026-09-01T17:30:16.052Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.7633335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09941562, "avg_duration_per_task_ms": 141031.62, "avg_output_tokens_per_task": 6151.46, "last_run_timestamp": "2026-09-19T08:12:15.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7633331666666667, "accuracy_std_dev": 0.02631724543853748, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.5436579066666667, "avg_duration_per_task_ms": 181008.31666666668, "avg_output_tokens_per_task": 7997.5633333333335, "last_run_timestamp": "2026-09-01T13:02:58.914Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.7631372941176471, "accuracy_std_dev": 0.035211220731239695, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.05713043764705882, "avg_duration_per_task_ms": 206801.39411764705, "avg_output_tokens_per_task": 9987.18, "last_run_timestamp": "2026-08-29T17:42:47.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.7628574285714285, "accuracy_std_dev": 0.018807973414463997, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.4976220542857142, "avg_duration_per_task_ms": 133220.5942857143, "avg_output_tokens_per_task": 8119.648571428572, "last_run_timestamp": "2026-09-19T08:12:01.797Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7626666, "accuracy_std_dev": 0.03227647784749755, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.146569104, "avg_duration_per_task_ms": 373419.58, "avg_output_tokens_per_task": 24831.152, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Io Net", "accuracy": 0.7626261818181818, "accuracy_std_dev": 0.028670609588697295, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.021812868686868686, "avg_duration_per_task_ms": 396305.23232323234, "avg_output_tokens_per_task": 9161.318181818182, "last_run_timestamp": "2026-08-01T08:05:58.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.021998563367234276, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3364002133333333, "avg_duration_per_task_ms": 240112.26, "avg_output_tokens_per_task": 11849.62, "last_run_timestamp": "2026-09-01T13:04:16.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10776526666666666, "avg_duration_per_task_ms": 82608.36666666667, "avg_output_tokens_per_task": 4630.54, "last_run_timestamp": "2026-09-19T08:12:48.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7622221666666666, "accuracy_std_dev": 0.026851245460975486, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03588941, "avg_duration_per_task_ms": 516654.7166666667, "avg_output_tokens_per_task": 13637.093333333334, "last_run_timestamp": "2026-09-19T08:06:14.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.762, "accuracy_std_dev": 0.029522112241504674, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.517524358, "avg_duration_per_task_ms": 222700.632, "avg_output_tokens_per_task": 10257.826, "last_run_timestamp": "2026-09-19T08:15:44.074Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Chutes", "accuracy": 0.7619049795918367, "accuracy_std_dev": 0.01847199999999999, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.1408940612244898, "avg_duration_per_task_ms": 345711.98979591834, "avg_output_tokens_per_task": 8276.57142857143, "last_run_timestamp": "2026-08-01T08:06:00.902Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.7619047142857143, "accuracy_std_dev": 0.022172283571511325, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.41295487714285717, "avg_duration_per_task_ms": 99216.78285714285, "avg_output_tokens_per_task": 5645.82, "last_run_timestamp": "2026-09-19T08:02:29.843Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.76166675, "accuracy_std_dev": 0.002886895683515417, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.070178685, "avg_duration_per_task_ms": 33757.38, "avg_output_tokens_per_task": 4761.855, "last_run_timestamp": "2026-09-19T08:08:46.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Ambient", "accuracy": 0.7616666666666667, "accuracy_std_dev": 0.03782268572636741, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09442261499999999, "avg_duration_per_task_ms": 188047.78333333333, "avg_output_tokens_per_task": 3614.1383333333333, "last_run_timestamp": "2026-06-20T09:40:45.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.7615604913294798, "accuracy_std_dev": 0.02313264800127777, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.20780732369942198, "avg_duration_per_task_ms": 649367.0115606936, "avg_output_tokens_per_task": 52988.69942196532, "last_run_timestamp": "2026-09-19T08:07:24.262Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.7614156780821918, "accuracy_std_dev": 0.0228848139730987, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.024887965753424657, "avg_duration_per_task_ms": 458119.4315068493, "avg_output_tokens_per_task": 13217.239726027397, "last_run_timestamp": "2026-08-29T16:59:36.100Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7613333999999999, "accuracy_std_dev": 0.02166412395505529, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.6074076380000001, "avg_duration_per_task_ms": 125643.917, "avg_output_tokens_per_task": 8207.207, "last_run_timestamp": "2026-09-19T08:12:07.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.7612698571428571, "accuracy_std_dev": 0.019584924351059803, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.712182400952381, "avg_duration_per_task_ms": 75822.46, "avg_output_tokens_per_task": 5652.990476190476, "last_run_timestamp": "2026-09-19T08:15:48.243Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7610441927710843, "accuracy_std_dev": 0.04002571797891325, "run_count": 20, "total_questions": 996, "avg_cost_per_task_usd": 0.016136098393574293, "avg_duration_per_task_ms": 328656.0351405622, "avg_output_tokens_per_task": 8275.835341365462, "last_run_timestamp": "2026-08-29T16:33:19.761Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.7610256538461538, "accuracy_std_dev": 0.03457639874910488, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.11041888999999995, "avg_duration_per_task_ms": 290001.2453846154, "avg_output_tokens_per_task": 16905.416153846152, "last_run_timestamp": "2026-08-29T16:25:47.585Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.7610153534482759, "accuracy_std_dev": 0.044412661393182805, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.06370122844827586, "avg_duration_per_task_ms": 278513.2284482759, "avg_output_tokens_per_task": 9221.025862068966, "last_run_timestamp": "2026-08-29T17:06:46.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "WandB", "accuracy": 0.7609524285714285, "accuracy_std_dev": 0.018662793573288653, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.021895571428571427, "avg_duration_per_task_ms": 241457.5, "avg_output_tokens_per_task": 8715.474285714286, "last_run_timestamp": "2026-07-18T08:03:40.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.7609522857142857, "accuracy_std_dev": 0.017969598920035443, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08491382571428571, "avg_duration_per_task_ms": 105368.18285714286, "avg_output_tokens_per_task": 24364.49142857143, "last_run_timestamp": "2026-09-01T17:31:46.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Chutes", "accuracy": 0.7609489251824817, "accuracy_std_dev": 0.033505551928248105, "run_count": 22, "total_questions": 1096, "avg_cost_per_task_usd": 0.08300357664233575, "avg_duration_per_task_ms": 369852.16697080294, "avg_output_tokens_per_task": 15814.203467153284, "last_run_timestamp": "2026-08-15T14:15:18.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.7601780858778625, "accuracy_std_dev": 0.02811995722322186, "run_count": 21, "total_questions": 1048, "avg_cost_per_task_usd": 0.0422277356870229, "avg_duration_per_task_ms": 162724.19942748093, "avg_output_tokens_per_task": 7589.8158396946565, "last_run_timestamp": "2026-08-29T16:25:15.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.7600003333333333, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.2124113333333333, "avg_duration_per_task_ms": 390658.8466666667, "avg_output_tokens_per_task": 13665.773333333333, "last_run_timestamp": "2026-09-19T08:01:34.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.7600002499999999, "accuracy_std_dev": 0.018257509870940786, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.14997882, "avg_duration_per_task_ms": 322280.56, "avg_output_tokens_per_task": 20351.64, "last_run_timestamp": "2026-09-19T08:16:41.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Ionstream", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01747962, "avg_duration_per_task_ms": 589736.28, "avg_output_tokens_per_task": 13675.16, "last_run_timestamp": "2026-08-01T08:06:01.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.76, "accuracy_std_dev": 0.015634861336129593, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.038081735, "avg_duration_per_task_ms": 203007.56, "avg_output_tokens_per_task": 17943.12, "last_run_timestamp": "2026-09-19T08:18:11.626Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.76, "accuracy_std_dev": 0.024688728874297516, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.007911357142857142, "avg_duration_per_task_ms": 267145.0057142857, "avg_output_tokens_per_task": 15598.677142857143, "last_run_timestamp": "2026-09-19T08:07:37.278Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.76, "accuracy_std_dev": 0.01247214674785381, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021322305, "avg_duration_per_task_ms": 109483.545, "avg_output_tokens_per_task": 16713.61, "last_run_timestamp": "2026-09-12T08:11:31.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.05555669, "avg_duration_per_task_ms": 95287.39, "avg_output_tokens_per_task": 15474.24, "last_run_timestamp": "2026-09-19T08:01:53.394Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.76, "accuracy_std_dev": 0.016996895142937136, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.10744996000000001, "avg_duration_per_task_ms": 335547.935, "avg_output_tokens_per_task": 20056.545, "last_run_timestamp": "2026-09-19T08:08:42.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.15061989, "avg_duration_per_task_ms": 306259.77, "avg_output_tokens_per_task": 19605.64, "last_run_timestamp": "2026-09-19T08:01:34.094Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.76, "accuracy_std_dev": 0.033333, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11355728, "avg_duration_per_task_ms": 147828.6, "avg_output_tokens_per_task": 17720.73, "last_run_timestamp": "2026-09-01T17:30:58.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Venice", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19103682, "avg_duration_per_task_ms": 233230.24, "avg_output_tokens_per_task": 15236.66, "last_run_timestamp": "2026-08-04T03:12:00.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03723274, "avg_duration_per_task_ms": 352117.32, "avg_output_tokens_per_task": 12561.96, "last_run_timestamp": "2026-09-19T08:12:48.363Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.76, "accuracy_std_dev": 0.022705750619611758, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.077266932, "avg_duration_per_task_ms": 260289.312, "avg_output_tokens_per_task": 11860.184, "last_run_timestamp": "2026-09-19T08:01:53.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.35068044, "avg_duration_per_task_ms": 248200.34, "avg_output_tokens_per_task": 11571.02, "last_run_timestamp": "2026-09-01T12:41:31.788Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.76, "accuracy_std_dev": 0.03265986323710907, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.05974882666666666, "avg_duration_per_task_ms": 170255.57333333333, "avg_output_tokens_per_task": 4552.706666666667, "last_run_timestamp": "2026-09-19T08:16:20.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.76, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04466577, "avg_duration_per_task_ms": 116479.54, "avg_output_tokens_per_task": 4371.72, "last_run_timestamp": "2026-09-19T08:15:43.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05904302, "avg_duration_per_task_ms": 70821.54, "avg_output_tokens_per_task": 4624.02, "last_run_timestamp": "2026-09-19T08:01:53.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.76, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014824000000000002, "avg_duration_per_task_ms": 335920.77, "avg_output_tokens_per_task": 4448.51, "last_run_timestamp": "2026-09-12T08:11:26.740Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.7593333999999999, "accuracy_std_dev": 0.014360396158880871, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.1251978, "avg_duration_per_task_ms": 443535.78, "avg_output_tokens_per_task": 22633.804, "last_run_timestamp": "2026-09-19T08:17:30.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.7590475714285715, "accuracy_std_dev": 0.014443569136046514, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.008607337142857142, "avg_duration_per_task_ms": 374540.4914285714, "avg_output_tokens_per_task": 13803.397142857142, "last_run_timestamp": "2026-09-19T08:14:25.256Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Phala", "accuracy": 0.759040841981132, "accuracy_std_dev": 0.036093876622690174, "run_count": 17, "total_questions": 848, "avg_cost_per_task_usd": 0.08171993514150944, "avg_duration_per_task_ms": 269188.51061320753, "avg_output_tokens_per_task": 4527.392688679245, "last_run_timestamp": "2026-07-11T15:34:16.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7588095, "accuracy_std_dev": 0.034551746496585184, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.052020888571428586, "avg_duration_per_task_ms": 138271.82571428572, "avg_output_tokens_per_task": 5773.107857142857, "last_run_timestamp": "2026-08-29T16:32:40.387Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7586668, "accuracy_std_dev": 0.01654634971708262, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.27091952399999997, "avg_duration_per_task_ms": 154225.472, "avg_output_tokens_per_task": 23085.08, "last_run_timestamp": "2026-09-01T17:32:40.487Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7585878072519083, "accuracy_std_dev": 0.03764932431268607, "run_count": 21, "total_questions": 1048, "avg_cost_per_task_usd": 0.0847812709923664, "avg_duration_per_task_ms": 181956.90935114503, "avg_output_tokens_per_task": 7087.411259541985, "last_run_timestamp": "2026-08-29T16:48:46.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.015184156584660873, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.163021, "avg_duration_per_task_ms": 227043.59, "avg_output_tokens_per_task": 11333.075, "last_run_timestamp": "2026-08-29T16:55:33.128Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7583331666666667, "accuracy_std_dev": 0.013844473427047428, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09591996333333334, "avg_duration_per_task_ms": 214673.9, "avg_output_tokens_per_task": 9689.54, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.758, "accuracy_std_dev": 0.03148898925815182, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.19792223, "avg_duration_per_task_ms": 121774.793, "avg_output_tokens_per_task": 7801.686, "last_run_timestamp": "2026-09-19T08:15:55.793Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.758, "accuracy_std_dev": 0.03709447398198281, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.079319272, "avg_duration_per_task_ms": 620750.224, "avg_output_tokens_per_task": 10895.356, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.7578826216216217, "accuracy_std_dev": 0.006612862617656559, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.020859918918918918, "avg_duration_per_task_ms": 101574.95945945945, "avg_output_tokens_per_task": 13789.418918918918, "last_run_timestamp": "2026-09-19T08:10:29.894Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.757778, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.006585946666666667, "avg_duration_per_task_ms": 132381.80666666667, "avg_output_tokens_per_task": 4303.826666666667, "last_run_timestamp": "2026-09-19T08:02:29.956Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.7577778333333334, "accuracy_std_dev": 0.037646776132079215, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08977579333333333, "avg_duration_per_task_ms": 61527.54, "avg_output_tokens_per_task": 12239.113333333333, "last_run_timestamp": "2026-08-29T16:25:14.949Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.017497908167041637, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.006062186666666667, "avg_duration_per_task_ms": 148585.37333333332, "avg_output_tokens_per_task": 4189.8, "last_run_timestamp": "2026-09-19T08:06:51.600Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.7574602857142857, "accuracy_std_dev": 0.030548852532643923, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.010414794285714285, "avg_duration_per_task_ms": 137613.95523809525, "avg_output_tokens_per_task": 9270.931428571428, "last_run_timestamp": "2026-08-29T16:48:14.107Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.7571428571428571, "accuracy_std_dev": 0.029783673564212878, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.19815623142857142, "avg_duration_per_task_ms": 133956.76285714286, "avg_output_tokens_per_task": 7857.111428571428, "last_run_timestamp": "2026-09-19T08:06:50.238Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7568027346938776, "accuracy_std_dev": 0.038809665680729605, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.3020268469387756, "avg_duration_per_task_ms": 145185.1581632653, "avg_output_tokens_per_task": 20600.117346938776, "last_run_timestamp": "2026-09-01T17:31:47.860Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01828589, "avg_duration_per_task_ms": 240722.54, "avg_output_tokens_per_task": 15147.8, "last_run_timestamp": "2026-09-19T08:02:39.158Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7566667499999999, "accuracy_std_dev": 0.013743584364622665, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.085799, "avg_duration_per_task_ms": 225293.88, "avg_output_tokens_per_task": 12078.17, "last_run_timestamp": "2026-09-19T08:13:58.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.7566667499999999, "accuracy_std_dev": 0.013743948165920146, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07678554, "avg_duration_per_task_ms": 131792.745, "avg_output_tokens_per_task": 4961.875, "last_run_timestamp": "2026-09-19T08:14:27.258Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7566667, "accuracy_std_dev": 0.024630617804066543, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.02296196, "avg_duration_per_task_ms": 168920.314, "avg_output_tokens_per_task": 18473.208, "last_run_timestamp": "2026-09-19T08:15:24.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.7566666666666667, "accuracy_std_dev": 0.009999888893827235, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.035569739999999996, "avg_duration_per_task_ms": 285830.0233333333, "avg_output_tokens_per_task": 13698.04, "last_run_timestamp": "2026-09-19T08:01:53.309Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7566665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10920926999999998, "avg_duration_per_task_ms": 101251.93, "avg_output_tokens_per_task": 25100.99, "last_run_timestamp": "2026-09-01T17:23:16.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.7566665, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.42697405000000005, "avg_duration_per_task_ms": 522179.69, "avg_output_tokens_per_task": 37590.96, "last_run_timestamp": "2026-09-01T12:41:24.474Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.7566665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29550982, "avg_duration_per_task_ms": 152697.48, "avg_output_tokens_per_task": 10806.55, "last_run_timestamp": "2026-09-01T17:30:15.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.7564705294117647, "accuracy_std_dev": 0.03521125545336429, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.008393454117647059, "avg_duration_per_task_ms": 146412.67529411765, "avg_output_tokens_per_task": 9095.61294117647, "last_run_timestamp": "2026-08-29T16:54:19.718Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7564102600732601, "accuracy_std_dev": 0.0234723881283154, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.02063667582417582, "avg_duration_per_task_ms": 550307.2802197802, "avg_output_tokens_per_task": 17523.981684981685, "last_run_timestamp": "2026-09-19T08:08:46.172Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.7561938175675675, "accuracy_std_dev": 0.02331332973798942, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.20687053378378378, "avg_duration_per_task_ms": 486380.6959459459, "avg_output_tokens_per_task": 20517.347972972973, "last_run_timestamp": "2026-09-19T08:13:57.893Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7561904285714286, "accuracy_std_dev": 0.02465178812324715, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3425374, "avg_duration_per_task_ms": 222364.59142857144, "avg_output_tokens_per_task": 11866.32, "last_run_timestamp": "2026-09-19T08:08:02.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7559998, "accuracy_std_dev": 0.029393940419072766, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.550821472, "avg_duration_per_task_ms": 179743.46, "avg_output_tokens_per_task": 10831.904, "last_run_timestamp": "2026-09-19T08:02:29.757Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.7559259444444444, "accuracy_std_dev": 0.03443646283531493, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.05180537888888888, "avg_duration_per_task_ms": 157203.02777777778, "avg_output_tokens_per_task": 6346.41, "last_run_timestamp": "2026-08-29T17:22:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.7556390701754386, "accuracy_std_dev": 0.0748655823238389, "run_count": 16, "total_questions": 798, "avg_cost_per_task_usd": 0.1314042468671679, "avg_duration_per_task_ms": 282979.45112781954, "avg_output_tokens_per_task": 13309.800751879699, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7555927852348994, "accuracy_std_dev": 0.017391785347782254, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.04238428187919464, "avg_duration_per_task_ms": 512275.43624161073, "avg_output_tokens_per_task": 21918.96979865772, "last_run_timestamp": "2026-09-19T16:00:24.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7554257228714524, "accuracy_std_dev": 0.04133335435850861, "run_count": 24, "total_questions": 1198, "avg_cost_per_task_usd": 0.05269022621035059, "avg_duration_per_task_ms": 158796.83138564273, "avg_output_tokens_per_task": 6105.924040066778, "last_run_timestamp": "2026-08-29T16:45:34.327Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.7553846923076923, "accuracy_std_dev": 0.03916955640129917, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.19397882, "avg_duration_per_task_ms": 104918.87538461538, "avg_output_tokens_per_task": 7649.404615384616, "last_run_timestamp": "2026-09-19T08:12:02.264Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.7552381428571429, "accuracy_std_dev": 0.025625984192548305, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.5360551200000001, "avg_duration_per_task_ms": 191556.04571428572, "avg_output_tokens_per_task": 10244.057142857142, "last_run_timestamp": "2026-09-19T08:01:55.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.7552380000000001, "accuracy_std_dev": 0.043456084643826665, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.19647111142857143, "avg_duration_per_task_ms": 107624.06285714285, "avg_output_tokens_per_task": 7682.417142857143, "last_run_timestamp": "2026-09-19T08:14:23.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Ambient", "accuracy": 0.755, "accuracy_std_dev": 0.021794494717703387, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.049016185, "avg_duration_per_task_ms": 156512.84, "avg_output_tokens_per_task": 3394.74, "last_run_timestamp": "2026-04-25T11:20:08.052Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.7547709770992367, "accuracy_std_dev": 0.037708741308906626, "run_count": 21, "total_questions": 1048, "avg_cost_per_task_usd": 0.06855190171755726, "avg_duration_per_task_ms": 232773.2967557252, "avg_output_tokens_per_task": 7463.7099236641225, "last_run_timestamp": "2026-08-29T16:33:36.999Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7543859845201238, "accuracy_std_dev": 0.03746648424278414, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.06285631269349846, "avg_duration_per_task_ms": 163970.43962848297, "avg_output_tokens_per_task": 7998.698142414861, "last_run_timestamp": "2026-08-29T16:44:10.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ionstream", "accuracy": 0.7542953608247424, "accuracy_std_dev": 0.02652500000000002, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.021299783505154636, "avg_duration_per_task_ms": 633495.5257731959, "avg_output_tokens_per_task": 26267.113402061856, "last_run_timestamp": "2026-08-08T08:30:53.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.7542857142857143, "accuracy_std_dev": 0.02676848927758313, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.5495588485714286, "avg_duration_per_task_ms": 192291.6257142857, "avg_output_tokens_per_task": 10585.511428571428, "last_run_timestamp": "2026-09-19T08:12:49.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7542857142857143, "accuracy_std_dev": 0.023752365678957953, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.023086720000000005, "avg_duration_per_task_ms": 119604, "avg_output_tokens_per_task": 15643.062857142857, "last_run_timestamp": "2026-09-20T10:30:07.540Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7542856785714285, "accuracy_std_dev": 0.04289943601032333, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.050694632142857146, "avg_duration_per_task_ms": 138996.95214285713, "avg_output_tokens_per_task": 5646.897857142857, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.7538889166666667, "accuracy_std_dev": 0.03686387162127335, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.06654354, "avg_duration_per_task_ms": 244757.80833333332, "avg_output_tokens_per_task": 10635.376666666667, "last_run_timestamp": "2026-08-29T16:33:11.313Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7535714285714286, "accuracy_std_dev": 0.04815591057583436, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.04571141285714286, "avg_duration_per_task_ms": 126701.03428571428, "avg_output_tokens_per_task": 5683.6, "last_run_timestamp": "2026-08-29T16:58:32.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.7533335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0840499, "avg_duration_per_task_ms": 277299.54, "avg_output_tokens_per_task": 19530.63, "last_run_timestamp": "2026-09-19T08:08:43.307Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5044717400000001, "avg_duration_per_task_ms": 153665.59, "avg_output_tokens_per_task": 8278.2, "last_run_timestamp": "2026-09-01T12:41:31.807Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01056969, "avg_duration_per_task_ms": 192129.64, "avg_output_tokens_per_task": 4500.45, "last_run_timestamp": "2026-09-19T08:15:09.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7533333999999999, "accuracy_std_dev": 0.028284365530094557, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.194302048, "avg_duration_per_task_ms": 129735.288, "avg_output_tokens_per_task": 7732.768, "last_run_timestamp": "2026-09-19T08:02:29.788Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.02372709031653247, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.027049573333333334, "avg_duration_per_task_ms": 129531.18666666666, "avg_output_tokens_per_task": 16746.393333333333, "last_run_timestamp": "2026-09-19T08:12:02.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Morph", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021082753333333332, "avg_duration_per_task_ms": 915986.2133333334, "avg_output_tokens_per_task": 13268.026666666667, "last_run_timestamp": "2026-08-15T08:15:52.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Together", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.044721359549995794, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.08804555777777777, "avg_duration_per_task_ms": 73317.22, "avg_output_tokens_per_task": 4526.791111111111, "last_run_timestamp": "2026-06-20T18:29:02.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.019626210745383905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.15976216666666668, "avg_duration_per_task_ms": 259508.71333333335, "avg_output_tokens_per_task": 8887.92, "last_run_timestamp": "2026-09-19T08:11:59.701Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08241664, "avg_duration_per_task_ms": 157302.79333333333, "avg_output_tokens_per_task": 4631.613333333334, "last_run_timestamp": "2026-09-19T08:08:45.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7533333214285716, "accuracy_std_dev": 0.043861247675444484, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.036317195, "avg_duration_per_task_ms": 122335.48285714285, "avg_output_tokens_per_task": 5681.47, "last_run_timestamp": "2026-08-29T16:11:46.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7533333, "accuracy_std_dev": 0.020870052975735364, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.02329514, "avg_duration_per_task_ms": 307869.234, "avg_output_tokens_per_task": 17664.796, "last_run_timestamp": "2026-09-19T08:06:51.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.7533332857142857, "accuracy_std_dev": 0.03804763096829185, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.22995042095238097, "avg_duration_per_task_ms": 102003.29238095238, "avg_output_tokens_per_task": 7520.403809523809, "last_run_timestamp": "2026-09-19T08:15:48.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.7533332500000001, "accuracy_std_dev": 0.016996797086142445, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.036730425000000004, "avg_duration_per_task_ms": 213947.705, "avg_output_tokens_per_task": 18813.24, "last_run_timestamp": "2026-09-19T08:13:20.914Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.7533332, "accuracy_std_dev": 0.03675737870088128, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.42735367599999996, "avg_duration_per_task_ms": 94631.4, "avg_output_tokens_per_task": 5528.516, "last_run_timestamp": "2026-09-19T08:15:14.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.753333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.36418883999999996, "avg_duration_per_task_ms": 138377.24, "avg_output_tokens_per_task": 12769.81, "last_run_timestamp": "2026-09-01T17:31:47.883Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02738974, "avg_duration_per_task_ms": 82133.22, "avg_output_tokens_per_task": 9075.3, "last_run_timestamp": "2026-09-19T16:00:30.815Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.41880896, "avg_duration_per_task_ms": 447303.41, "avg_output_tokens_per_task": 36551.01, "last_run_timestamp": "2026-09-01T12:59:50.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.6857741799999999, "avg_duration_per_task_ms": 105619.36, "avg_output_tokens_per_task": 7597.08, "last_run_timestamp": "2026-09-01T13:02:16.941Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0196027, "avg_duration_per_task_ms": 76385.02, "avg_output_tokens_per_task": 9950, "last_run_timestamp": "2026-07-18T08:04:02.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "CoreWeave", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10139794, "avg_duration_per_task_ms": 119629.34, "avg_output_tokens_per_task": 9506.02, "last_run_timestamp": "2026-08-01T17:46:04.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2831028, "avg_duration_per_task_ms": 544501.1, "avg_output_tokens_per_task": 8096.24, "last_run_timestamp": "2026-08-23T18:00:01.881Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ionstream", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09723594, "avg_duration_per_task_ms": 178772.1, "avg_output_tokens_per_task": 8310.58, "last_run_timestamp": "2026-08-01T08:05:58.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Friendli", "accuracy": 0.7529824736842106, "accuracy_std_dev": 0.1257765564827976, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.04461744315789474, "avg_duration_per_task_ms": 41775.2, "avg_output_tokens_per_task": 3596.8168421052633, "last_run_timestamp": "2026-07-01T02:07:35.472Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.7527272727272727, "accuracy_std_dev": 0.016441999795926792, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.05201301999999999, "avg_duration_per_task_ms": 275857.43454545457, "avg_output_tokens_per_task": 30292.38909090909, "last_run_timestamp": "2026-09-19T16:00:28.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.7521489914040115, "accuracy_std_dev": 0.01514003431640894, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.013516624641833811, "avg_duration_per_task_ms": 212155.76217765044, "avg_output_tokens_per_task": 16555.63323782235, "last_run_timestamp": "2026-09-19T08:18:14.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.7521211818181818, "accuracy_std_dev": 0.03491049713147814, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.18250516363636365, "avg_duration_per_task_ms": 550090.9563636364, "avg_output_tokens_per_task": 26501.607272727273, "last_run_timestamp": "2026-09-19T08:14:01.746Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Ambient", "accuracy": 0.752, "accuracy_std_dev": 0.02400000000000002, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.031335448, "avg_duration_per_task_ms": 314672.816, "avg_output_tokens_per_task": 20249.848, "last_run_timestamp": "2026-06-13T19:57:55.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.752, "accuracy_std_dev": 0.025785560129653974, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.12171823600000001, "avg_duration_per_task_ms": 271294.616, "avg_output_tokens_per_task": 12082.784, "last_run_timestamp": "2026-09-19T08:02:39.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "WandB", "accuracy": 0.7519999333333334, "accuracy_std_dev": 0.042457799179054144, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.07472457066666666, "avg_duration_per_task_ms": 97051.48533333333, "avg_output_tokens_per_task": 11508.92, "last_run_timestamp": "2026-07-18T08:03:41.319Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.7517460476190476, "accuracy_std_dev": 0.03886781145721899, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.0710823885714286, "avg_duration_per_task_ms": 177196.37523809524, "avg_output_tokens_per_task": 5993.54, "last_run_timestamp": "2026-08-29T16:12:10.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.7517008979591836, "accuracy_std_dev": 0.00847199999999998, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.008818877551020407, "avg_duration_per_task_ms": 183019.20408163266, "avg_output_tokens_per_task": 3602.8367346938776, "last_run_timestamp": "2026-09-19T08:15:22.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.7516667499999999, "accuracy_std_dev": 0.031046636068783707, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08865978499999999, "avg_duration_per_task_ms": 293099.965, "avg_output_tokens_per_task": 12000.945, "last_run_timestamp": "2026-09-19T08:02:29.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.7516665, "accuracy_std_dev": 0.03378863221188454, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.025036024999999996, "avg_duration_per_task_ms": 332301.835, "avg_output_tokens_per_task": 13337.575, "last_run_timestamp": "2026-08-29T17:19:05.764Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.7513043478260869, "accuracy_std_dev": 0.033096829546845286, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.008512693043478257, "avg_duration_per_task_ms": 141943.57217391304, "avg_output_tokens_per_task": 8289.525217391305, "last_run_timestamp": "2026-08-29T16:33:30.698Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7510040481927711, "accuracy_std_dev": 0.022883894021997232, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.007396576305220883, "avg_duration_per_task_ms": 255494.20682730924, "avg_output_tokens_per_task": 11511.991967871487, "last_run_timestamp": "2026-08-29T17:00:01.341Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.75100005, "accuracy_std_dev": 0.022634698291505915, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.370373307, "avg_duration_per_task_ms": 170883.772, "avg_output_tokens_per_task": 11914.488, "last_run_timestamp": "2026-09-19T08:15:51.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7508740961538461, "accuracy_std_dev": 0.08177386372104274, "run_count": 23, "total_questions": 1144, "avg_cost_per_task_usd": 0.05178484615384617, "avg_duration_per_task_ms": 150234.52097902098, "avg_output_tokens_per_task": 20045.546328671328, "last_run_timestamp": "2026-08-29T16:10:33.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.7506666, "accuracy_std_dev": 0.029242276854581632, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.336778826, "avg_duration_per_task_ms": 211305.436, "avg_output_tokens_per_task": 11826.724, "last_run_timestamp": "2026-09-19T08:14:36.128Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AkashML", "accuracy": 0.7505555833333333, "accuracy_std_dev": 0.037757344555857254, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.016059435, "avg_duration_per_task_ms": 293963.405, "avg_output_tokens_per_task": 5996.211666666667, "last_run_timestamp": "2026-07-18T08:03:38.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.7505555, "accuracy_std_dev": 0.03274012621004996, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02124632666666667, "avg_duration_per_task_ms": 354797.86666666664, "avg_output_tokens_per_task": 18494.39, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.7504762857142857, "accuracy_std_dev": 0.034523051552401846, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.022417262857142857, "avg_duration_per_task_ms": 166619.53714285715, "avg_output_tokens_per_task": 14078.568571428572, "last_run_timestamp": "2026-09-19T08:12:29.747Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7504761428571428, "accuracy_std_dev": 0.029354239613134367, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.013216268571428573, "avg_duration_per_task_ms": 156715.7, "avg_output_tokens_per_task": 16682.777142857143, "last_run_timestamp": "2026-09-19T08:07:36.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.7504761428571428, "accuracy_std_dev": 0.023328467908964635, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02283790857142857, "avg_duration_per_task_ms": 124586.03428571428, "avg_output_tokens_per_task": 7423.028571428571, "last_run_timestamp": "2026-09-19T16:00:32.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2253275, "avg_duration_per_task_ms": 98359.26, "avg_output_tokens_per_task": 5434.86, "last_run_timestamp": "2026-09-01T13:03:25.552Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Nebius", "accuracy": 0.75, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11403106000000002, "avg_duration_per_task_ms": 126507.38, "avg_output_tokens_per_task": 4905.84, "last_run_timestamp": "2026-06-20T10:57:18.552Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.7497436153846154, "accuracy_std_dev": 0.0478327757706888, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.08224532461538461, "avg_duration_per_task_ms": 182548.32615384617, "avg_output_tokens_per_task": 7228.066153846154, "last_run_timestamp": "2026-09-19T08:13:49.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.7495238571428572, "accuracy_std_dev": 0.028918598713475383, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.48306242571428565, "avg_duration_per_task_ms": 195308.45714285714, "avg_output_tokens_per_task": 9967.34857142857, "last_run_timestamp": "2026-09-19T08:17:29.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.7495237857142857, "accuracy_std_dev": 0.031138569678084756, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.5694101414285715, "avg_duration_per_task_ms": 203607.95571428572, "avg_output_tokens_per_task": 9913.828571428572, "last_run_timestamp": "2026-09-19T08:14:26.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.74916675, "accuracy_std_dev": 0.018837731788554076, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021088265000000002, "avg_duration_per_task_ms": 122926.245, "avg_output_tokens_per_task": 16099.0975, "last_run_timestamp": "2026-09-19T08:12:00.893Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7491665000000001, "accuracy_std_dev": 0.04238543054222759, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.019819255, "avg_duration_per_task_ms": 82254.675, "avg_output_tokens_per_task": 4110.175, "last_run_timestamp": "2026-09-19T08:15:43.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.7490876715328467, "accuracy_std_dev": 0.019546598328901252, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.0068139452554744516, "avg_duration_per_task_ms": 184530.0200729927, "avg_output_tokens_per_task": 12199.35583941606, "last_run_timestamp": "2026-09-19T08:10:30.975Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.749, "accuracy_std_dev": 0.027830806272905576, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.033878175999999996, "avg_duration_per_task_ms": 238766.5, "avg_output_tokens_per_task": 5451.338, "last_run_timestamp": "2026-08-29T16:10:33.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.748889, "accuracy_std_dev": 0.03281068781154497, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03277637333333334, "avg_duration_per_task_ms": 282626.5333333333, "avg_output_tokens_per_task": 18948.02, "last_run_timestamp": "2026-09-19T08:01:36.428Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "WandB", "accuracy": 0.7488585479452055, "accuracy_std_dev": 0.02333557629219577, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.13015020547945205, "avg_duration_per_task_ms": 94301.3698630137, "avg_output_tokens_per_task": 9432.931506849314, "last_run_timestamp": "2026-07-18T08:04:45.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7485715714285714, "accuracy_std_dev": 0.030386743739697415, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.27195099142857143, "avg_duration_per_task_ms": 529335.9085714286, "avg_output_tokens_per_task": 9553.417142857143, "last_run_timestamp": "2026-08-29T16:51:55.567Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.7483332500000001, "accuracy_std_dev": 0.01723988431479461, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.010718860000000002, "avg_duration_per_task_ms": 133490.525, "avg_output_tokens_per_task": 4191.97, "last_run_timestamp": "2026-09-19T08:01:53.307Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7483330000000001, "accuracy_std_dev": 0.02598076211353318, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.024764005, "avg_duration_per_task_ms": 370315.33, "avg_output_tokens_per_task": 4324.135, "last_run_timestamp": "2026-09-19T08:15:51.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7480001, "accuracy_std_dev": 0.02612788548830541, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.35369032, "avg_duration_per_task_ms": 233260.038, "avg_output_tokens_per_task": 10816.624, "last_run_timestamp": "2026-09-19T08:19:00.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7477778333333334, "accuracy_std_dev": 0.02759945204417816, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10948868333333332, "avg_duration_per_task_ms": 328943.99666666664, "avg_output_tokens_per_task": 18817.083333333332, "last_run_timestamp": "2026-09-19T08:02:29.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7476191428571428, "accuracy_std_dev": 0.035710930637265534, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.5503071828571428, "avg_duration_per_task_ms": 184401.16, "avg_output_tokens_per_task": 10642.84857142857, "last_run_timestamp": "2026-09-19T08:12:56.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.7476191428571428, "accuracy_std_dev": 0.02090910249101076, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3218781142857143, "avg_duration_per_task_ms": 199636.20857142858, "avg_output_tokens_per_task": 11432.102857142858, "last_run_timestamp": "2026-09-19T08:12:55.142Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Fireworks", "accuracy": 0.7474746666666666, "accuracy_std_dev": 0.013351080637442798, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.14427683333333333, "avg_duration_per_task_ms": 274328.10101010103, "avg_output_tokens_per_task": 7734.318181818182, "last_run_timestamp": "2026-07-18T08:04:29.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Morph", "accuracy": 0.7473334, "accuracy_std_dev": 0.02318052631067726, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.019461324000000002, "avg_duration_per_task_ms": 260422.68, "avg_output_tokens_per_task": 9034.204, "last_run_timestamp": "2026-08-08T08:29:47.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.7472726363636364, "accuracy_std_dev": 0.0339286130072398, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.09873066363636362, "avg_duration_per_task_ms": 157555.76, "avg_output_tokens_per_task": 8576.863636363636, "last_run_timestamp": "2026-09-19T08:15:47.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7466670000000001, "accuracy_std_dev": 0.03999999999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.44994164999999997, "avg_duration_per_task_ms": 128787.73, "avg_output_tokens_per_task": 20836.09, "last_run_timestamp": "2026-09-01T17:32:23.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.746667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.3847332, "avg_duration_per_task_ms": 155324.72, "avg_output_tokens_per_task": 13079.53, "last_run_timestamp": "2026-09-01T17:31:47.956Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.746667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.05560944, "avg_duration_per_task_ms": 195101.15, "avg_output_tokens_per_task": 14684.06, "last_run_timestamp": "2026-09-19T08:08:01.712Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "StepFun", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02698528, "avg_duration_per_task_ms": 147907.84, "avg_output_tokens_per_task": 11065.38, "last_run_timestamp": "2026-07-18T08:04:04.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.010886757164351354, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0582748, "avg_duration_per_task_ms": 126998.29333333333, "avg_output_tokens_per_task": 4129.593333333333, "last_run_timestamp": "2026-09-19T08:17:43.474Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.7466666666666667, "accuracy_std_dev": 0.028803034820811656, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.009823306666666667, "avg_duration_per_task_ms": 205830.71333333335, "avg_output_tokens_per_task": 4522.053333333333, "last_run_timestamp": "2026-09-19T08:08:18.540Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.7466665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.31129542, "avg_duration_per_task_ms": 181218.77, "avg_output_tokens_per_task": 16905.63, "last_run_timestamp": "2026-09-19T08:02:29.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.42183384, "avg_duration_per_task_ms": 170456.6, "avg_output_tokens_per_task": 28298.04, "last_run_timestamp": "2026-09-01T17:30:52.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.011547101609639251, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04422623666666667, "avg_duration_per_task_ms": 266653.04, "avg_output_tokens_per_task": 13360.296666666667, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.7466665, "accuracy_std_dev": 0.011546716708657956, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.1707169, "avg_duration_per_task_ms": 302727.215, "avg_output_tokens_per_task": 10539.265, "last_run_timestamp": "2026-09-19T08:02:39.146Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.7466665, "accuracy_std_dev": 0.019999888889274717, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06283585333333332, "avg_duration_per_task_ms": 361191.69, "avg_output_tokens_per_task": 8869.28, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.7466665, "accuracy_std_dev": 0.03126950946609175, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.007685050000000001, "avg_duration_per_task_ms": 185392.305, "avg_output_tokens_per_task": 4351.84, "last_run_timestamp": "2026-09-19T08:09:43.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03271783, "avg_duration_per_task_ms": 346196.84, "avg_output_tokens_per_task": 4294.49, "last_run_timestamp": "2026-08-29T08:54:52.477Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7461684252873563, "accuracy_std_dev": 0.022865583595063955, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.14753891091954022, "avg_duration_per_task_ms": 476020.4597701149, "avg_output_tokens_per_task": 10344.031609195403, "last_run_timestamp": "2026-08-22T08:22:21.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.745819397993311, "accuracy_std_dev": 0.027782945492193997, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.03796495317725753, "avg_duration_per_task_ms": 162257.28929765886, "avg_output_tokens_per_task": 7729.632107023412, "last_run_timestamp": "2026-09-19T08:15:37.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.7458193712374581, "accuracy_std_dev": 0.020725900684590307, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.03691470903010033, "avg_duration_per_task_ms": 270360.6237458194, "avg_output_tokens_per_task": 23763.14381270903, "last_run_timestamp": "2026-09-19T16:00:32.493Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.7456140464396285, "accuracy_std_dev": 0.047057992260714526, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.011993541795665636, "avg_duration_per_task_ms": 632093.0835913313, "avg_output_tokens_per_task": 8861.049535603715, "last_run_timestamp": "2026-08-29T16:44:33.448Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.7454545454545455, "accuracy_std_dev": 0.045199256436691, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.10578343272727275, "avg_duration_per_task_ms": 664793.96, "avg_output_tokens_per_task": 24353.294545454544, "last_run_timestamp": "2026-09-19T16:00:22.544Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7453334, "accuracy_std_dev": 0.028094905827213558, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.348214824, "avg_duration_per_task_ms": 202851.62, "avg_output_tokens_per_task": 11895.116, "last_run_timestamp": "2026-09-19T08:06:16.236Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.7452631578947368, "accuracy_std_dev": 0.03464626575042041, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.37969141473684204, "avg_duration_per_task_ms": 207895.33052631578, "avg_output_tokens_per_task": 11954.22, "last_run_timestamp": "2026-09-19T08:16:29.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7451159230769231, "accuracy_std_dev": 0.042379107322140314, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.11416542307692307, "avg_duration_per_task_ms": 465658.6978021978, "avg_output_tokens_per_task": 24052.309523809523, "last_run_timestamp": "2026-09-19T08:13:25.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.7450981176470588, "accuracy_std_dev": 0.03403410890628669, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.08884740352941176, "avg_duration_per_task_ms": 247677.12588235294, "avg_output_tokens_per_task": 7114.02, "last_run_timestamp": "2026-08-29T16:43:42.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Io Net", "accuracy": 0.745, "accuracy_std_dev": 0.051071844820148544, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.05017629, "avg_duration_per_task_ms": 100886.12833333333, "avg_output_tokens_per_task": 4164.488333333334, "last_run_timestamp": "2026-06-20T12:09:06.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.745, "accuracy_std_dev": 0.009860245661239886, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.45607220499999995, "avg_duration_per_task_ms": 481202.88, "avg_output_tokens_per_task": 36078.37, "last_run_timestamp": "2026-09-01T12:59:55.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.7448276551724138, "accuracy_std_dev": 0.0035438860340341496, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.022715406896551726, "avg_duration_per_task_ms": 289516.0620689655, "avg_output_tokens_per_task": 10103.63448275862, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7444446666666666, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12615662666666666, "avg_duration_per_task_ms": 491171.2133333333, "avg_output_tokens_per_task": 19592.646666666667, "last_run_timestamp": "2026-09-19T08:16:40.730Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.7444446666666666, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.022935546666666664, "avg_duration_per_task_ms": 247026.00666666665, "avg_output_tokens_per_task": 4198.3133333333335, "last_run_timestamp": "2026-09-19T08:15:43.987Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.03823267621928775, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08758441666666665, "avg_duration_per_task_ms": 265114.18333333335, "avg_output_tokens_per_task": 12207.423333333334, "last_run_timestamp": "2026-09-19T08:12:01.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7442919863013698, "accuracy_std_dev": 0.006500008529908962, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.012089356164383563, "avg_duration_per_task_ms": 63776.74657534246, "avg_output_tokens_per_task": 3609.541095890411, "last_run_timestamp": "2026-09-19T08:07:54.122Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.744221408759124, "accuracy_std_dev": 0.0694836949766747, "run_count": 22, "total_questions": 1096, "avg_cost_per_task_usd": 0.010320726277372263, "avg_duration_per_task_ms": 190073.67335766423, "avg_output_tokens_per_task": 8391.975364963504, "last_run_timestamp": "2026-08-29T16:29:07.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.7441045812619503, "accuracy_std_dev": 0.03256003617627679, "run_count": 21, "total_questions": 1046, "avg_cost_per_task_usd": 0.061009545889101346, "avg_duration_per_task_ms": 226154.17782026768, "avg_output_tokens_per_task": 8148.751434034417, "last_run_timestamp": "2026-08-29T16:33:12.146Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7439998000000001, "accuracy_std_dev": 0.025508095447524134, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.010447856000000002, "avg_duration_per_task_ms": 427694.796, "avg_output_tokens_per_task": 12591.384, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.7438095714285714, "accuracy_std_dev": 0.03184425677241906, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.51650314, "avg_duration_per_task_ms": 230071.5342857143, "avg_output_tokens_per_task": 10020.925714285715, "last_run_timestamp": "2026-09-19T08:10:31.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.743589829431438, "accuracy_std_dev": 0.021188684958014764, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.028942936454849497, "avg_duration_per_task_ms": 685874.1371237459, "avg_output_tokens_per_task": 21249.709030100334, "last_run_timestamp": "2026-09-19T16:00:32.531Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.7433335, "accuracy_std_dev": 0.012018411802314, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.14946610500000002, "avg_duration_per_task_ms": 148382.875, "avg_output_tokens_per_task": 20254.93, "last_run_timestamp": "2026-09-19T08:01:53.309Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7433335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0032184600000000002, "avg_duration_per_task_ms": 64941.97, "avg_output_tokens_per_task": 11846.7, "last_run_timestamp": "2026-08-19T18:00:02.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.7433335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.4675528, "avg_duration_per_task_ms": 415045.58, "avg_output_tokens_per_task": 10466.34, "last_run_timestamp": "2026-09-19T08:02:29.875Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7433332500000001, "accuracy_std_dev": 0.028867484592314266, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.00987351, "avg_duration_per_task_ms": 304668.76, "avg_output_tokens_per_task": 4196.865, "last_run_timestamp": "2026-09-19T08:16:20.350Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.7433330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01160009, "avg_duration_per_task_ms": 224850.4, "avg_output_tokens_per_task": 4363.45, "last_run_timestamp": "2026-09-19T08:12:02.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7432844835164835, "accuracy_std_dev": 0.04305427570650295, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.16399129853479852, "avg_duration_per_task_ms": 413747.94505494507, "avg_output_tokens_per_task": 24261.919413919415, "last_run_timestamp": "2026-09-12T16:00:20.988Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.7427776666666667, "accuracy_std_dev": 0.036434422742175517, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3671939166666667, "avg_duration_per_task_ms": 137596.83, "avg_output_tokens_per_task": 5212.336666666667, "last_run_timestamp": "2026-09-01T13:00:30.911Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.7426668, "accuracy_std_dev": 0.03690245640821219, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011645404, "avg_duration_per_task_ms": 585840.916, "avg_output_tokens_per_task": 18794.172, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7426666000000001, "accuracy_std_dev": 0.03641722913457311, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.038778624, "avg_duration_per_task_ms": 203795.688, "avg_output_tokens_per_task": 13449.364, "last_run_timestamp": "2026-09-19T08:01:36.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7422839058641976, "accuracy_std_dev": 0.037889381261823826, "run_count": 26, "total_questions": 1296, "avg_cost_per_task_usd": 0.05804707330246913, "avg_duration_per_task_ms": 156059.7939814815, "avg_output_tokens_per_task": 6063.099537037037, "last_run_timestamp": "2026-08-29T16:07:02.345Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029752413333333328, "avg_duration_per_task_ms": 184850.22, "avg_output_tokens_per_task": 18168.666666666668, "last_run_timestamp": "2026-09-19T08:01:34.087Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7422221666666666, "accuracy_std_dev": 0.01791636261835036, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3279842533333333, "avg_duration_per_task_ms": 196889.34333333332, "avg_output_tokens_per_task": 10978.62, "last_run_timestamp": "2026-09-01T17:31:47.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.742222, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.17656242666666666, "avg_duration_per_task_ms": 161433.59333333332, "avg_output_tokens_per_task": 12527.553333333333, "last_run_timestamp": "2026-09-19T08:01:53.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7418405617977527, "accuracy_std_dev": 0.038139869614963295, "run_count": 25, "total_questions": 1246, "avg_cost_per_task_usd": 0.07584754895666132, "avg_duration_per_task_ms": 258067.6573033708, "avg_output_tokens_per_task": 5980.402889245586, "last_run_timestamp": "2026-08-29T16:31:41.307Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.741791919866444, "accuracy_std_dev": 0.04646084840389512, "run_count": 12, "total_questions": 599, "avg_cost_per_task_usd": 0.06980493155258766, "avg_duration_per_task_ms": 1045800.3572621035, "avg_output_tokens_per_task": 6776.651085141903, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.7416665, "accuracy_std_dev": 0.030322223240554103, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.008989355000000001, "avg_duration_per_task_ms": 277870.45, "avg_output_tokens_per_task": 5790.995, "last_run_timestamp": "2026-09-19T08:18:29.751Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7413501746835442, "accuracy_std_dev": 0.03375155758853722, "run_count": 16, "total_questions": 790, "avg_cost_per_task_usd": 0.08565632531645569, "avg_duration_per_task_ms": 345925.9759493671, "avg_output_tokens_per_task": 10900.69746835443, "last_run_timestamp": "2026-08-29T16:54:12.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.7412597005604483, "accuracy_std_dev": 0.13410921080552224, "run_count": 25, "total_questions": 1249, "avg_cost_per_task_usd": 0.09422745396317052, "avg_duration_per_task_ms": 253939.67013610888, "avg_output_tokens_per_task": 13385.784627702162, "last_run_timestamp": "2026-08-29T17:19:03.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.7411111666666667, "accuracy_std_dev": 0.016063189268393202, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04227563666666667, "avg_duration_per_task_ms": 199528.75333333333, "avg_output_tokens_per_task": 12876.693333333333, "last_run_timestamp": "2026-09-19T08:08:37.614Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7411110000000001, "accuracy_std_dev": 0.01901924301858516, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.009757083333333335, "avg_duration_per_task_ms": 401716.95666666667, "avg_output_tokens_per_task": 13276.11, "last_run_timestamp": "2026-09-19T08:02:29.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.7410255384615385, "accuracy_std_dev": 0.022470704136772563, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.02259469692307692, "avg_duration_per_task_ms": 344926.68615384615, "avg_output_tokens_per_task": 11307.772307692308, "last_run_timestamp": "2026-09-19T08:14:32.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.7409522857142857, "accuracy_std_dev": 0.03499274949273386, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.028752597142857144, "avg_duration_per_task_ms": 301953.97142857144, "avg_output_tokens_per_task": 17336.282857142858, "last_run_timestamp": "2026-09-19T08:14:26.428Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Cerebras", "accuracy": 0.7408834144838213, "accuracy_std_dev": 0.03308486202761159, "run_count": 26, "total_questions": 1298, "avg_cost_per_task_usd": 0.23451982434514637, "avg_duration_per_task_ms": 25362.9437596302, "avg_output_tokens_per_task": 5252.202619414484, "last_run_timestamp": "2026-08-15T14:16:11.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7406779661016949, "accuracy_std_dev": 0.0192449897289312, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.02004586440677966, "avg_duration_per_task_ms": 1117850.5423728814, "avg_output_tokens_per_task": 22657.922033898307, "last_run_timestamp": "2026-09-12T08:11:08.573Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.7402778333333333, "accuracy_std_dev": 0.039085846085536155, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.11041305000000001, "avg_duration_per_task_ms": 227804.8675, "avg_output_tokens_per_task": 5546.4675, "last_run_timestamp": "2026-08-29T16:32:10.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.7400002499999999, "accuracy_std_dev": 0.023093866430017766, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.22648705, "avg_duration_per_task_ms": 279140.11, "avg_output_tokens_per_task": 9247.74, "last_run_timestamp": "2026-09-20T10:30:07.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.74, "accuracy_std_dev": 0.03211111264967318, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.352840368, "avg_duration_per_task_ms": 224909.196, "avg_output_tokens_per_task": 10939.108, "last_run_timestamp": "2026-09-19T08:02:39.143Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.74, "accuracy_std_dev": 0.013333400001499999, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.11328389999999999, "avg_duration_per_task_ms": 818117.716, "avg_output_tokens_per_task": 22197.328, "last_run_timestamp": "2026-09-19T08:17:23.601Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.020059399999999998, "avg_duration_per_task_ms": 171159.34, "avg_output_tokens_per_task": 17767.96, "last_run_timestamp": "2026-09-19T06:00:02.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Ambient", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05110574, "avg_duration_per_task_ms": 158677.44, "avg_output_tokens_per_task": 8249.06, "last_run_timestamp": "2026-07-18T16:00:09.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AkashML", "accuracy": 0.74, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06375521, "avg_duration_per_task_ms": 66151.97, "avg_output_tokens_per_task": 3290.09, "last_run_timestamp": "2026-06-20T10:18:25.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.74, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.054267579999999996, "avg_duration_per_task_ms": 797443.04, "avg_output_tokens_per_task": 42406.53, "last_run_timestamp": "2026-08-29T16:33:37.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Novita", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02561014, "avg_duration_per_task_ms": 147210.62, "avg_output_tokens_per_task": 10787.18, "last_run_timestamp": "2026-07-18T08:04:42.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.74, "accuracy_std_dev": 0.01247214674785381, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07827993, "avg_duration_per_task_ms": 152857.22, "avg_output_tokens_per_task": 4638.005, "last_run_timestamp": "2026-09-19T08:18:06.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.74, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.060206680000000005, "avg_duration_per_task_ms": 137710.6, "avg_output_tokens_per_task": 4712.36, "last_run_timestamp": "2026-09-12T08:11:07.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.74, "accuracy_std_dev": 0.01247214674785381, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01460845, "avg_duration_per_task_ms": 194359.135, "avg_output_tokens_per_task": 4223.43, "last_run_timestamp": "2026-09-19T08:19:08.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.7399997500000001, "accuracy_std_dev": 0.031622882011092875, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17434206, "avg_duration_per_task_ms": 333592.31, "avg_output_tokens_per_task": 11320.165, "last_run_timestamp": "2026-09-19T08:12:48.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.7398561463790447, "accuracy_std_dev": 0.12410568998523018, "run_count": 26, "total_questions": 1298, "avg_cost_per_task_usd": 0.03732514175654854, "avg_duration_per_task_ms": 125480.83204930663, "avg_output_tokens_per_task": 5421.03312788906, "last_run_timestamp": "2026-08-29T16:11:58.048Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.7394636465517241, "accuracy_std_dev": 0.03855261066843646, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.027936172413793103, "avg_duration_per_task_ms": 390627.1882183908, "avg_output_tokens_per_task": 12578.12356321839, "last_run_timestamp": "2026-09-19T08:12:27.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.7393940909090909, "accuracy_std_dev": 0.022822002176745397, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5216491690909092, "avg_duration_per_task_ms": 720472.1036363636, "avg_output_tokens_per_task": 8824.263636363636, "last_run_timestamp": "2026-09-19T08:12:49.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.7393334, "accuracy_std_dev": 0.04024381912592295, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.161637104, "avg_duration_per_task_ms": 338949.096, "avg_output_tokens_per_task": 23077.328, "last_run_timestamp": "2026-09-19T08:16:41.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.7389743076923077, "accuracy_std_dev": 0.037834173804165296, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.06794495307692308, "avg_duration_per_task_ms": 166675.83615384615, "avg_output_tokens_per_task": 5442.511538461538, "last_run_timestamp": "2026-08-29T16:44:49.542Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.738889, "accuracy_std_dev": 0.017812306747489716, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.022993023333333338, "avg_duration_per_task_ms": 282964.01666666666, "avg_output_tokens_per_task": 13071.093333333334, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.7386666000000001, "accuracy_std_dev": 0.04470162612075761, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.005570916, "avg_duration_per_task_ms": 333893.98, "avg_output_tokens_per_task": 16678.144, "last_run_timestamp": "2026-09-19T08:07:54.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.7380952857142857, "accuracy_std_dev": 0.02487169616086462, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0953370257142857, "avg_duration_per_task_ms": 895388.7342857143, "avg_output_tokens_per_task": 40354.45428571429, "last_run_timestamp": "2026-09-19T16:00:28.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7377779999999999, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.030773986666666673, "avg_duration_per_task_ms": 124426.42666666667, "avg_output_tokens_per_task": 16143.133333333333, "last_run_timestamp": "2026-09-19T08:12:32.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.7377779999999999, "accuracy_std_dev": 0.01257094435593444, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08044259333333334, "avg_duration_per_task_ms": 104455.07333333333, "avg_output_tokens_per_task": 4500.253333333333, "last_run_timestamp": "2026-09-19T08:08:42.732Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.7377778333333334, "accuracy_std_dev": 0.023700698016561092, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06858023666666667, "avg_duration_per_task_ms": 76768.37666666666, "avg_output_tokens_per_task": 8623.293333333333, "last_run_timestamp": "2026-09-19T08:02:29.843Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.7375757272727272, "accuracy_std_dev": 0.015640496284214565, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.008148763636363637, "avg_duration_per_task_ms": 115345.35272727272, "avg_output_tokens_per_task": 10256.063636363637, "last_run_timestamp": "2026-08-29T17:19:58.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.7372451530612244, "accuracy_std_dev": 0.017853512726841714, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.022030132653061228, "avg_duration_per_task_ms": 127977.05612244898, "avg_output_tokens_per_task": 3810.637755102041, "last_run_timestamp": "2026-09-19T08:02:39.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.7368888666666666, "accuracy_std_dev": 0.02741899242706698, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.06124634666666667, "avg_duration_per_task_ms": 229793.576, "avg_output_tokens_per_task": 9405.036, "last_run_timestamp": "2026-08-29T16:16:50.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.7367347959183673, "accuracy_std_dev": 0.023800704305545237, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.01745185306122449, "avg_duration_per_task_ms": 351822.13469387754, "avg_output_tokens_per_task": 13937.518367346938, "last_run_timestamp": "2026-09-20T10:30:07.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.7366665, "accuracy_std_dev": 0.05666649999999995, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5401649000000001, "avg_duration_per_task_ms": 161301.85, "avg_output_tokens_per_task": 8843.07, "last_run_timestamp": "2026-09-01T13:02:17.066Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.7365892425940752, "accuracy_std_dev": 0.023972801823733503, "run_count": 25, "total_questions": 1249, "avg_cost_per_task_usd": 0.10787875340272218, "avg_duration_per_task_ms": 162741.05764611688, "avg_output_tokens_per_task": 8093.5548438751, "last_run_timestamp": "2026-09-19T08:15:13.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7365404329896906, "accuracy_std_dev": 0.04686504205487875, "run_count": 6, "total_questions": 291, "avg_cost_per_task_usd": 0.08916013058419243, "avg_duration_per_task_ms": 682362.1065292096, "avg_output_tokens_per_task": 7674.209621993127, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.7362636996336996, "accuracy_std_dev": 0.025314314324162108, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.1250874249084249, "avg_duration_per_task_ms": 496350.12637362635, "avg_output_tokens_per_task": 24530.12454212454, "last_run_timestamp": "2026-09-19T16:00:28.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.7360844606525911, "accuracy_std_dev": 0.10956776688246402, "run_count": 21, "total_questions": 1042, "avg_cost_per_task_usd": 0.0232232245681382, "avg_duration_per_task_ms": 366903.96257197694, "avg_output_tokens_per_task": 8722.57773512476, "last_run_timestamp": "2026-08-29T17:21:09.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.7358309365994236, "accuracy_std_dev": 0.0391618221121395, "run_count": 14, "total_questions": 694, "avg_cost_per_task_usd": 0.021028703170028824, "avg_duration_per_task_ms": 301715.76657060516, "avg_output_tokens_per_task": 12016.485590778098, "last_run_timestamp": "2026-09-19T08:12:02.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.7357541832402235, "accuracy_std_dev": 0.09788376088289748, "run_count": 18, "total_questions": 895, "avg_cost_per_task_usd": 0.09288570949720674, "avg_duration_per_task_ms": 339472.96424581006, "avg_output_tokens_per_task": 14305.882681564246, "last_run_timestamp": "2026-08-29T16:30:43.592Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7357256822966507, "accuracy_std_dev": 0.0866496604884399, "run_count": 21, "total_questions": 1045, "avg_cost_per_task_usd": 0.04395789377990429, "avg_duration_per_task_ms": 207924.5062200957, "avg_output_tokens_per_task": 6683.989473684211, "last_run_timestamp": "2026-08-29T17:16:16.996Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.012570472951413688, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13620342666666665, "avg_duration_per_task_ms": 101042.81333333334, "avg_output_tokens_per_task": 8769.966666666667, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.25940026000000005, "avg_duration_per_task_ms": 198685.81333333332, "avg_output_tokens_per_task": 13136.706666666667, "last_run_timestamp": "2026-09-19T08:07:33.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.7355555925925926, "accuracy_std_dev": 0.028688777003300966, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.03225971111111112, "avg_duration_per_task_ms": 115799.6525925926, "avg_output_tokens_per_task": 5434.608148148148, "last_run_timestamp": "2026-08-22T16:12:21.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7355553333333333, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13248254666666667, "avg_duration_per_task_ms": 94960.21333333333, "avg_output_tokens_per_task": 24879.886666666665, "last_run_timestamp": "2026-09-01T17:31:46.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.7351751940700809, "accuracy_std_dev": 0.08409205356369888, "run_count": 15, "total_questions": 742, "avg_cost_per_task_usd": 0.05205572641509434, "avg_duration_per_task_ms": 112166.10916442049, "avg_output_tokens_per_task": 4389.22371967655, "last_run_timestamp": "2026-08-01T08:06:01.594Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7351129425051335, "accuracy_std_dev": 0.026203090894778056, "run_count": 10, "total_questions": 487, "avg_cost_per_task_usd": 0.016946958932238188, "avg_duration_per_task_ms": 95053.74127310062, "avg_output_tokens_per_task": 5135.201232032854, "last_run_timestamp": "2026-09-19T16:00:28.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.735, "accuracy_std_dev": 0.15570928990590127, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.1830062116666667, "avg_duration_per_task_ms": 302480.2633333333, "avg_output_tokens_per_task": 14547.835833333333, "last_run_timestamp": "2026-08-29T16:58:32.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7349019999999999, "accuracy_std_dev": 0.03131376846391639, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.3548720011764706, "avg_duration_per_task_ms": 207107.23529411765, "avg_output_tokens_per_task": 10311.083529411764, "last_run_timestamp": "2026-09-19T08:12:57.394Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.7346031428571429, "accuracy_std_dev": 0.06504760378682801, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.05404573238095239, "avg_duration_per_task_ms": 148223.61523809523, "avg_output_tokens_per_task": 6767.127619047619, "last_run_timestamp": "2026-08-29T17:19:05.156Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.7339999, "accuracy_std_dev": 0.02772076878966382, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.07026960600000001, "avg_duration_per_task_ms": 306542.06, "avg_output_tokens_per_task": 10129.164, "last_run_timestamp": "2026-09-19T08:15:44.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.7338094642857144, "accuracy_std_dev": 0.04249849817302972, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.037900790000000004, "avg_duration_per_task_ms": 160187.04071428571, "avg_output_tokens_per_task": 5735.9064285714285, "last_run_timestamp": "2026-08-30T00:01:02.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.7333335, "accuracy_std_dev": 0.009428090417293783, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.38862442999999997, "avg_duration_per_task_ms": 146345.67333333334, "avg_output_tokens_per_task": 10987.79, "last_run_timestamp": "2026-09-19T08:08:42.374Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09979016999999998, "avg_duration_per_task_ms": 95640.78, "avg_output_tokens_per_task": 4171.89, "last_run_timestamp": "2026-09-01T12:57:13.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.733333347826087, "accuracy_std_dev": 0.1334925998058413, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.07098599043478261, "avg_duration_per_task_ms": 169848.0147826087, "avg_output_tokens_per_task": 6912.339130434782, "last_run_timestamp": "2026-08-29T17:05:17.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008480786666666665, "avg_duration_per_task_ms": 397846.0266666667, "avg_output_tokens_per_task": 19127.326666666668, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Fireworks", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.024944382578492966, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.035478496666666665, "avg_duration_per_task_ms": 63670.043333333335, "avg_output_tokens_per_task": 4095.423333333333, "last_run_timestamp": "2026-05-23T14:30:04.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.03250838448428001, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06780395555555555, "avg_duration_per_task_ms": 133879.79333333333, "avg_output_tokens_per_task": 7446.308888888889, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09733843333333332, "avg_duration_per_task_ms": 435402.0933333333, "avg_output_tokens_per_task": 12527.286666666667, "last_run_timestamp": "2026-09-19T08:13:35.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.73333325, "accuracy_std_dev": 0.018257509870940786, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.018071915, "avg_duration_per_task_ms": 95128.1, "avg_output_tokens_per_task": 4179.29, "last_run_timestamp": "2026-09-19T08:08:46.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2412942, "avg_duration_per_task_ms": 207134.84, "avg_output_tokens_per_task": 12538.34, "last_run_timestamp": "2026-08-01T08:06:12.346Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7329792361275088, "accuracy_std_dev": 0.03497211277654473, "run_count": 17, "total_questions": 847, "avg_cost_per_task_usd": 0.052468580873671775, "avg_duration_per_task_ms": 188219.26328217238, "avg_output_tokens_per_task": 7981.263282172373, "last_run_timestamp": "2026-08-29T16:55:12.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.7328770821917807, "accuracy_std_dev": 0.020632433065981837, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.011465595890410957, "avg_duration_per_task_ms": 215673.5684931507, "avg_output_tokens_per_task": 4199.7054794520545, "last_run_timestamp": "2026-09-19T08:07:58.188Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.7328571428571429, "accuracy_std_dev": 0.02487604007260557, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.3672934914285714, "avg_duration_per_task_ms": 232269.72285714286, "avg_output_tokens_per_task": 10834.185714285713, "last_run_timestamp": "2026-09-19T08:15:37.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "SiliconFlow", "accuracy": 0.7328571428571429, "accuracy_std_dev": 0.02986363565766697, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.01948525, "avg_duration_per_task_ms": 74288.42, "avg_output_tokens_per_task": 3454.5, "last_run_timestamp": "2026-06-06T18:57:18.338Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.7328309698492462, "accuracy_std_dev": 0.02818147501795285, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.07178455778894473, "avg_duration_per_task_ms": 101856.46733668342, "avg_output_tokens_per_task": 4297.814070351758, "last_run_timestamp": "2026-09-19T08:01:36.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.7327989038076153, "accuracy_std_dev": 0.1632218869480668, "run_count": 20, "total_questions": 998, "avg_cost_per_task_usd": 0.015171186372745492, "avg_duration_per_task_ms": 223130.1102204409, "avg_output_tokens_per_task": 8631.145290581162, "last_run_timestamp": "2026-08-29T16:47:28.160Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7327778333333334, "accuracy_std_dev": 0.04525304505193238, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09581732333333334, "avg_duration_per_task_ms": 160631.045, "avg_output_tokens_per_task": 8478.698333333334, "last_run_timestamp": "2026-09-19T08:07:22.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.7326667, "accuracy_std_dev": 0.04049139881271083, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.06109215, "avg_duration_per_task_ms": 223095.216, "avg_output_tokens_per_task": 7046.56, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7326621543624161, "accuracy_std_dev": 0.030789543369281885, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.09465771812080537, "avg_duration_per_task_ms": 390427.067114094, "avg_output_tokens_per_task": 11230.369127516778, "last_run_timestamp": "2026-08-29T16:33:38.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7326390000000002, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.014196437499999999, "avg_duration_per_task_ms": 145457.3125, "avg_output_tokens_per_task": 11907.520833333334, "last_run_timestamp": "2026-09-19T08:13:54.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AkashML", "accuracy": 0.7326315789473684, "accuracy_std_dev": 0.099281507423305, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.07054750842105263, "avg_duration_per_task_ms": 389640.21789473685, "avg_output_tokens_per_task": 37304.677894736844, "last_run_timestamp": "2026-08-15T14:08:35.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.7326006153846155, "accuracy_std_dev": 0.02879146589925737, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.09310996886446886, "avg_duration_per_task_ms": 226920.88644688643, "avg_output_tokens_per_task": 8628.521978021978, "last_run_timestamp": "2026-09-19T08:02:39.232Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.7325926666666667, "accuracy_std_dev": 0.02318878093197464, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.00917222, "avg_duration_per_task_ms": 347978.0577777778, "avg_output_tokens_per_task": 13519.62, "last_run_timestamp": "2026-09-19T08:14:36.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7322223333333333, "accuracy_std_dev": 0.024770556995935476, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07378156, "avg_duration_per_task_ms": 94238.19333333333, "avg_output_tokens_per_task": 11168.733333333334, "last_run_timestamp": "2026-09-19T16:00:24.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.7322221666666667, "accuracy_std_dev": 0.036599313561034026, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03977107333333332, "avg_duration_per_task_ms": 224271.51666666666, "avg_output_tokens_per_task": 20806.72333333333, "last_run_timestamp": "2026-09-19T08:01:53.320Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.7321211818181819, "accuracy_std_dev": 0.03937046246944339, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.04438583818181819, "avg_duration_per_task_ms": 119798.46909090909, "avg_output_tokens_per_task": 8507.941818181818, "last_run_timestamp": "2026-09-19T08:15:44.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.7318549798387096, "accuracy_std_dev": 0.04649888648139865, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.08042021975806452, "avg_duration_per_task_ms": 208719.5060483871, "avg_output_tokens_per_task": 8252.131048387097, "last_run_timestamp": "2026-09-19T08:12:24.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "NextBit", "accuracy": 0.7315789473684211, "accuracy_std_dev": 0.061133249354843225, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.0719111105263158, "avg_duration_per_task_ms": 470454.90105263155, "avg_output_tokens_per_task": 22870.695789473684, "last_run_timestamp": "2026-07-18T16:00:08.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.7314969432387313, "accuracy_std_dev": 0.09382820702418469, "run_count": 24, "total_questions": 1198, "avg_cost_per_task_usd": 0.07530729799666108, "avg_duration_per_task_ms": 205955.36894824708, "avg_output_tokens_per_task": 22577.263772954924, "last_run_timestamp": "2026-08-29T16:12:10.428Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7311556834170854, "accuracy_std_dev": 0.027058284160943663, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.02928007286432161, "avg_duration_per_task_ms": 481200.60552763817, "avg_output_tokens_per_task": 8314.90703517588, "last_run_timestamp": "2026-08-29T16:49:08.522Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.035416115465514666, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.3332058333333333, "avg_duration_per_task_ms": 117249.60666666667, "avg_output_tokens_per_task": 11086.966666666667, "last_run_timestamp": "2026-09-01T17:31:47.546Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Cloudflare", "accuracy": 0.730909090909091, "accuracy_std_dev": 0.04541816726107927, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.020825716363636365, "avg_duration_per_task_ms": 64202.28, "avg_output_tokens_per_task": 2951.6781818181817, "last_run_timestamp": "2026-05-23T09:28:32.982Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.7302799198473282, "accuracy_std_dev": 0.08316610177772252, "run_count": 21, "total_questions": 1048, "avg_cost_per_task_usd": 0.009317546755725192, "avg_duration_per_task_ms": 128886.75190839695, "avg_output_tokens_per_task": 7970.051526717557, "last_run_timestamp": "2026-08-29T17:11:34.630Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.7300784646464646, "accuracy_std_dev": 0.0316177947350811, "run_count": 12, "total_questions": 594, "avg_cost_per_task_usd": 0.04173444949494949, "avg_duration_per_task_ms": 219437.9090909091, "avg_output_tokens_per_task": 8350.580808080807, "last_run_timestamp": "2026-09-19T08:02:40.716Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.7300089191374662, "accuracy_std_dev": 0.028871736576028047, "run_count": 15, "total_questions": 742, "avg_cost_per_task_usd": 0.05800620754716981, "avg_duration_per_task_ms": 288668.6940700809, "avg_output_tokens_per_task": 8401.919137466308, "last_run_timestamp": "2026-08-29T16:47:08.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7300001666666667, "accuracy_std_dev": 0.03605552816336059, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11662036, "avg_duration_per_task_ms": 82058.93666666666, "avg_output_tokens_per_task": 9160.97, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.73, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.029231550000000005, "avg_duration_per_task_ms": 115126.35, "avg_output_tokens_per_task": 15983.89, "last_run_timestamp": "2026-09-19T08:14:06.082Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.73, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.013875090000000001, "avg_duration_per_task_ms": 398843.69, "avg_output_tokens_per_task": 15479.71, "last_run_timestamp": "2026-09-19T08:01:36.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.73, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23651355, "avg_duration_per_task_ms": 142691.29, "avg_output_tokens_per_task": 22898.27, "last_run_timestamp": "2026-09-01T17:31:43.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.73, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17840627, "avg_duration_per_task_ms": 413224.86, "avg_output_tokens_per_task": 8594.17, "last_run_timestamp": "2026-09-19T08:19:01.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.73, "accuracy_std_dev": 0.011385630973585375, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.26713958666666665, "avg_duration_per_task_ms": 328724.2033333333, "avg_output_tokens_per_task": 9769.553333333333, "last_run_timestamp": "2026-09-19T08:02:39.150Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.73, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.30719247000000005, "avg_duration_per_task_ms": 162813.34, "avg_output_tokens_per_task": 10978.12, "last_run_timestamp": "2026-09-01T17:32:02.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.7298765555555555, "accuracy_std_dev": 0.03755363117160363, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.035774098518518524, "avg_duration_per_task_ms": 170692.90592592594, "avg_output_tokens_per_task": 5967.2385185185185, "last_run_timestamp": "2026-08-30T00:01:09.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.7295238571428572, "accuracy_std_dev": 0.054494033042758974, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.007525531428571428, "avg_duration_per_task_ms": 323281.29714285716, "avg_output_tokens_per_task": 15952.262857142858, "last_run_timestamp": "2026-09-19T08:01:53.413Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7293334, "accuracy_std_dev": 0.03336017053073918, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.5038457519999999, "avg_duration_per_task_ms": 191881.072, "avg_output_tokens_per_task": 9815.496, "last_run_timestamp": "2026-09-19T08:08:36.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Phala", "accuracy": 0.729167, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.044939625000000004, "avg_duration_per_task_ms": 321431, "avg_output_tokens_per_task": 27916.854166666668, "last_run_timestamp": "2026-07-11T08:03:26.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Together", "accuracy": 0.7288888888888889, "accuracy_std_dev": 0.04121608220220313, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.013316022222222224, "avg_duration_per_task_ms": 85589.12666666666, "avg_output_tokens_per_task": 3636.0377777777776, "last_run_timestamp": "2026-06-20T09:31:46.744Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.7287470402684564, "accuracy_std_dev": 0.02324185312940621, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.007624875838926174, "avg_duration_per_task_ms": 319985.19295302016, "avg_output_tokens_per_task": 14093.862416107382, "last_run_timestamp": "2026-09-19T08:08:01.540Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.728581156626506, "accuracy_std_dev": 0.046862578058083824, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.017320277108433733, "avg_duration_per_task_ms": 435636.6847389558, "avg_output_tokens_per_task": 8247.0140562249, "last_run_timestamp": "2026-08-29T16:32:40.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Together", "accuracy": 0.7285714285714285, "accuracy_std_dev": 0.04323641700120444, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.09354192, "avg_duration_per_task_ms": 71979.10142857143, "avg_output_tokens_per_task": 3596.4542857142856, "last_run_timestamp": "2026-06-20T10:33:11.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.7285395183946488, "accuracy_std_dev": 0.03368260924011414, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.07459216722408027, "avg_duration_per_task_ms": 211118.72575250836, "avg_output_tokens_per_task": 8202.498327759198, "last_run_timestamp": "2026-09-19T08:15:24.969Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "Together", "accuracy": 0.727891030612245, "accuracy_std_dev": 0.023240919929888738, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.32119672959183676, "avg_duration_per_task_ms": 168213.4693877551, "avg_output_tokens_per_task": 61497.88775510204, "last_run_timestamp": "2026-07-04T08:06:25.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.72777775, "accuracy_std_dev": 0.028523387632271777, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.07063925333333335, "avg_duration_per_task_ms": 211161.295, "avg_output_tokens_per_task": 7779.595, "last_run_timestamp": "2026-09-19T08:02:29.968Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.7277776666666667, "accuracy_std_dev": 0.021914694321897864, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06174203666666666, "avg_duration_per_task_ms": 69022.76333333334, "avg_output_tokens_per_task": 9250.976666666667, "last_run_timestamp": "2026-09-19T08:02:29.873Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.7276787589285714, "accuracy_std_dev": 0.023663327303270934, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.019942993303571428, "avg_duration_per_task_ms": 481307.24776785716, "avg_output_tokens_per_task": 14518.11830357143, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.7276743037249284, "accuracy_std_dev": 0.10515567507327442, "run_count": 28, "total_questions": 1396, "avg_cost_per_task_usd": 0.06674029011461317, "avg_duration_per_task_ms": 57192.48782234957, "avg_output_tokens_per_task": 5553.101002865329, "last_run_timestamp": "2026-08-30T00:01:10.606Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.727619, "accuracy_std_dev": 0.031153175806924456, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3295257971428571, "avg_duration_per_task_ms": 203387.6257142857, "avg_output_tokens_per_task": 10294.07142857143, "last_run_timestamp": "2026-09-19T08:12:07.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.7274073888888889, "accuracy_std_dev": 0.03589348401581003, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.018933201111111114, "avg_duration_per_task_ms": 205864.55777777778, "avg_output_tokens_per_task": 6492.102222222222, "last_run_timestamp": "2026-07-18T16:00:09.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.7271793846153847, "accuracy_std_dev": 0.020500031911939387, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.06991619076923077, "avg_duration_per_task_ms": 246359.73846153845, "avg_output_tokens_per_task": 8152.42923076923, "last_run_timestamp": "2026-09-19T08:14:58.451Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7270382569659443, "accuracy_std_dev": 0.0355308519613672, "run_count": 13, "total_questions": 646, "avg_cost_per_task_usd": 0.03975771362229102, "avg_duration_per_task_ms": 168092.38235294117, "avg_output_tokens_per_task": 8093.3250773993805, "last_run_timestamp": "2026-09-19T08:12:07.843Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "OpenAI", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.27023132, "avg_duration_per_task_ms": 228964.68, "avg_output_tokens_per_task": 14058.24, "last_run_timestamp": "2026-08-01T08:05:59.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.726667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02366282, "avg_duration_per_task_ms": 103172.5, "avg_output_tokens_per_task": 4199.07, "last_run_timestamp": "2026-09-19T08:01:53.400Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.7266666818181818, "accuracy_std_dev": 0.0799747182275785, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.0664802909090909, "avg_duration_per_task_ms": 285594.60545454547, "avg_output_tokens_per_task": 27787.627272727274, "last_run_timestamp": "2026-08-29T17:00:54.109Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.030306899350184633, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.436704, "avg_duration_per_task_ms": 230592.73333333334, "avg_output_tokens_per_task": 8499.046666666667, "last_run_timestamp": "2026-09-19T08:21:11.819Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7266665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06289768999999999, "avg_duration_per_task_ms": 327232.67, "avg_output_tokens_per_task": 8904.97, "last_run_timestamp": "2026-09-19T08:08:46.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.7266665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20674873, "avg_duration_per_task_ms": 122950.24, "avg_output_tokens_per_task": 5001.27, "last_run_timestamp": "2026-09-01T13:00:51.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Chutes", "accuracy": 0.726324590738423, "accuracy_std_dev": 0.05193905756264829, "run_count": 16, "total_questions": 799, "avg_cost_per_task_usd": 0.026944361702127665, "avg_duration_per_task_ms": 165610.5068836045, "avg_output_tokens_per_task": 4209.88986232791, "last_run_timestamp": "2026-07-25T08:07:26.285Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.726, "accuracy_std_dev": 0.02707188783960217, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.029576488, "avg_duration_per_task_ms": 174487.376, "avg_output_tokens_per_task": 9637.224, "last_run_timestamp": "2026-09-19T16:00:32.511Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.7257141428571429, "accuracy_std_dev": 0.03693465701721885, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.2823705371428571, "avg_duration_per_task_ms": 385575.6, "avg_output_tokens_per_task": 8735.288571428571, "last_run_timestamp": "2026-09-19T08:02:40.804Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.7256411282051283, "accuracy_std_dev": 0.021901115569475008, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.02016126153846154, "avg_duration_per_task_ms": 253617.73333333334, "avg_output_tokens_per_task": 9900.05641025641, "last_run_timestamp": "2026-09-19T08:02:29.876Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7255556666666666, "accuracy_std_dev": 0.02114024890003792, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.058183759999999994, "avg_duration_per_task_ms": 881187.9266666666, "avg_output_tokens_per_task": 30140.16, "last_run_timestamp": "2026-09-19T08:15:00.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7253915503355705, "accuracy_std_dev": 0.1089886276971389, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.13323116275167785, "avg_duration_per_task_ms": 557640.1711409396, "avg_output_tokens_per_task": 25872.763422818793, "last_run_timestamp": "2026-09-19T08:15:43.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DekaLLM", "accuracy": 0.725, "accuracy_std_dev": 0.05545268253204706, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.06306274499999999, "avg_duration_per_task_ms": 307709.445, "avg_output_tokens_per_task": 29336.17, "last_run_timestamp": "2026-05-16T11:09:22.970Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.7249544826958105, "accuracy_std_dev": 0.08103658076897909, "run_count": 22, "total_questions": 1098, "avg_cost_per_task_usd": 0.056935027322404366, "avg_duration_per_task_ms": 478336.01821493625, "avg_output_tokens_per_task": 36495.10746812386, "last_run_timestamp": "2026-08-29T16:07:02.358Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.7247474713804714, "accuracy_std_dev": 0.08284079836897833, "run_count": 12, "total_questions": 594, "avg_cost_per_task_usd": 0.06815684343434343, "avg_duration_per_task_ms": 133143.7861952862, "avg_output_tokens_per_task": 12155.594276094276, "last_run_timestamp": "2026-08-29T16:57:02.331Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.7244444999999999, "accuracy_std_dev": 0.032640910103988625, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.10185021333333334, "avg_duration_per_task_ms": 107547.30666666667, "avg_output_tokens_per_task": 8542.055, "last_run_timestamp": "2026-09-19T08:02:40.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7244444666666666, "accuracy_std_dev": 0.030663471653563444, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.09078783866666666, "avg_duration_per_task_ms": 211616.812, "avg_output_tokens_per_task": 7390.202666666667, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.7244443333333334, "accuracy_std_dev": 0.022662526857249517, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.47615288666666666, "avg_duration_per_task_ms": 770487.4266666666, "avg_output_tokens_per_task": 9661.306666666667, "last_run_timestamp": "2026-09-19T08:14:36.033Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.7243589615384616, "accuracy_std_dev": 0.033839378789852756, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.03414883846153846, "avg_duration_per_task_ms": 357708.1330769231, "avg_output_tokens_per_task": 8693.559230769231, "last_run_timestamp": "2026-08-29T17:14:29.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.7241015454545454, "accuracy_std_dev": 0.06989016607046616, "run_count": 19, "total_questions": 946, "avg_cost_per_task_usd": 0.05174934883720931, "avg_duration_per_task_ms": 224979.1617336152, "avg_output_tokens_per_task": 6867.136363636364, "last_run_timestamp": "2026-08-29T16:53:19.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.7237445479452055, "accuracy_std_dev": 0.01534481785127761, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.060365219178082186, "avg_duration_per_task_ms": 536636.493150685, "avg_output_tokens_per_task": 12764.582191780823, "last_run_timestamp": "2026-09-19T08:12:49.799Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7233335, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02063872, "avg_duration_per_task_ms": 638594.61, "avg_output_tokens_per_task": 13867.61, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.7233335, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.16905831, "avg_duration_per_task_ms": 501852.61, "avg_output_tokens_per_task": 40516.58, "last_run_timestamp": "2026-09-19T08:15:01.262Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.7233335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020558879999999998, "avg_duration_per_task_ms": 168582.3, "avg_output_tokens_per_task": 7751, "last_run_timestamp": "2026-09-19T16:00:28.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.7233335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.013037239999999999, "avg_duration_per_task_ms": 45998.83, "avg_output_tokens_per_task": 3876.1, "last_run_timestamp": "2026-09-19T08:01:36.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.7233333076923077, "accuracy_std_dev": 0.03726778815831465, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.031507099230769234, "avg_duration_per_task_ms": 241507.73384615386, "avg_output_tokens_per_task": 8188.238461538462, "last_run_timestamp": "2026-08-29T16:13:54.386Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09717974, "avg_duration_per_task_ms": 146982.06, "avg_output_tokens_per_task": 51201.48, "last_run_timestamp": "2026-09-01T17:23:14.324Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.7225326592178771, "accuracy_std_dev": 0.03504286667804778, "run_count": 18, "total_questions": 895, "avg_cost_per_task_usd": 0.11005581787709497, "avg_duration_per_task_ms": 148535.98547486032, "avg_output_tokens_per_task": 8251.711731843576, "last_run_timestamp": "2026-09-19T08:12:39.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.7224880535885168, "accuracy_std_dev": 0.07672207222462067, "run_count": 21, "total_questions": 1045, "avg_cost_per_task_usd": 0.10305108325358853, "avg_duration_per_task_ms": 101482.16076555024, "avg_output_tokens_per_task": 7390.176076555024, "last_run_timestamp": "2026-09-19T08:18:15.795Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Morph", "accuracy": 0.722222316091954, "accuracy_std_dev": 0.031278207950127646, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.1106247183908046, "avg_duration_per_task_ms": 557346.2183908046, "avg_output_tokens_per_task": 24361.238505747126, "last_run_timestamp": "2026-08-15T08:15:35.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.7222222356902357, "accuracy_std_dev": 0.024462876516718586, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.3977955387205387, "avg_duration_per_task_ms": 396867.7037037037, "avg_output_tokens_per_task": 8974.410774410775, "last_run_timestamp": "2026-09-19T08:08:39.796Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.722222, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.019033145833333334, "avg_duration_per_task_ms": 65993.54166666667, "avg_output_tokens_per_task": 3355.9791666666665, "last_run_timestamp": "2026-09-12T08:12:22.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.7221293377926422, "accuracy_std_dev": 0.03777626478101768, "run_count": 24, "total_questions": 1196, "avg_cost_per_task_usd": 0.041799248327759206, "avg_duration_per_task_ms": 295964.41471571906, "avg_output_tokens_per_task": 9079.295986622074, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.7220366861435726, "accuracy_std_dev": 0.042939457372476286, "run_count": 24, "total_questions": 1198, "avg_cost_per_task_usd": 0.038994771285475786, "avg_duration_per_task_ms": 293176.03756260435, "avg_output_tokens_per_task": 8310.446577629382, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.7216665, "accuracy_std_dev": 0.0016665000000000152, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.010394970000000002, "avg_duration_per_task_ms": 238439.51, "avg_output_tokens_per_task": 14864.89, "last_run_timestamp": "2026-09-19T08:12:48.672Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.721603583518931, "accuracy_std_dev": 0.028097817578797066, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.0643464008908686, "avg_duration_per_task_ms": 278378.1247216036, "avg_output_tokens_per_task": 8733.859688195991, "last_run_timestamp": "2026-09-19T08:18:14.730Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.7214111715328467, "accuracy_std_dev": 0.029999031700351234, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.10158374817518248, "avg_duration_per_task_ms": 91062.5401459854, "avg_output_tokens_per_task": 10938.527372262773, "last_run_timestamp": "2026-09-19T08:15:48.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.7210941505922165, "accuracy_std_dev": 0.03106900471084039, "run_count": 12, "total_questions": 591, "avg_cost_per_task_usd": 0.1026584399323181, "avg_duration_per_task_ms": 291565.6379018613, "avg_output_tokens_per_task": 13900.038917089678, "last_run_timestamp": "2026-09-19T08:12:26.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.7205555555555555, "accuracy_std_dev": 0.06057596801741523, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.0473878211111111, "avg_duration_per_task_ms": 163562.02, "avg_output_tokens_per_task": 7193.1466666666665, "last_run_timestamp": "2026-08-29T17:08:03.684Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7205390303030302, "accuracy_std_dev": 0.006190499999999988, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.1266920202020202, "avg_duration_per_task_ms": 184926.9090909091, "avg_output_tokens_per_task": 9286.343434343435, "last_run_timestamp": "2026-09-19T16:00:24.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.7203921176470588, "accuracy_std_dev": 0.042378295969520495, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.08408886352941176, "avg_duration_per_task_ms": 167842.00235294117, "avg_output_tokens_per_task": 6586.608235294118, "last_run_timestamp": "2026-08-29T16:46:44.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.7201584739970283, "accuracy_std_dev": 0.027151418343285973, "run_count": 27, "total_questions": 1346, "avg_cost_per_task_usd": 0.02274276225854383, "avg_duration_per_task_ms": 279206.04011887073, "avg_output_tokens_per_task": 8300.97028231798, "last_run_timestamp": "2026-08-29T16:11:34.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "ModelRun", "accuracy": 0.720112529535865, "accuracy_std_dev": 0.054372151849016614, "run_count": 19, "total_questions": 948, "avg_cost_per_task_usd": 0.01960626371308017, "avg_duration_per_task_ms": 52898.649789029536, "avg_output_tokens_per_task": 3908.25, "last_run_timestamp": "2026-07-11T08:05:22.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.72, "accuracy_std_dev": 0.11915537433269066, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.013665298181818183, "avg_duration_per_task_ms": 146966.81818181818, "avg_output_tokens_per_task": 8880.298181818182, "last_run_timestamp": "2026-08-29T16:56:37.261Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03307506, "avg_duration_per_task_ms": 411711.74, "avg_output_tokens_per_task": 12561.32, "last_run_timestamp": "2026-09-19T08:09:58.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.72, "accuracy_std_dev": 0.030306862688176752, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.3160981133333334, "avg_duration_per_task_ms": 461033.8333333333, "avg_output_tokens_per_task": 8961.443333333333, "last_run_timestamp": "2026-09-19T08:01:53.394Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "BaseTen", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15154258, "avg_duration_per_task_ms": 105755.46, "avg_output_tokens_per_task": 13133.32, "last_run_timestamp": "2026-08-04T03:12:01.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.72, "accuracy_std_dev": 0.01845912401316305, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.13026846666666667, "avg_duration_per_task_ms": 109723.80666666667, "avg_output_tokens_per_task": 4025.5433333333335, "last_run_timestamp": "2026-09-01T13:00:52.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.028463379999999996, "avg_duration_per_task_ms": 583244.64, "avg_output_tokens_per_task": 43866.54, "last_run_timestamp": "2026-08-08T08:27:52.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.72, "accuracy_std_dev": 0.013332999999999984, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.013628439999999999, "avg_duration_per_task_ms": 407344.14, "avg_output_tokens_per_task": 6225.73, "last_run_timestamp": "2026-09-19T08:19:00.695Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.71999975, "accuracy_std_dev": 0.008164965810553043, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.010504485, "avg_duration_per_task_ms": 163005.685, "avg_output_tokens_per_task": 18389.035, "last_run_timestamp": "2026-09-19T08:08:45.471Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.7193335, "accuracy_std_dev": 0.02882120747036806, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.021860742000000002, "avg_duration_per_task_ms": 113473.506, "avg_output_tokens_per_task": 6563.78, "last_run_timestamp": "2026-09-19T16:00:28.611Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.7191666250000001, "accuracy_std_dev": 0.020666060140587385, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0268572725, "avg_duration_per_task_ms": 236107.135, "avg_output_tokens_per_task": 10157.88, "last_run_timestamp": "2026-08-29T17:16:15.215Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.7186897976878613, "accuracy_std_dev": 0.027015736117292515, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.02169187283236994, "avg_duration_per_task_ms": 314303.23988439306, "avg_output_tokens_per_task": 7575.85549132948, "last_run_timestamp": "2026-08-29T16:48:46.265Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7184294759615385, "accuracy_std_dev": 0.1596697904675889, "run_count": 21, "total_questions": 1040, "avg_cost_per_task_usd": 0.00781735096153846, "avg_duration_per_task_ms": 183262.19615384616, "avg_output_tokens_per_task": 7099.238461538462, "last_run_timestamp": "2026-08-29T16:33:29.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.7180951428571429, "accuracy_std_dev": 0.029323408959048256, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.35473523142857144, "avg_duration_per_task_ms": 239903.80285714284, "avg_output_tokens_per_task": 10770.68857142857, "last_run_timestamp": "2026-09-19T08:12:57.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7175246360294117, "accuracy_std_dev": 0.06146597028417599, "run_count": 11, "total_questions": 544, "avg_cost_per_task_usd": 0.12441467830882355, "avg_duration_per_task_ms": 999918.8308823529, "avg_output_tokens_per_task": 23722.5, "last_run_timestamp": "2026-09-19T16:00:30.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.7174358846153847, "accuracy_std_dev": 0.03935710293759606, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.03685178846153846, "avg_duration_per_task_ms": 327901.3692307692, "avg_output_tokens_per_task": 11511.836153846154, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.7174358461538461, "accuracy_std_dev": 0.03186920095989991, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.06687680461538463, "avg_duration_per_task_ms": 239262.95230769232, "avg_output_tokens_per_task": 7639.2738461538465, "last_run_timestamp": "2026-09-19T08:13:05.171Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.71733335, "accuracy_std_dev": 0.14592836629294353, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.013306622999999998, "avg_duration_per_task_ms": 168498.887, "avg_output_tokens_per_task": 7340.109, "last_run_timestamp": "2026-08-29T16:25:14.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7166666, "accuracy_std_dev": 0.033993437276039026, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.006462468, "avg_duration_per_task_ms": 374148.128, "avg_output_tokens_per_task": 12453.44, "last_run_timestamp": "2026-09-19T08:17:29.225Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.041574799999999995, "avg_duration_per_task_ms": 130749.55, "avg_output_tokens_per_task": 23910.86, "last_run_timestamp": "2026-09-19T08:12:53.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "model_slug": "nvidia/nemotron-3-ultra-550b-a55b", "model_name": "NVIDIA: Nemotron 3 Ultra", "provider_name": "DeepInfra", "accuracy": 0.7162163243243244, "accuracy_std_dev": 0.04755751813214069, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.07101010135135136, "avg_duration_per_task_ms": 195853.2972972973, "avg_output_tokens_per_task": 10371.256756756757, "last_run_timestamp": "2026-06-30T19:48:55.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7161505810564663, "accuracy_std_dev": 0.134318387636952, "run_count": 22, "total_questions": 1098, "avg_cost_per_task_usd": 0.07765216575591986, "avg_duration_per_task_ms": 227644.69489981784, "avg_output_tokens_per_task": 5419.0373406193075, "last_run_timestamp": "2026-08-29T16:44:15.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "SiliconFlow", "accuracy": 0.7161505118397086, "accuracy_std_dev": 0.03840795876268506, "run_count": 22, "total_questions": 1098, "avg_cost_per_task_usd": 0.04527011657559199, "avg_duration_per_task_ms": 358551.0282331512, "avg_output_tokens_per_task": 7699.158469945355, "last_run_timestamp": "2026-08-01T17:38:16.661Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Together", "accuracy": 0.716, "accuracy_std_dev": 0.024979991993593582, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.035929062000000005, "avg_duration_per_task_ms": 69854.32, "avg_output_tokens_per_task": 3091.818, "last_run_timestamp": "2026-05-16T13:21:53.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7155646528925619, "accuracy_std_dev": 0.07922982153103719, "run_count": 5, "total_questions": 242, "avg_cost_per_task_usd": 0.11734726446280992, "avg_duration_per_task_ms": 254637.0785123967, "avg_output_tokens_per_task": 42006.561983471074, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7155556666666666, "accuracy_std_dev": 0.040855070557058414, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012743326666666667, "avg_duration_per_task_ms": 1224143.2066666668, "avg_output_tokens_per_task": 18669.653333333332, "last_run_timestamp": "2026-08-29T09:20:05.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.715, "accuracy_std_dev": 0.0016670000000000296, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04256534, "avg_duration_per_task_ms": 221321.22, "avg_output_tokens_per_task": 26641.22, "last_run_timestamp": "2026-09-19T08:12:49.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.715, "accuracy_std_dev": 0.05624832837071695, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17734545000000002, "avg_duration_per_task_ms": 304190.625, "avg_output_tokens_per_task": 10162.455, "last_run_timestamp": "2026-09-19T08:01:53.330Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.715, "accuracy_std_dev": 0.04499999999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.16922785, "avg_duration_per_task_ms": 299708.78, "avg_output_tokens_per_task": 9901.85, "last_run_timestamp": "2026-08-29T08:38:56.982Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.71466672, "accuracy_std_dev": 0.03461532403144019, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.04134587520000001, "avg_duration_per_task_ms": 175611.5344, "avg_output_tokens_per_task": 5977.7496, "last_run_timestamp": "2026-08-30T00:01:09.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.7145237857142858, "accuracy_std_dev": 0.03996240254052832, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.038241464999999995, "avg_duration_per_task_ms": 157853.17285714287, "avg_output_tokens_per_task": 5122.270714285714, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.7142854897959183, "accuracy_std_dev": 0.039861000000000035, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.11979946938775511, "avg_duration_per_task_ms": 677914.6734693878, "avg_output_tokens_per_task": 54287.663265306124, "last_run_timestamp": "2026-09-01T17:30:15.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.7141900923694778, "accuracy_std_dev": 0.03172465567661847, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.03655370080321286, "avg_duration_per_task_ms": 44500.680722891564, "avg_output_tokens_per_task": 5750.947791164659, "last_run_timestamp": "2026-08-08T08:31:07.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.7141667083333333, "accuracy_std_dev": 0.07459301504524356, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.05860941499999999, "avg_duration_per_task_ms": 55370.06916666667, "avg_output_tokens_per_task": 5220.491666666667, "last_run_timestamp": "2026-08-29T16:33:39.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7138048585858586, "accuracy_std_dev": 0.07263833137825143, "run_count": 12, "total_questions": 594, "avg_cost_per_task_usd": 0.04328885521885522, "avg_duration_per_task_ms": 234338.40235690237, "avg_output_tokens_per_task": 24374.796296296296, "last_run_timestamp": "2026-09-19T16:00:22.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.7137096552419355, "accuracy_std_dev": 0.08299828694453579, "run_count": 20, "total_questions": 992, "avg_cost_per_task_usd": 0.05201826612903226, "avg_duration_per_task_ms": 463441.87298387097, "avg_output_tokens_per_task": 37741.06451612903, "last_run_timestamp": "2026-08-29T17:09:45.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.7135207630522089, "accuracy_std_dev": 0.04226729284636053, "run_count": 20, "total_questions": 996, "avg_cost_per_task_usd": 0.0358368875502008, "avg_duration_per_task_ms": 206817.1746987952, "avg_output_tokens_per_task": 6480.377510040161, "last_run_timestamp": "2026-08-29T16:33:11.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.7133335, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.041616520000000004, "avg_duration_per_task_ms": 141684.91, "avg_output_tokens_per_task": 13886.08, "last_run_timestamp": "2026-09-19T08:06:54.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.7133335, "accuracy_std_dev": 0.02708037419885478, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.31487034, "avg_duration_per_task_ms": 416220.92, "avg_output_tokens_per_task": 8817.575, "last_run_timestamp": "2026-09-19T08:08:01.627Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7133335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06446003, "avg_duration_per_task_ms": 348420.47, "avg_output_tokens_per_task": 34152.4, "last_run_timestamp": "2026-09-01T12:41:25.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.7133333333333334, "accuracy_std_dev": 0.019625871060640555, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12069053999999999, "avg_duration_per_task_ms": 78204.86666666667, "avg_output_tokens_per_task": 4256.786666666667, "last_run_timestamp": "2026-09-01T12:59:40.225Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.7133332857142858, "accuracy_std_dev": 0.03703300972960005, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.32220135714285714, "avg_duration_per_task_ms": 407576.0685714286, "avg_output_tokens_per_task": 9593.411428571428, "last_run_timestamp": "2026-09-19T08:14:26.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.7133330000000001, "accuracy_std_dev": 0.040000000000000036, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07565915000000001, "avg_duration_per_task_ms": 95308.62, "avg_output_tokens_per_task": 3758.94, "last_run_timestamp": "2026-09-01T13:00:41.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.57503108, "avg_duration_per_task_ms": 657829.88, "avg_output_tokens_per_task": 51958.96, "last_run_timestamp": "2026-09-01T13:00:31.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0723164, "avg_duration_per_task_ms": 346522.38, "avg_output_tokens_per_task": 59152.72, "last_run_timestamp": "2026-09-12T16:00:24.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.713333, "accuracy_std_dev": 0.037416573867739444, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.23679781500000005, "avg_duration_per_task_ms": 241514.25, "avg_output_tokens_per_task": 20554.75, "last_run_timestamp": "2026-09-19T08:07:26.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03806284, "avg_duration_per_task_ms": 229552.52, "avg_output_tokens_per_task": 12003.52, "last_run_timestamp": "2026-09-19T08:01:53.307Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "BaseTen", "accuracy": 0.7131498513761468, "accuracy_std_dev": 0.15547389772642844, "run_count": 11, "total_questions": 545, "avg_cost_per_task_usd": 0.07655383853211008, "avg_duration_per_task_ms": 42631.811009174315, "avg_output_tokens_per_task": 4027.154128440367, "last_run_timestamp": "2026-07-18T08:06:26.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "WandB", "accuracy": 0.7123519847715736, "accuracy_std_dev": 0.02073271203919062, "run_count": 4, "total_questions": 197, "avg_cost_per_task_usd": 0.08227161928934011, "avg_duration_per_task_ms": 178521.6345177665, "avg_output_tokens_per_task": 15413.27918781726, "last_run_timestamp": "2026-07-18T08:00:15.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.7121428571428572, "accuracy_std_dev": 0.03285629950656808, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.06669165142857143, "avg_duration_per_task_ms": 190237.55285714284, "avg_output_tokens_per_task": 9631.787857142857, "last_run_timestamp": "2026-08-29T17:17:21.072Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.7121040825791854, "accuracy_std_dev": 0.036688548792126165, "run_count": 18, "total_questions": 884, "avg_cost_per_task_usd": 0.09261668552036198, "avg_duration_per_task_ms": 73568.46040723982, "avg_output_tokens_per_task": 7541.7307692307695, "last_run_timestamp": "2026-09-19T08:14:35.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7117119324324325, "accuracy_std_dev": 0.01247848271581482, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.6902402702702702, "avg_duration_per_task_ms": 688360.5945945946, "avg_output_tokens_per_task": 56108.03378378379, "last_run_timestamp": "2026-09-01T13:02:58.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.7116858275862069, "accuracy_std_dev": 0.05977603864750285, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.024618107758620687, "avg_duration_per_task_ms": 136410.2212643678, "avg_output_tokens_per_task": 6874.7801724137935, "last_run_timestamp": "2026-09-19T16:00:24.105Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.7116787627737226, "accuracy_std_dev": 0.14935343511392893, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.10796777189781023, "avg_duration_per_task_ms": 148941.3010948905, "avg_output_tokens_per_task": 7939.4324817518245, "last_run_timestamp": "2026-09-19T08:12:25.116Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.7115308465171193, "accuracy_std_dev": 0.1512936932248644, "run_count": 17, "total_questions": 847, "avg_cost_per_task_usd": 0.0987406765053129, "avg_duration_per_task_ms": 230873.95631641085, "avg_output_tokens_per_task": 6065.546635182999, "last_run_timestamp": "2026-08-29T16:35:10.931Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Fireworks", "accuracy": 0.7114285714285714, "accuracy_std_dev": 0.04323641700120444, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.013884184285714283, "avg_duration_per_task_ms": 81133.41285714286, "avg_output_tokens_per_task": 3767.2885714285712, "last_run_timestamp": "2026-06-20T08:38:34.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.7110792363896848, "accuracy_std_dev": 0.04266601421056426, "run_count": 28, "total_questions": 1396, "avg_cost_per_task_usd": 0.01819265616045845, "avg_duration_per_task_ms": 135492.12392550142, "avg_output_tokens_per_task": 5865.690544412608, "last_run_timestamp": "2026-08-29T16:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "auto-routing", "accuracy": 0.711, "accuracy_std_dev": 0.034786648141204975, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.014063111, "avg_duration_per_task_ms": 146044.761, "avg_output_tokens_per_task": 4579.124, "last_run_timestamp": "2026-08-01T17:39:06.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7109804117647058, "accuracy_std_dev": 0.05357489628334922, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.07005771294117646, "avg_duration_per_task_ms": 197809.54470588235, "avg_output_tokens_per_task": 7675.065882352941, "last_run_timestamp": "2026-08-29T16:43:32.997Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.7108696086956522, "accuracy_std_dev": 0.03557966787700785, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.04292889913043479, "avg_duration_per_task_ms": 192476.08521739132, "avg_output_tokens_per_task": 4315.182608695653, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.71033335, "accuracy_std_dev": 0.028555927386927915, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.031039679999999997, "avg_duration_per_task_ms": 206099.06, "avg_output_tokens_per_task": 6448.078, "last_run_timestamp": "2026-08-29T16:33:11.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.7102565384615385, "accuracy_std_dev": 0.0380387521707805, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.04511441076923077, "avg_duration_per_task_ms": 639644.943076923, "avg_output_tokens_per_task": 6877.484615384616, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.71, "accuracy_std_dev": 0.023332999999999993, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.022729809999999996, "avg_duration_per_task_ms": 177235.62, "avg_output_tokens_per_task": 14095.11, "last_run_timestamp": "2026-09-19T08:19:02.035Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.71, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.009475860000000001, "avg_duration_per_task_ms": 101117.88, "avg_output_tokens_per_task": 5437.48, "last_run_timestamp": "2026-09-01T17:32:44.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Parasail", "accuracy": 0.71, "accuracy_std_dev": 0.03958114029012637, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.019811326666666667, "avg_duration_per_task_ms": 130532.76, "avg_output_tokens_per_task": 3719.2966666666666, "last_run_timestamp": "2026-05-16T23:02:52.223Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DekaLLM", "accuracy": 0.71, "accuracy_std_dev": 0.043588989435406726, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02106588, "avg_duration_per_task_ms": 66495.715, "avg_output_tokens_per_task": 1737.11, "last_run_timestamp": "2026-05-02T22:49:37.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7098539343065693, "accuracy_std_dev": 0.053780596538925134, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.09852927554744526, "avg_duration_per_task_ms": 122619.91970802919, "avg_output_tokens_per_task": 8950.552919708029, "last_run_timestamp": "2026-09-19T08:06:45.320Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7094593108108108, "accuracy_std_dev": 0.025596061967072654, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.29917547972972974, "avg_duration_per_task_ms": 177661.23648648648, "avg_output_tokens_per_task": 4943.472972972973, "last_run_timestamp": "2026-09-01T13:00:41.783Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.7093334, "accuracy_std_dev": 0.04572378520463939, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.041164836, "avg_duration_per_task_ms": 324796.104, "avg_output_tokens_per_task": 14246.772, "last_run_timestamp": "2026-09-19T08:07:37.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.7091666875, "accuracy_std_dev": 0.07271997399676271, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.047436745, "avg_duration_per_task_ms": 166812.7925, "avg_output_tokens_per_task": 6450.30625, "last_run_timestamp": "2026-08-29T16:45:57.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.7088888888888889, "accuracy_std_dev": 0.032810672760363424, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08025308222222222, "avg_duration_per_task_ms": 72592.39333333333, "avg_output_tokens_per_task": 8513.65111111111, "last_run_timestamp": "2026-09-19T08:07:49.572Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7088886666666667, "accuracy_std_dev": 0.0339208519579853, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.08842117999999999, "avg_duration_per_task_ms": 151404.06666666668, "avg_output_tokens_per_task": 46789.46333333333, "last_run_timestamp": "2026-09-01T17:31:47.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7087543737373738, "accuracy_std_dev": 0.06057975459466635, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.2684698484848485, "avg_duration_per_task_ms": 369025.3080808081, "avg_output_tokens_per_task": 8958.444444444445, "last_run_timestamp": "2026-09-19T08:20:02.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7087053928571428, "accuracy_std_dev": 0.05722548330707566, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.04908466741071428, "avg_duration_per_task_ms": 738553.59375, "avg_output_tokens_per_task": 50678.734375, "last_run_timestamp": "2026-09-12T16:00:26.253Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7083335, "accuracy_std_dev": 0.00833350000000005, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.040671439999999996, "avg_duration_per_task_ms": 147596.48, "avg_output_tokens_per_task": 22423.42, "last_run_timestamp": "2026-09-19T08:09:43.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.7083334549549549, "accuracy_std_dev": 0.032652328452317504, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.020894054054054055, "avg_duration_per_task_ms": 99127.93243243243, "avg_output_tokens_per_task": 6316.898648648648, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.70833325, "accuracy_std_dev": 0.02511102568967462, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.37473283499999993, "avg_duration_per_task_ms": 183894.045, "avg_output_tokens_per_task": 11774.475, "last_run_timestamp": "2026-09-19T08:06:51.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7078693118040089, "accuracy_std_dev": 0.037495695230779585, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.13154344097995546, "avg_duration_per_task_ms": 74759.52783964365, "avg_output_tokens_per_task": 10228.11135857461, "last_run_timestamp": "2026-08-29T17:11:32.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.7073808571428571, "accuracy_std_dev": 0.0460964802078317, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.02892358, "avg_duration_per_task_ms": 123301.34714285714, "avg_output_tokens_per_task": 4939.147142857143, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.7073252731854838, "accuracy_std_dev": 0.20674647626668274, "run_count": 20, "total_questions": 992, "avg_cost_per_task_usd": 0.10164331249999999, "avg_duration_per_task_ms": 197665.9133064516, "avg_output_tokens_per_task": 10714.620967741936, "last_run_timestamp": "2026-07-18T16:00:09.881Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.7069688236775818, "accuracy_std_dev": 0.03585183782858977, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.3883721914357683, "avg_duration_per_task_ms": 201494.02770780856, "avg_output_tokens_per_task": 8021.390428211587, "last_run_timestamp": "2026-09-19T08:15:14.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.706867824120603, "accuracy_std_dev": 0.024112945099248537, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.12147276381909548, "avg_duration_per_task_ms": 366567.0226130653, "avg_output_tokens_per_task": 8862.291457286432, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openrouter/auto-beta", "model_slug": "openrouter/auto-beta", "model_name": "Auto Router (Beta)", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2407015, "avg_duration_per_task_ms": 149076.32, "avg_output_tokens_per_task": 17938.72, "last_run_timestamp": "2026-07-22T00:25:39.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.3541182, "avg_duration_per_task_ms": 1646927.84, "avg_output_tokens_per_task": 24534.74, "last_run_timestamp": "2026-09-10T00:00:02.948Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02753204, "avg_duration_per_task_ms": 825683.52, "avg_output_tokens_per_task": 15373.7, "last_run_timestamp": "2026-09-12T08:11:33.505Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02431272, "avg_duration_per_task_ms": 304327.62, "avg_output_tokens_per_task": 14297.9, "last_run_timestamp": "2026-09-12T08:11:25.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "WandB", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08614948, "avg_duration_per_task_ms": 52133.28, "avg_output_tokens_per_task": 7649.3, "last_run_timestamp": "2026-07-18T08:04:46.262Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00934721, "avg_duration_per_task_ms": 103901.71, "avg_output_tokens_per_task": 5395.91, "last_run_timestamp": "2026-09-01T17:30:15.713Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "ModelRun", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1838298, "avg_duration_per_task_ms": 329811.9, "avg_output_tokens_per_task": 9540.84, "last_run_timestamp": "2026-08-15T00:16:18.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06360505, "avg_duration_per_task_ms": 327814.04, "avg_output_tokens_per_task": 33622.71, "last_run_timestamp": "2026-09-01T13:01:01.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.033993463423951896, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017786823333333337, "avg_duration_per_task_ms": 128082.29, "avg_output_tokens_per_task": 8061.84, "last_run_timestamp": "2026-05-23T11:46:07.128Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7064618061465722, "accuracy_std_dev": 0.09810038286783905, "run_count": 17, "total_questions": 846, "avg_cost_per_task_usd": 0.11687022458628843, "avg_duration_per_task_ms": 102570.99054373523, "avg_output_tokens_per_task": 7925.6973995271865, "last_run_timestamp": "2026-09-20T10:30:07.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.7061538717948719, "accuracy_std_dev": 0.03305471857947331, "run_count": 39, "total_questions": 1950, "avg_cost_per_task_usd": 0.024778430256410244, "avg_duration_per_task_ms": 140799.78256410256, "avg_output_tokens_per_task": 6649.691794871795, "last_run_timestamp": "2026-08-29T16:54:26.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7055556666666667, "accuracy_std_dev": 0.03430073605462263, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.038495376666666664, "avg_duration_per_task_ms": 114809.37333333334, "avg_output_tokens_per_task": 5174.493333333333, "last_run_timestamp": "2026-09-01T17:31:46.866Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.7048717692307692, "accuracy_std_dev": 0.05145929153321672, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.03895267076923078, "avg_duration_per_task_ms": 181504.9023076923, "avg_output_tokens_per_task": 5031.964615384615, "last_run_timestamp": "2026-08-29T16:33:36.567Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.7046486190476191, "accuracy_std_dev": 0.037923417525396554, "run_count": 6, "total_questions": 294, "avg_cost_per_task_usd": 0.11717974149659866, "avg_duration_per_task_ms": 474565.40476190473, "avg_output_tokens_per_task": 22901.343537414967, "last_run_timestamp": "2026-08-22T08:19:48.101Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.704, "accuracy_std_dev": 0.02370180490173691, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.008193151999999999, "avg_duration_per_task_ms": 308148.616, "avg_output_tokens_per_task": 14154.342, "last_run_timestamp": "2026-09-19T08:01:55.621Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.7033335, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00633007, "avg_duration_per_task_ms": 346835.76, "avg_output_tokens_per_task": 4243.93, "last_run_timestamp": "2026-09-19T08:15:35.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Inceptron", "accuracy": 0.7033333333333334, "accuracy_std_dev": 0.04955356249106169, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02144292, "avg_duration_per_task_ms": 65470.65, "avg_output_tokens_per_task": 3098.5466666666666, "last_run_timestamp": "2026-05-02T08:19:29.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7033330000000001, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03169822, "avg_duration_per_task_ms": 206845.29, "avg_output_tokens_per_task": 16212.42, "last_run_timestamp": "2026-09-19T08:09:55.413Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.4077153666666667, "avg_duration_per_task_ms": 153641.3, "avg_output_tokens_per_task": 23369.493333333332, "last_run_timestamp": "2026-09-01T17:32:03.083Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.70166675, "accuracy_std_dev": 0.03507946138679295, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.155233565, "avg_duration_per_task_ms": 432057.795, "avg_output_tokens_per_task": 9028.84, "last_run_timestamp": "2026-09-19T08:03:46.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Parasail", "accuracy": 0.7016666666666667, "accuracy_std_dev": 0.045061686115319244, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.022215448333333332, "avg_duration_per_task_ms": 72920.13666666667, "avg_output_tokens_per_task": 3012.59, "last_run_timestamp": "2026-06-06T08:18:22.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "AtlasCloud", "accuracy": 0.7015590066815145, "accuracy_std_dev": 0.12728152373817306, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.049956171492204896, "avg_duration_per_task_ms": 149113.59020044544, "avg_output_tokens_per_task": 5756.837416481069, "last_run_timestamp": "2026-08-15T08:13:33.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Novita", "accuracy": 0.7008889333333334, "accuracy_std_dev": 0.03191351306362591, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.03097094533333333, "avg_duration_per_task_ms": 191571.11733333333, "avg_output_tokens_per_task": 4884.424, "last_run_timestamp": "2026-08-01T17:38:23.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.7, "accuracy_std_dev": 0.006667000000000034, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.030492239999999997, "avg_duration_per_task_ms": 126354.09, "avg_output_tokens_per_task": 20020.43, "last_run_timestamp": "2026-09-19T08:14:06.252Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Nebius", "accuracy": 0.7, "accuracy_std_dev": 0.03999999999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06787243, "avg_duration_per_task_ms": 42898.22, "avg_output_tokens_per_task": 4562.86, "last_run_timestamp": "2026-05-23T10:53:37.096Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "model_slug": "openai/gpt-5.2", "model_name": "OpenAI: GPT-5.2", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.31292506, "avg_duration_per_task_ms": 261751.42, "avg_output_tokens_per_task": 14948.8, "last_run_timestamp": "2026-08-01T08:06:08.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7, "accuracy_std_dev": 0.020547857418718854, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.11523170499999999, "avg_duration_per_task_ms": 699435.85, "avg_output_tokens_per_task": 20321.33, "last_run_timestamp": "2026-09-19T08:14:58.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.9826790200000001, "avg_duration_per_task_ms": 401624.28, "avg_output_tokens_per_task": 9843.34, "last_run_timestamp": "2026-08-26T00:00:04.572Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0112321, "avg_duration_per_task_ms": 596563.34, "avg_output_tokens_per_task": 4097.46, "last_run_timestamp": "2026-09-12T09:21:58.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.6995833125, "accuracy_std_dev": 0.02938897155975083, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.033650589999999994, "avg_duration_per_task_ms": 154427.47, "avg_output_tokens_per_task": 6103.86125, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.6993176578947369, "accuracy_std_dev": 0.09243577928000629, "run_count": 7, "total_questions": 342, "avg_cost_per_task_usd": 0.02560508187134503, "avg_duration_per_task_ms": 1176961.7573099416, "avg_output_tokens_per_task": 38128.90935672515, "last_run_timestamp": "2026-09-12T16:00:20.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.69833325, "accuracy_std_dev": 0.009859949857250788, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.14240881, "avg_duration_per_task_ms": 514302.935, "avg_output_tokens_per_task": 58927.42, "last_run_timestamp": "2026-09-01T17:31:48.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6975208464025869, "accuracy_std_dev": 0.06687846100100092, "run_count": 25, "total_questions": 1237, "avg_cost_per_task_usd": 0.09316261762328211, "avg_duration_per_task_ms": 251175.84963621665, "avg_output_tokens_per_task": 6799.790622473727, "last_run_timestamp": "2026-08-29T16:44:17.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.696923076923077, "accuracy_std_dev": 0.07467836376443007, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.023498743076923078, "avg_duration_per_task_ms": 88729.55230769231, "avg_output_tokens_per_task": 6683.635384615384, "last_run_timestamp": "2026-09-19T16:00:28.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.6966669999999999, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12995234, "avg_duration_per_task_ms": 694377.26, "avg_output_tokens_per_task": 58360.09, "last_run_timestamp": "2026-09-01T17:23:14.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.69666675, "accuracy_std_dev": 0.019720195515955205, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.27175437, "avg_duration_per_task_ms": 364436.225, "avg_output_tokens_per_task": 35416.46, "last_run_timestamp": "2026-09-01T13:04:22.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.6958332500000001, "accuracy_std_dev": 0.010374963189693743, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.042198395, "avg_duration_per_task_ms": 172150.8225, "avg_output_tokens_per_task": 5272.2875, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Morph", "accuracy": 0.6947618571428572, "accuracy_std_dev": 0.029726664088604592, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.03172098571428572, "avg_duration_per_task_ms": 126873.46571428572, "avg_output_tokens_per_task": 4470.107142857143, "last_run_timestamp": "2026-07-18T08:04:08.785Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.6941175882352941, "accuracy_std_dev": 0.04089728697036432, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.04082929647058823, "avg_duration_per_task_ms": 422759.3047058823, "avg_output_tokens_per_task": 12444.558823529413, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Chutes", "accuracy": 0.693333375, "accuracy_std_dev": 0.1814600292688017, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.042266376249999994, "avg_duration_per_task_ms": 100861.29625, "avg_output_tokens_per_task": 3413.32875, "last_run_timestamp": "2026-07-04T16:00:23.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Ionstream", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015592, "avg_duration_per_task_ms": 1084747.56, "avg_output_tokens_per_task": 17745.48, "last_run_timestamp": "2026-07-31T03:44:20.825Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "stepfun/step-3.7-flash-20260528", "model_slug": "stepfun/step-3.7-flash", "model_name": "StepFun: Step 3.7 Flash", "provider_name": "Ambient", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02187568, "avg_duration_per_task_ms": 236451.76, "avg_output_tokens_per_task": 11760.74, "last_run_timestamp": "2026-07-18T08:04:03.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "SiliconFlow", "accuracy": 0.692, "accuracy_std_dev": 0.04489988864128728, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046083804, "avg_duration_per_task_ms": 295008.8, "avg_output_tokens_per_task": 3260.444, "last_run_timestamp": "2026-05-02T08:48:58.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.6915385, "accuracy_std_dev": 0.08956798464005299, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.044391347692307695, "avg_duration_per_task_ms": 193285.48538461537, "avg_output_tokens_per_task": 6191.080769230769, "last_run_timestamp": "2026-08-29T17:03:20.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "SiliconFlow", "accuracy": 0.6907936666666666, "accuracy_std_dev": 0.03223436396490896, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.023469611428571423, "avg_duration_per_task_ms": 207909.9838095238, "avg_output_tokens_per_task": 4966.9914285714285, "last_run_timestamp": "2026-08-01T17:38:21.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Z.AI", "accuracy": 0.6904283835341366, "accuracy_std_dev": 0.027558639980730167, "run_count": 20, "total_questions": 996, "avg_cost_per_task_usd": 0.012428607429718873, "avg_duration_per_task_ms": 150833.32128514056, "avg_output_tokens_per_task": 5172.266064257028, "last_run_timestamp": "2026-08-01T17:38:51.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.6901754210526316, "accuracy_std_dev": 0.039015354444811766, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.014771705263157893, "avg_duration_per_task_ms": 199050.82631578948, "avg_output_tokens_per_task": 12000.06, "last_run_timestamp": "2026-08-29T17:51:56.583Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.6901235111111111, "accuracy_std_dev": 0.05518567527960597, "run_count": 11, "total_questions": 540, "avg_cost_per_task_usd": 0.026533475925925927, "avg_duration_per_task_ms": 754731.6203703703, "avg_output_tokens_per_task": 34989.23888888889, "last_run_timestamp": "2026-09-19T16:00:28.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.69, "accuracy_std_dev": 0.032088096328701056, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.18925239666666666, "avg_duration_per_task_ms": 1912773.68, "avg_output_tokens_per_task": 13075.366666666667, "last_run_timestamp": "2026-08-29T16:50:22.450Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "AtlasCloud", "accuracy": 0.69, "accuracy_std_dev": 0.04013864859597431, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.04868145222222221, "avg_duration_per_task_ms": 105751.87222222223, "avg_output_tokens_per_task": 3340.511111111111, "last_run_timestamp": "2026-06-20T12:59:17.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Io Net", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01987944, "avg_duration_per_task_ms": 47600.21, "avg_output_tokens_per_task": 2953.26, "last_run_timestamp": "2026-04-18T11:46:57.679Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "Xiaomi", "accuracy": 0.69, "accuracy_std_dev": 0.040532174168888865, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.005806647142857143, "avg_duration_per_task_ms": 77101.00428571428, "avg_output_tokens_per_task": 3994.6614285714286, "last_run_timestamp": "2026-06-13T08:42:56.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Io Net", "accuracy": 0.6899998333333334, "accuracy_std_dev": 0.03415651881860261, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09539628666666666, "avg_duration_per_task_ms": 502649.19, "avg_output_tokens_per_task": 22095.93666666667, "last_run_timestamp": "2026-08-15T08:15:24.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.6896024357798165, "accuracy_std_dev": 0.12893777217195382, "run_count": 9, "total_questions": 436, "avg_cost_per_task_usd": 0.05226660550458716, "avg_duration_per_task_ms": 99364.6376146789, "avg_output_tokens_per_task": 6918.672018348624, "last_run_timestamp": "2026-09-19T08:17:29.815Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.6895207458193979, "accuracy_std_dev": 0.06906980435656501, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.10185452675585285, "avg_duration_per_task_ms": 74719.04347826086, "avg_output_tokens_per_task": 7900.401337792642, "last_run_timestamp": "2026-09-19T08:15:19.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.689041575052854, "accuracy_std_dev": 0.03094305825454303, "run_count": 19, "total_questions": 946, "avg_cost_per_task_usd": 0.021175065539112053, "avg_duration_per_task_ms": 438783.39852008456, "avg_output_tokens_per_task": 11967.172304439746, "last_run_timestamp": "2026-08-29T17:42:21.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "auto-routing", "accuracy": 0.6882603594890511, "accuracy_std_dev": 0.11042330833399057, "run_count": 22, "total_questions": 1096, "avg_cost_per_task_usd": 0.04567830748175184, "avg_duration_per_task_ms": 105376.38777372263, "avg_output_tokens_per_task": 3958.9105839416056, "last_run_timestamp": "2026-08-01T17:38:27.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6878786363636364, "accuracy_std_dev": 0.02442357319280149, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11701100909090908, "avg_duration_per_task_ms": 78384.59636363636, "avg_output_tokens_per_task": 9374.68909090909, "last_run_timestamp": "2026-09-17T02:10:06.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Novita", "accuracy": 0.6878611676300578, "accuracy_std_dev": 0.042049584814859244, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.013951661849710981, "avg_duration_per_task_ms": 210919.60693641618, "avg_output_tokens_per_task": 8494.329479768787, "last_run_timestamp": "2026-08-01T17:38:31.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.6875626118355065, "accuracy_std_dev": 0.0373234995892668, "run_count": 20, "total_questions": 997, "avg_cost_per_task_usd": 0.03210182146439318, "avg_duration_per_task_ms": 179885.19558676027, "avg_output_tokens_per_task": 7259.881644934804, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6866663333333333, "accuracy_std_dev": 0.024944382578492918, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013524920000000001, "avg_duration_per_task_ms": 73418.06, "avg_output_tokens_per_task": 5390.6866666666665, "last_run_timestamp": "2026-09-01T17:31:44.289Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.6863945836734694, "accuracy_std_dev": 0.10682812745363461, "run_count": 10, "total_questions": 490, "avg_cost_per_task_usd": 0.13472426326530615, "avg_duration_per_task_ms": 872956.7653061225, "avg_output_tokens_per_task": 33505.848979591836, "last_run_timestamp": "2026-09-19T16:00:28.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.6857144285714285, "accuracy_std_dev": 0.042077516201333307, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.030182771428571425, "avg_duration_per_task_ms": 323904.1228571429, "avg_output_tokens_per_task": 10162.908571428572, "last_run_timestamp": "2026-08-29T16:50:26.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.685699512345679, "accuracy_std_dev": 0.13961279039785202, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.05491914969135803, "avg_duration_per_task_ms": 390942.3487654321, "avg_output_tokens_per_task": 6689.006172839506, "last_run_timestamp": "2026-09-19T08:21:42.831Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.6842424545454545, "accuracy_std_dev": 0.03293989573412772, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07802852727272729, "avg_duration_per_task_ms": 647259.4745454545, "avg_output_tokens_per_task": 16952.354545454546, "last_run_timestamp": "2026-09-19T08:02:29.964Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Io Net", "accuracy": 0.684, "accuracy_std_dev": 0.02939387691339811, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.011617532, "avg_duration_per_task_ms": 184984.468, "avg_output_tokens_per_task": 3721.176, "last_run_timestamp": "2026-06-13T09:50:28.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.683673530612245, "accuracy_std_dev": 0.03777750000000002, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.6821915918367347, "avg_duration_per_task_ms": 803506.224489796, "avg_output_tokens_per_task": 61161.97959183674, "last_run_timestamp": "2026-09-01T12:56:59.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.6836363636363636, "accuracy_std_dev": 0.09197555610128431, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.032429353636363636, "avg_duration_per_task_ms": 279572.37363636366, "avg_output_tokens_per_task": 9297.255454545455, "last_run_timestamp": "2026-08-29T17:02:01.052Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6826308620689655, "accuracy_std_dev": 0.039302124463935585, "run_count": 21, "total_questions": 1044, "avg_cost_per_task_usd": 0.01650072318007663, "avg_duration_per_task_ms": 286180.8256704981, "avg_output_tokens_per_task": 13045.386973180077, "last_run_timestamp": "2026-08-29T16:12:10.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AkashML", "accuracy": 0.6816424057971014, "accuracy_std_dev": 0.18410546186501534, "run_count": 7, "total_questions": 345, "avg_cost_per_task_usd": 0.10899141739130434, "avg_duration_per_task_ms": 177787.73623188405, "avg_output_tokens_per_task": 6552.942028985507, "last_run_timestamp": "2026-08-15T08:10:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "auto-routing", "accuracy": 0.6814285714285714, "accuracy_std_dev": 0.06300307734628557, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.005582074285714287, "avg_duration_per_task_ms": 43375.77857142857, "avg_output_tokens_per_task": 3572.1442857142856, "last_run_timestamp": "2026-06-13T09:16:25.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.6813061126126125, "accuracy_std_dev": 0.17362449557921197, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.04568808558558559, "avg_duration_per_task_ms": 182111.46846846846, "avg_output_tokens_per_task": 5794.326576576576, "last_run_timestamp": "2026-08-15T08:17:17.868Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.6811110666666667, "accuracy_std_dev": 0.12911485116049026, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.1191761946666667, "avg_duration_per_task_ms": 130000.048, "avg_output_tokens_per_task": 7795.806666666666, "last_run_timestamp": "2026-09-19T08:15:16.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.6808227067307693, "accuracy_std_dev": 0.12201399623797919, "run_count": 25, "total_questions": 1248, "avg_cost_per_task_usd": 0.040376496794871784, "avg_duration_per_task_ms": 109025.02884615384, "avg_output_tokens_per_task": 5601.289262820513, "last_run_timestamp": "2026-08-29T17:03:33.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.6805555, "accuracy_std_dev": 0.16692092143412302, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.052043624999999996, "avg_duration_per_task_ms": 145763.23666666666, "avg_output_tokens_per_task": 17883.305, "last_run_timestamp": "2026-09-19T16:00:30.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "AtlasCloud", "accuracy": 0.6803882349397591, "accuracy_std_dev": 0.08651381511485608, "run_count": 20, "total_questions": 996, "avg_cost_per_task_usd": 0.04140781124497992, "avg_duration_per_task_ms": 212525.218875502, "avg_output_tokens_per_task": 6209.691767068273, "last_run_timestamp": "2026-08-01T17:38:10.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33301777, "avg_duration_per_task_ms": 171524.99, "avg_output_tokens_per_task": 8948.71, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.036475640000000004, "avg_duration_per_task_ms": 622815.32, "avg_output_tokens_per_task": 34740.02, "last_run_timestamp": "2026-09-16T09:00:02.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Decart", "accuracy": 0.68, "accuracy_std_dev": 0.03999999999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.028420829999999998, "avg_duration_per_task_ms": 35643.83, "avg_output_tokens_per_task": 3017.33, "last_run_timestamp": "2026-06-20T09:16:42.673Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.68, "accuracy_std_dev": 0, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.088178175, "avg_duration_per_task_ms": 74028.285, "avg_output_tokens_per_task": 3643.675, "last_run_timestamp": "2026-09-19T08:08:39.705Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06724904000000001, "avg_duration_per_task_ms": 102008.58, "avg_output_tokens_per_task": 2510.66, "last_run_timestamp": "2026-08-22T08:23:03.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15835572, "avg_duration_per_task_ms": 627337.14, "avg_output_tokens_per_task": 4929.46, "last_run_timestamp": "2026-09-14T09:00:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.6788891666666665, "accuracy_std_dev": 0.08738317986587706, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.057002019999999994, "avg_duration_per_task_ms": 515207.3233333333, "avg_output_tokens_per_task": 18064.933333333334, "last_run_timestamp": "2026-08-29T17:31:24.151Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "SambaNova", "accuracy": 0.6783625714285715, "accuracy_std_dev": 0.18503792379476175, "run_count": 24, "total_questions": 1197, "avg_cost_per_task_usd": 0.06683292147034252, "avg_duration_per_task_ms": 46030.891395154555, "avg_output_tokens_per_task": 5800.272347535505, "last_run_timestamp": "2026-08-15T14:12:46.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "Novita", "accuracy": 0.6783333, "accuracy_std_dev": 0.029354164951672524, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.049139350000000005, "avg_duration_per_task_ms": 440231.73, "avg_output_tokens_per_task": 6660.06, "last_run_timestamp": "2026-07-18T16:00:08.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "model_slug": "deepseek/deepseek-v3.2-exp", "model_name": "DeepSeek: DeepSeek V3.2 Exp", "provider_name": "auto-routing", "accuracy": 0.6779184535864979, "accuracy_std_dev": 0.04493753781451566, "run_count": 19, "total_questions": 948, "avg_cost_per_task_usd": 0.05880673312236288, "avg_duration_per_task_ms": 487546.12974683545, "avg_output_tokens_per_task": 9480.43670886076, "last_run_timestamp": "2026-08-01T17:38:07.305Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.6778149866220735, "accuracy_std_dev": 0.13741656972108465, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.04846960535117057, "avg_duration_per_task_ms": 155359.52173913043, "avg_output_tokens_per_task": 7194.195652173913, "last_run_timestamp": "2026-09-19T08:15:15.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.6766999163879599, "accuracy_std_dev": 0.167303531565863, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.06077755685618729, "avg_duration_per_task_ms": 163271.508361204, "avg_output_tokens_per_task": 6916.148829431439, "last_run_timestamp": "2026-09-19T08:15:01.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.6751515454545455, "accuracy_std_dev": 0.21232880724838227, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.044593925454545456, "avg_duration_per_task_ms": 85905.94363636363, "avg_output_tokens_per_task": 3735.2381818181816, "last_run_timestamp": "2026-06-30T19:07:40.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.6742424671717172, "accuracy_std_dev": 0.0476536543875113, "run_count": 16, "total_questions": 792, "avg_cost_per_task_usd": 0.017849546717171715, "avg_duration_per_task_ms": 164206.20707070708, "avg_output_tokens_per_task": 12386.181818181818, "last_run_timestamp": "2026-08-29T17:44:03.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.6734920476190476, "accuracy_std_dev": 0.06836312161606557, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.024787600000000003, "avg_duration_per_task_ms": 231263.06666666668, "avg_output_tokens_per_task": 6580.758095238095, "last_run_timestamp": "2026-08-22T16:09:51.985Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.6733836574307305, "accuracy_std_dev": 0.036202263205970635, "run_count": 16, "total_questions": 794, "avg_cost_per_task_usd": 0.011785144836272042, "avg_duration_per_task_ms": 140094.4420654912, "avg_output_tokens_per_task": 7374.278337531486, "last_run_timestamp": "2026-08-30T00:01:09.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Baidu", "accuracy": 0.6733333333333333, "accuracy_std_dev": 0.0524933858267454, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.009767394999999998, "avg_duration_per_task_ms": 49065.985, "avg_output_tokens_per_task": 3511.735, "last_run_timestamp": "2026-06-06T17:11:08.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1116277, "avg_duration_per_task_ms": 79582.44, "avg_output_tokens_per_task": 4280.42, "last_run_timestamp": "2026-08-01T08:05:57.504Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "model_slug": "moonshotai/kimi-k2-thinking", "model_name": "MoonshotAI: Kimi K2 Thinking", "provider_name": "Google", "accuracy": 0.6731745714285714, "accuracy_std_dev": 0.04441834647612931, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.07005776761904764, "avg_duration_per_task_ms": 72619.98952380952, "avg_output_tokens_per_task": 4610.935238095238, "last_run_timestamp": "2026-08-01T17:38:46.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.670833375, "accuracy_std_dev": 0.061131717422581666, "run_count": 40, "total_questions": 2000, "avg_cost_per_task_usd": 0.023635378499999995, "avg_duration_per_task_ms": 110968.634, "avg_output_tokens_per_task": 7070.978, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.6700102376543211, "accuracy_std_dev": 0.10496553242878909, "run_count": 13, "total_questions": 648, "avg_cost_per_task_usd": 0.027610222222222223, "avg_duration_per_task_ms": 162024.1913580247, "avg_output_tokens_per_task": 7429.558641975309, "last_run_timestamp": "2026-08-29T16:15:12.662Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.67, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03314721, "avg_duration_per_task_ms": 151834.99, "avg_output_tokens_per_task": 17524.36, "last_run_timestamp": "2026-09-01T17:29:39.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.67, "accuracy_std_dev": 0.031446692107437925, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.07852493499999999, "avg_duration_per_task_ms": 68196.455, "avg_output_tokens_per_task": 3640.76, "last_run_timestamp": "2026-09-19T08:12:48.272Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DigitalOcean", "accuracy": 0.67, "accuracy_std_dev": 0.02999999999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014700359999999999, "avg_duration_per_task_ms": 1773933.08, "avg_output_tokens_per_task": 23989.78, "last_run_timestamp": "2026-07-18T08:04:08.007Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.6695237142857143, "accuracy_std_dev": 0.036663425955241954, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11802848000000002, "avg_duration_per_task_ms": 120456.28, "avg_output_tokens_per_task": 12635.617142857143, "last_run_timestamp": "2026-09-19T08:12:38.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Minimax", "accuracy": 0.6692929393939393, "accuracy_std_dev": 0.05867076717192114, "run_count": 33, "total_questions": 1650, "avg_cost_per_task_usd": 0.01908582484848485, "avg_duration_per_task_ms": 102657.98787878788, "avg_output_tokens_per_task": 6209.364848484848, "last_run_timestamp": "2026-08-01T17:38:12.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.6675946726057906, "accuracy_std_dev": 0.062223605171164056, "run_count": 18, "total_questions": 898, "avg_cost_per_task_usd": 0.02003099443207127, "avg_duration_per_task_ms": 278955.9331848552, "avg_output_tokens_per_task": 10682.610244988864, "last_run_timestamp": "2026-08-29T16:44:07.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Together", "accuracy": 0.6672268756302522, "accuracy_std_dev": 0.19692376950802565, "run_count": 12, "total_questions": 595, "avg_cost_per_task_usd": 0.04049237815126051, "avg_duration_per_task_ms": 57680.32773109244, "avg_output_tokens_per_task": 4304.621848739495, "last_run_timestamp": "2026-08-04T03:12:18.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.6666665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11778575999999999, "avg_duration_per_task_ms": 148387.42, "avg_output_tokens_per_task": 58241.15, "last_run_timestamp": "2026-09-01T17:32:23.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.6655251643835617, "accuracy_std_dev": 0.050919510401766024, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.0655772808219178, "avg_duration_per_task_ms": 50910.90410958904, "avg_output_tokens_per_task": 3110.958904109589, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.6652707604690117, "accuracy_std_dev": 0.19745415809175107, "run_count": 12, "total_questions": 597, "avg_cost_per_task_usd": 0.09212694807370185, "avg_duration_per_task_ms": 267596.5443886097, "avg_output_tokens_per_task": 7550.497487437186, "last_run_timestamp": "2026-09-19T08:12:06.923Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6646666, "accuracy_std_dev": 0.03190959382129456, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.12408255800000001, "avg_duration_per_task_ms": 116498.477, "avg_output_tokens_per_task": 12443.676, "last_run_timestamp": "2026-09-19T08:14:21.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "auto-routing", "accuracy": 0.6644444166666666, "accuracy_std_dev": 0.036497949376091285, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.014842355833333333, "avg_duration_per_task_ms": 83587.3425, "avg_output_tokens_per_task": 5542.576666666667, "last_run_timestamp": "2026-08-01T17:38:10.364Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Z.AI", "accuracy": 0.6633333333333333, "accuracy_std_dev": 0.046785562825394, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.006290366666666667, "avg_duration_per_task_ms": 170099.27333333335, "avg_output_tokens_per_task": 5352.633333333333, "last_run_timestamp": "2026-05-09T21:59:21.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.6631578421052632, "accuracy_std_dev": 0.03436091497198395, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.016520328421052635, "avg_duration_per_task_ms": 162137.86210526316, "avg_output_tokens_per_task": 6763.676842105263, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6622221666666667, "accuracy_std_dev": 0.0198761101108564, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.31710583, "avg_duration_per_task_ms": 192864.10333333333, "avg_output_tokens_per_task": 8814.376666666667, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.6614287142857143, "accuracy_std_dev": 0.021518416940673253, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11292796, "avg_duration_per_task_ms": 122765.54, "avg_output_tokens_per_task": 12047.011428571428, "last_run_timestamp": "2026-09-19T08:19:00.786Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Phala", "accuracy": 0.6611111111111111, "accuracy_std_dev": 0.02865288212939472, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.00995818888888889, "avg_duration_per_task_ms": 85091.12666666666, "avg_output_tokens_per_task": 4232.724444444444, "last_run_timestamp": "2026-06-20T21:23:09.799Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.6611110833333333, "accuracy_std_dev": 0.01760328077120064, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03028098166666666, "avg_duration_per_task_ms": 310414.11, "avg_output_tokens_per_task": 9537.056666666667, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0482247, "avg_duration_per_task_ms": 1006365.36, "avg_output_tokens_per_task": 46674.68, "last_run_timestamp": "2026-09-19T08:08:42.462Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Inceptron", "accuracy": 0.6573610416666666, "accuracy_std_dev": 0.035732853578650324, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.013916276666666665, "avg_duration_per_task_ms": 148914.81333333332, "avg_output_tokens_per_task": 6375.918333333333, "last_run_timestamp": "2026-08-15T14:15:19.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.6573334, "accuracy_std_dev": 0.2126070154586626, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.04436226000000001, "avg_duration_per_task_ms": 85213.446, "avg_output_tokens_per_task": 3863.376, "last_run_timestamp": "2026-08-04T03:12:10.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.6571529281183933, "accuracy_std_dev": 0.05360818932369472, "run_count": 19, "total_questions": 946, "avg_cost_per_task_usd": 0.02660617758985201, "avg_duration_per_task_ms": 237280.09619450316, "avg_output_tokens_per_task": 13635.822410147992, "last_run_timestamp": "2026-08-29T16:50:48.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.6566665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.017312400000000002, "avg_duration_per_task_ms": 126237.63, "avg_output_tokens_per_task": 21774.52, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6565315, "accuracy_std_dev": 0.011837568509716087, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.014823912162162164, "avg_duration_per_task_ms": 271525.86486486485, "avg_output_tokens_per_task": 19985.412162162163, "last_run_timestamp": "2026-09-19T08:01:36.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.6554808926174497, "accuracy_std_dev": 0.13732673855009286, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.04849104697986577, "avg_duration_per_task_ms": 2325789.067114094, "avg_output_tokens_per_task": 67477.9798657718, "last_run_timestamp": "2026-09-19T08:07:37.370Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.655102163265306, "accuracy_std_dev": 0.015608943341559031, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.012813330612244898, "avg_duration_per_task_ms": 278688.9183673469, "avg_output_tokens_per_task": 16201.359183673469, "last_run_timestamp": "2026-08-29T16:33:39.228Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.6548717692307693, "accuracy_std_dev": 0.04499981444945309, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.01632250307692308, "avg_duration_per_task_ms": 92656.01, "avg_output_tokens_per_task": 6382.130769230769, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Inceptron", "accuracy": 0.6546666, "accuracy_std_dev": 0.14424683227800877, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.041911724, "avg_duration_per_task_ms": 194571.3, "avg_output_tokens_per_task": 4511.004, "last_run_timestamp": "2026-06-30T19:22:10.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.6542856190476191, "accuracy_std_dev": 0.030948730921709834, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.12287592761904761, "avg_duration_per_task_ms": 119817.67714285714, "avg_output_tokens_per_task": 12557.709523809524, "last_run_timestamp": "2026-09-19T08:16:41.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.652778, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.048472999999999995, "avg_duration_per_task_ms": 691836.3541666666, "avg_output_tokens_per_task": 18257.291666666668, "last_run_timestamp": "2026-08-21T21:00:03.652Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.6518518333333333, "accuracy_std_dev": 0.03388838939199286, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.026386046666666666, "avg_duration_per_task_ms": 174668.08444444445, "avg_output_tokens_per_task": 7625.071111111111, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.65166675, "accuracy_std_dev": 0.04506177549972369, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04474220500000001, "avg_duration_per_task_ms": 212693.005, "avg_output_tokens_per_task": 3236.785, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.6512820769230769, "accuracy_std_dev": 0.0493641089572974, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.01929119692307693, "avg_duration_per_task_ms": 131140.2476923077, "avg_output_tokens_per_task": 6527.093076923077, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.6506172962962963, "accuracy_std_dev": 0.04126635072919341, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.006890773333333334, "avg_duration_per_task_ms": 195510.75185185185, "avg_output_tokens_per_task": 7239.452592592593, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Together", "accuracy": 0.65, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.013433730000000001, "avg_duration_per_task_ms": 30166.39, "avg_output_tokens_per_task": 3575.99, "last_run_timestamp": "2026-04-25T08:11:30.451Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.6485714285714286, "accuracy_std_dev": 0.061434100526748275, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.01923216428571428, "avg_duration_per_task_ms": 162362.36428571428, "avg_output_tokens_per_task": 6530.884285714285, "last_run_timestamp": "2026-08-29T16:37:21.285Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.6481482020202021, "accuracy_std_dev": 0.0054059166891471805, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.009114631313131312, "avg_duration_per_task_ms": 604721.6666666666, "avg_output_tokens_per_task": 19746.29292929293, "last_run_timestamp": "2026-08-22T16:10:30.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.6466666818181818, "accuracy_std_dev": 0.11090082019181345, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.03871445727272728, "avg_duration_per_task_ms": 229660.09454545454, "avg_output_tokens_per_task": 6490.092727272728, "last_run_timestamp": "2026-08-29T16:11:23.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.6460429376257546, "accuracy_std_dev": 0.1396968497913965, "run_count": 20, "total_questions": 994, "avg_cost_per_task_usd": 0.013013100603621729, "avg_duration_per_task_ms": 399798.5160965795, "avg_output_tokens_per_task": 11031.460764587526, "last_run_timestamp": "2026-08-29T16:48:06.159Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.6457142857142857, "accuracy_std_dev": 0.02700476652484195, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11825372571428569, "avg_duration_per_task_ms": 150221.76285714286, "avg_output_tokens_per_task": 12779.42857142857, "last_run_timestamp": "2026-09-19T08:12:02.575Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.644781292929293, "accuracy_std_dev": 0.015373999999999999, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.018105767676767675, "avg_duration_per_task_ms": 185321.31313131313, "avg_output_tokens_per_task": 22168.010101010103, "last_run_timestamp": "2026-09-19T08:02:40.719Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DeepInfra", "accuracy": 0.6443191729323308, "accuracy_std_dev": 0.049327665162253266, "run_count": 16, "total_questions": 798, "avg_cost_per_task_usd": 0.01105938471177945, "avg_duration_per_task_ms": 153875.62656641603, "avg_output_tokens_per_task": 4867.625313283208, "last_run_timestamp": "2026-07-11T15:34:15.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.6438461923076922, "accuracy_std_dev": 0.04265441692704396, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.01644152, "avg_duration_per_task_ms": 177000.07846153845, "avg_output_tokens_per_task": 6952.727692307692, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.6433335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.034682500000000005, "avg_duration_per_task_ms": 159195.12, "avg_output_tokens_per_task": 18151.16, "last_run_timestamp": "2026-09-01T17:31:43.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.6433333333333333, "accuracy_std_dev": 0.14455548467015086, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10531282, "avg_duration_per_task_ms": 1453976.54, "avg_output_tokens_per_task": 18878.353333333333, "last_run_timestamp": "2026-08-08T08:21:31.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "WandB", "accuracy": 0.6427566687763713, "accuracy_std_dev": 0.029051197700409103, "run_count": 19, "total_questions": 948, "avg_cost_per_task_usd": 0.01150348523206751, "avg_duration_per_task_ms": 156638.9229957806, "avg_output_tokens_per_task": 9733.639240506329, "last_run_timestamp": "2026-07-18T16:00:10.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "DeepInfra", "accuracy": 0.6422413563218391, "accuracy_std_dev": 0.06412982505652585, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.02831189511494253, "avg_duration_per_task_ms": 340448.8103448276, "avg_output_tokens_per_task": 10172.28735632184, "last_run_timestamp": "2026-08-08T16:03:43.735Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.64166675, "accuracy_std_dev": 0.00985994985725082, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0329834, "avg_duration_per_task_ms": 238262.615, "avg_output_tokens_per_task": 16910.1, "last_run_timestamp": "2026-09-01T17:31:47.440Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "Novita", "accuracy": 0.641666625, "accuracy_std_dev": 0.03575897388047335, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.014602430000000001, "avg_duration_per_task_ms": 80982.3175, "avg_output_tokens_per_task": 5601.455833333333, "last_run_timestamp": "2026-08-01T17:38:10.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6411111111111111, "accuracy_std_dev": 0.0300000329219984, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.015867753333333335, "avg_duration_per_task_ms": 224972.06444444443, "avg_output_tokens_per_task": 12173.745555555555, "last_run_timestamp": "2026-07-11T15:34:14.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AkashML", "accuracy": 0.640784294117647, "accuracy_std_dev": 0.08323160877777759, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.01927779294117647, "avg_duration_per_task_ms": 109439.3894117647, "avg_output_tokens_per_task": 3724.7023529411763, "last_run_timestamp": "2026-07-01T02:04:30.298Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SambaNova", "accuracy": 0.64, "accuracy_std_dev": 0.048989794855663564, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.029686422500000004, "avg_duration_per_task_ms": 51733.8325, "avg_output_tokens_per_task": 3684.0075, "last_run_timestamp": "2026-05-16T16:36:37.985Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.64, "accuracy_std_dev": 0.025647213408087847, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.218415728, "avg_duration_per_task_ms": 1219489.072, "avg_output_tokens_per_task": 31443.888, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.63984269527897, "accuracy_std_dev": 0.15007041362089646, "run_count": 28, "total_questions": 1398, "avg_cost_per_task_usd": 0.045527532904148786, "avg_duration_per_task_ms": 180007.91201716737, "avg_output_tokens_per_task": 5421.762517882689, "last_run_timestamp": "2026-08-29T17:36:31.188Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "WandB", "accuracy": 0.6396296111111112, "accuracy_std_dev": 0.04605268692021117, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.034697361111111116, "avg_duration_per_task_ms": 95669.66222222222, "avg_output_tokens_per_task": 4599.92, "last_run_timestamp": "2026-07-04T16:00:23.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.6387463811965812, "accuracy_std_dev": 0.07838394965714708, "run_count": 12, "total_questions": 585, "avg_cost_per_task_usd": 0.020067148717948718, "avg_duration_per_task_ms": 224214.09743589745, "avg_output_tokens_per_task": 22503.152136752138, "last_run_timestamp": "2026-09-19T16:00:28.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.6383547403846154, "accuracy_std_dev": 0.034423786918327294, "run_count": 25, "total_questions": 1248, "avg_cost_per_task_usd": 0.012904894230769234, "avg_duration_per_task_ms": 200720.20913461538, "avg_output_tokens_per_task": 13156.37580128205, "last_run_timestamp": "2026-08-29T16:25:26.695Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Mara", "accuracy": 0.637681, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.04568002173913044, "avg_duration_per_task_ms": 70889.17391304347, "avg_output_tokens_per_task": 8168.934782608696, "last_run_timestamp": "2026-08-01T17:41:39.392Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.6356409615384615, "accuracy_std_dev": 0.04264513255612286, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.01902188461538462, "avg_duration_per_task_ms": 148992.3046153846, "avg_output_tokens_per_task": 6265.466923076923, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.6354166666666666, "accuracy_std_dev": 0.07353388359834909, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.060891364999999996, "avg_duration_per_task_ms": 149518.805, "avg_output_tokens_per_task": 6065.214166666667, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.6339682380952381, "accuracy_std_dev": 0.05388220020687527, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.02847615904761905, "avg_duration_per_task_ms": 134848.23047619048, "avg_output_tokens_per_task": 4950.976190476191, "last_run_timestamp": "2026-07-18T16:00:10.793Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.6337365483870968, "accuracy_std_dev": 0.02579029640950258, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.016857699596774196, "avg_duration_per_task_ms": 45111.594758064515, "avg_output_tokens_per_task": 9635.451612903225, "last_run_timestamp": "2026-08-29T16:33:38.021Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6333335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02418533, "avg_duration_per_task_ms": 152335.58, "avg_output_tokens_per_task": 37821.34, "last_run_timestamp": "2026-09-19T08:06:26.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6333332500000001, "accuracy_std_dev": 0.03299835012826401, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.102073965, "avg_duration_per_task_ms": 160807.645, "avg_output_tokens_per_task": 12456.285, "last_run_timestamp": "2026-09-01T17:31:46.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6331093959731544, "accuracy_std_dev": 0.01453205911080741, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.012315053691275168, "avg_duration_per_task_ms": 1064929.322147651, "avg_output_tokens_per_task": 21067.724832214764, "last_run_timestamp": "2026-08-29T17:18:10.895Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6319998, "accuracy_std_dev": 0.0363685871784979, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.09467958400000001, "avg_duration_per_task_ms": 55836.916, "avg_output_tokens_per_task": 3550.436, "last_run_timestamp": "2026-08-29T16:15:31.797Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "CoreWeave", "accuracy": 0.63, "accuracy_std_dev": 0.023333000000000048, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07672618, "avg_duration_per_task_ms": 231628.06, "avg_output_tokens_per_task": 10580.58, "last_run_timestamp": "2026-08-01T17:40:48.165Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.63, "accuracy_std_dev": 0.050000000000000044, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09203941, "avg_duration_per_task_ms": 133131.18, "avg_output_tokens_per_task": 12511.49, "last_run_timestamp": "2026-09-01T17:23:15.998Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.6278446465863453, "accuracy_std_dev": 0.02683498380174657, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.03220683132530121, "avg_duration_per_task_ms": 87364.718875502, "avg_output_tokens_per_task": 22117.32730923695, "last_run_timestamp": "2026-08-29T17:17:57.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "AtlasCloud", "accuracy": 0.6251851666666667, "accuracy_std_dev": 0.04896739375935106, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.058607118888888886, "avg_duration_per_task_ms": 290047.4222222222, "avg_output_tokens_per_task": 13976.531111111111, "last_run_timestamp": "2026-07-01T03:04:05.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.6234782173913044, "accuracy_std_dev": 0.04417228647797023, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.008193006086956521, "avg_duration_per_task_ms": 187011.97043478262, "avg_output_tokens_per_task": 11024.112173913043, "last_run_timestamp": "2026-08-29T16:33:38.124Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.6230798605990784, "accuracy_std_dev": 0.04010606346417775, "run_count": 35, "total_questions": 1736, "avg_cost_per_task_usd": 0.014455639976958522, "avg_duration_per_task_ms": 224122.75806451612, "avg_output_tokens_per_task": 14008.978110599079, "last_run_timestamp": "2026-08-29T16:49:23.763Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6221364138996139, "accuracy_std_dev": 0.12792573087258097, "run_count": 26, "total_questions": 1295, "avg_cost_per_task_usd": 0.013064416216216216, "avg_duration_per_task_ms": 169022.18301158302, "avg_output_tokens_per_task": 5297.602316602317, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6214286071428571, "accuracy_std_dev": 0.14501615904180656, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.028539994285714288, "avg_duration_per_task_ms": 185000.78214285715, "avg_output_tokens_per_task": 5080.563571428572, "last_run_timestamp": "2026-08-30T00:01:10.585Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6203515192697769, "accuracy_std_dev": 0.2303888022849418, "run_count": 10, "total_questions": 493, "avg_cost_per_task_usd": 0.020992683569979714, "avg_duration_per_task_ms": 345295.43204868154, "avg_output_tokens_per_task": 11072.965517241379, "last_run_timestamp": "2026-09-19T08:18:35.767Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.620325766025641, "accuracy_std_dev": 0.05438397244850727, "run_count": 25, "total_questions": 1248, "avg_cost_per_task_usd": 0.03392773717948717, "avg_duration_per_task_ms": 185250.95272435897, "avg_output_tokens_per_task": 5975.309294871795, "last_run_timestamp": "2026-08-22T16:05:40.004Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.620246288590604, "accuracy_std_dev": 0.024043751131473293, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.19689856375838927, "avg_duration_per_task_ms": 175265.28187919463, "avg_output_tokens_per_task": 13638.724832214764, "last_run_timestamp": "2026-09-01T13:00:41.540Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.6200679918367347, "accuracy_std_dev": 0.08681822122573118, "run_count": 10, "total_questions": 490, "avg_cost_per_task_usd": 0.017946885714285717, "avg_duration_per_task_ms": 578490.112244898, "avg_output_tokens_per_task": 21616.84081632653, "last_run_timestamp": "2026-09-19T16:00:30.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "model_slug": "z-ai/glm-4.5-air", "model_name": "Z.ai: GLM 4.5 Air", "provider_name": "Ambient", "accuracy": 0.62, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01353782, "avg_duration_per_task_ms": 89487.42, "avg_output_tokens_per_task": 4959.24, "last_run_timestamp": "2026-06-13T15:23:43.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.6191536436525612, "accuracy_std_dev": 0.050396043955175315, "run_count": 27, "total_questions": 1347, "avg_cost_per_task_usd": 0.007358628062360802, "avg_duration_per_task_ms": 148190.81885671863, "avg_output_tokens_per_task": 7990.572383073497, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.6176811739130434, "accuracy_std_dev": 0.043985459303543334, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.05628466086956523, "avg_duration_per_task_ms": 389421.3947826087, "avg_output_tokens_per_task": 19986.14956521739, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.6176750230547552, "accuracy_std_dev": 0.22454644005875757, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.04794541498559077, "avg_duration_per_task_ms": 857866.4149855907, "avg_output_tokens_per_task": 11712.968299711816, "last_run_timestamp": "2026-08-29T17:09:01.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.6175489270386266, "accuracy_std_dev": 0.06455018061760406, "run_count": 28, "total_questions": 1398, "avg_cost_per_task_usd": 0.008319118025751073, "avg_duration_per_task_ms": 160872.3583690987, "avg_output_tokens_per_task": 8106.090128755365, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6159998000000001, "accuracy_std_dev": 0.03440923906976152, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.045486667999999994, "avg_duration_per_task_ms": 195084.972, "avg_output_tokens_per_task": 3358.316, "last_run_timestamp": "2026-09-19T08:08:42.956Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Parasail", "accuracy": 0.6156657788369876, "accuracy_std_dev": 0.06881313004209644, "run_count": 21, "total_questions": 1049, "avg_cost_per_task_usd": 0.018457796949475698, "avg_duration_per_task_ms": 105384.89418493804, "avg_output_tokens_per_task": 8663.15061963775, "last_run_timestamp": "2026-08-01T17:40:52.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.6156620170015455, "accuracy_std_dev": 0.09743333851751483, "run_count": 26, "total_questions": 1294, "avg_cost_per_task_usd": 0.05918442503863987, "avg_duration_per_task_ms": 375639.1128284389, "avg_output_tokens_per_task": 9550.46367851623, "last_run_timestamp": "2026-08-30T00:01:02.157Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.6154540773993807, "accuracy_std_dev": 0.05444941500976228, "run_count": 26, "total_questions": 1292, "avg_cost_per_task_usd": 0.025835168730650157, "avg_duration_per_task_ms": 55520.85758513932, "avg_output_tokens_per_task": 14734.878482972135, "last_run_timestamp": "2026-08-29T17:17:47.080Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.6133335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.22014136, "avg_duration_per_task_ms": 192160.83, "avg_output_tokens_per_task": 15089.79, "last_run_timestamp": "2026-09-01T12:57:05.201Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.613333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.04676464, "avg_duration_per_task_ms": 28214.94, "avg_output_tokens_per_task": 3108.98, "last_run_timestamp": "2026-09-19T08:06:47.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.6113782756410256, "accuracy_std_dev": 0.036669727145404285, "run_count": 25, "total_questions": 1248, "avg_cost_per_task_usd": 0.06140813701923077, "avg_duration_per_task_ms": 418606.3149038461, "avg_output_tokens_per_task": 21705.047275641027, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.6110183873121869, "accuracy_std_dev": 0.05300622363142776, "run_count": 24, "total_questions": 1198, "avg_cost_per_task_usd": 0.09442565108514192, "avg_duration_per_task_ms": 376469.8146911519, "avg_output_tokens_per_task": 18202.715358931553, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.608, "accuracy_std_dev": 0.25443270230062803, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.005877256, "avg_duration_per_task_ms": 51102.604, "avg_output_tokens_per_task": 3478.088, "last_run_timestamp": "2026-06-30T19:12:02.101Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6077776666666667, "accuracy_std_dev": 0.08590813209793095, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01715707, "avg_duration_per_task_ms": 128260.91, "avg_output_tokens_per_task": 13292.39, "last_run_timestamp": "2026-08-29T16:48:14.218Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.6066666666666667, "accuracy_std_dev": 0.15241273398753646, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.017056396666666668, "avg_duration_per_task_ms": 233868.72666666665, "avg_output_tokens_per_task": 8308.71, "last_run_timestamp": "2026-08-29T16:33:39.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.60599996, "accuracy_std_dev": 0.03504916217141859, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.038558521600000006, "avg_duration_per_task_ms": 20769.1408, "avg_output_tokens_per_task": 12079.4896, "last_run_timestamp": "2026-08-29T16:55:05.300Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6033335000000001, "accuracy_std_dev": 0.028867224784346696, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04717542499999999, "avg_duration_per_task_ms": 204909.125, "avg_output_tokens_per_task": 3753.575, "last_run_timestamp": "2026-09-19T08:01:34.089Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2-flash-20251210", "model_slug": "xiaomi/mimo-v2-flash", "model_name": "Xiaomi: MiMo-V2-Flash", "provider_name": "Novita", "accuracy": 0.6025, "accuracy_std_dev": 0.03799671038392668, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.005295742500000001, "avg_duration_per_task_ms": 41164.3, "avg_output_tokens_per_task": 1845.8825, "last_run_timestamp": "2026-05-23T08:43:48.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.6, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26466022, "avg_duration_per_task_ms": 947997.3, "avg_output_tokens_per_task": 14069, "last_run_timestamp": "2026-07-31T00:04:29.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Io Net", "accuracy": 0.6, "accuracy_std_dev": 0.03651483716701107, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.014077083333333332, "avg_duration_per_task_ms": 306993.37, "avg_output_tokens_per_task": 10503.493333333334, "last_run_timestamp": "2026-05-09T16:36:33.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Nebius", "accuracy": 0.5985185555555556, "accuracy_std_dev": 0.06724027958681894, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05260444666666667, "avg_duration_per_task_ms": 181957.32444444444, "avg_output_tokens_per_task": 8015.968888888889, "last_run_timestamp": "2026-08-15T14:16:11.776Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "AtlasCloud", "accuracy": 0.5980392352941176, "accuracy_std_dev": 0.04037501463781108, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.02223544235294118, "avg_duration_per_task_ms": 43177.25411764706, "avg_output_tokens_per_task": 1978.5894117647058, "last_run_timestamp": "2026-07-01T01:59:48.107Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Phala", "accuracy": 0.5966666428571429, "accuracy_std_dev": 0.06755366708414358, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.04054190857142857, "avg_duration_per_task_ms": 231295.97714285715, "avg_output_tokens_per_task": 10843.084285714285, "last_run_timestamp": "2026-07-04T16:00:25.388Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.5942841618589744, "accuracy_std_dev": 0.10277414844439432, "run_count": 25, "total_questions": 1248, "avg_cost_per_task_usd": 0.0771629423076923, "avg_duration_per_task_ms": 262057.74919871794, "avg_output_tokens_per_task": 6180.838141025641, "last_run_timestamp": "2026-08-29T16:33:38.223Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.5933335000000001, "accuracy_std_dev": 0.05333349999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08864457999999999, "avg_duration_per_task_ms": 125444.95, "avg_output_tokens_per_task": 11722.21, "last_run_timestamp": "2026-09-01T17:31:29.315Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.5922862479338843, "accuracy_std_dev": 0.1002871639271946, "run_count": 5, "total_questions": 242, "avg_cost_per_task_usd": 0.5630070619834712, "avg_duration_per_task_ms": 3123571.8512396696, "avg_output_tokens_per_task": 96760.30578512397, "last_run_timestamp": "2026-09-19T08:12:55.047Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "model_slug": "minimax/minimax-m2.1", "model_name": "MiniMax: MiniMax M2.1", "provider_name": "AtlasCloud", "accuracy": 0.5908772105263158, "accuracy_std_dev": 0.1441605041942301, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.009657277894736842, "avg_duration_per_task_ms": 71760.33368421052, "avg_output_tokens_per_task": 3688.9136842105263, "last_run_timestamp": "2026-07-01T02:00:24.123Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Wafer", "accuracy": 0.5893471237113401, "accuracy_std_dev": 0.30515532350212654, "run_count": 4, "total_questions": 194, "avg_cost_per_task_usd": 0.02748909793814433, "avg_duration_per_task_ms": 98623.75773195876, "avg_output_tokens_per_task": 4244.2525773195875, "last_run_timestamp": "2026-08-15T14:23:29.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.5866666666666667, "accuracy_std_dev": 0.0066666666750000055, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029609283333333326, "avg_duration_per_task_ms": 75747.02, "avg_output_tokens_per_task": 8199.643333333333, "last_run_timestamp": "2026-09-01T17:32:36.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.5861581787365177, "accuracy_std_dev": 0.08951304546845104, "run_count": 26, "total_questions": 1298, "avg_cost_per_task_usd": 0.013956122496147918, "avg_duration_per_task_ms": 66319.37981510015, "avg_output_tokens_per_task": 2672.7750385208014, "last_run_timestamp": "2026-08-29T16:08:08.875Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.581538423076923, "accuracy_std_dev": 0.06680269140718871, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.025179401538461547, "avg_duration_per_task_ms": 63238.23923076923, "avg_output_tokens_per_task": 2868.8115384615385, "last_run_timestamp": "2026-08-29T16:43:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.5811856778350516, "accuracy_std_dev": 0.03980730883718937, "run_count": 8, "total_questions": 388, "avg_cost_per_task_usd": 0.07594913402061856, "avg_duration_per_task_ms": 92087.82731958762, "avg_output_tokens_per_task": 5709.461340206186, "last_run_timestamp": "2026-08-29T16:11:10.141Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-chat-20251211", "model_slug": "openai/gpt-5.2-chat", "model_name": "OpenAI: GPT-5.2 Chat", "provider_name": "OpenAI", "accuracy": 0.58, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.10846708, "avg_duration_per_task_ms": 79571.2, "avg_output_tokens_per_task": 4199.82, "last_run_timestamp": "2026-08-01T08:05:35.385Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Together", "accuracy": 0.58, "accuracy_std_dev": 0.03577708763999662, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.04891755600000001, "avg_duration_per_task_ms": 28136.316, "avg_output_tokens_per_task": 1804.596, "last_run_timestamp": "2026-05-09T08:27:19.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.5766665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.028599199999999998, "avg_duration_per_task_ms": 78841.33, "avg_output_tokens_per_task": 8172.09, "last_run_timestamp": "2026-09-01T17:31:45.195Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.5747916875, "accuracy_std_dev": 0.09687443552720627, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.033850981249999995, "avg_duration_per_task_ms": 62474.61875, "avg_output_tokens_per_task": 3318.425, "last_run_timestamp": "2026-08-29T16:44:17.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.573333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012256119999999999, "avg_duration_per_task_ms": 120534.1, "avg_output_tokens_per_task": 16750, "last_run_timestamp": "2026-08-07T00:00:02.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5716783164335665, "accuracy_std_dev": 0.05890284538551935, "run_count": 23, "total_questions": 1144, "avg_cost_per_task_usd": 0.09208451311188812, "avg_duration_per_task_ms": 689217.8505244756, "avg_output_tokens_per_task": 15762.438811188811, "last_run_timestamp": "2026-08-29T16:31:42.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.5712366451612902, "accuracy_std_dev": 0.0858999966086146, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.038580427419354836, "avg_duration_per_task_ms": 2634832.3427419355, "avg_output_tokens_per_task": 80935.72983870968, "last_run_timestamp": "2026-08-29T17:24:25.648Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "AtlasCloud", "accuracy": 0.5706666666666667, "accuracy_std_dev": 0.07187643718369895, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.059964562666666665, "avg_duration_per_task_ms": 452906.82933333336, "avg_output_tokens_per_task": 11189.822666666667, "last_run_timestamp": "2026-06-20T22:23:15.613Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.569727612244898, "accuracy_std_dev": 0.026944500000000038, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.0024336428571428573, "avg_duration_per_task_ms": 317617.1530612245, "avg_output_tokens_per_task": 5820.755102040816, "last_run_timestamp": "2026-08-29T16:57:41.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "SiliconFlow", "accuracy": 0.5666666666666667, "accuracy_std_dev": 0.05120763831912405, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03156545333333333, "avg_duration_per_task_ms": 131519.01666666666, "avg_output_tokens_per_task": 1421.2433333333333, "last_run_timestamp": "2026-05-09T11:38:50.156Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Chutes", "accuracy": 0.5658332500000001, "accuracy_std_dev": 0.1112399227646599, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.03004894333333333, "avg_duration_per_task_ms": 247199.82, "avg_output_tokens_per_task": 13306.571666666667, "last_run_timestamp": "2026-07-18T16:00:09.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.5648772953020134, "accuracy_std_dev": 0.0424873867714486, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.1486826308724832, "avg_duration_per_task_ms": 18002.88590604027, "avg_output_tokens_per_task": 2546.3288590604025, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.5615522976098689, "accuracy_std_dev": 0.13428503772778175, "run_count": 26, "total_questions": 1297, "avg_cost_per_task_usd": 0.0262945235158057, "avg_duration_per_task_ms": 147034.7632999229, "avg_output_tokens_per_task": 7801.311488049345, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5604491347150259, "accuracy_std_dev": 0.10678459157041047, "run_count": 8, "total_questions": 386, "avg_cost_per_task_usd": 0.07120025388601035, "avg_duration_per_task_ms": 510444.22279792745, "avg_output_tokens_per_task": 29300.067357512955, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Novita", "accuracy": 0.560303, "accuracy_std_dev": 0.04800994961179683, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.12068492181818179, "avg_duration_per_task_ms": 185307.95727272728, "avg_output_tokens_per_task": 3636.26, "last_run_timestamp": "2026-08-01T17:38:24.442Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.5533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03619409, "avg_duration_per_task_ms": 60452.63, "avg_output_tokens_per_task": 6631.82, "last_run_timestamp": "2026-09-01T17:31:47.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "auto-routing", "accuracy": 0.5531269180327869, "accuracy_std_dev": 0.05461221697062189, "run_count": 22, "total_questions": 1098, "avg_cost_per_task_usd": 0.12032896812386154, "avg_duration_per_task_ms": 164673.07012750456, "avg_output_tokens_per_task": 3443.247723132969, "last_run_timestamp": "2026-08-01T17:38:23.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.5500352600422833, "accuracy_std_dev": 0.058938257867956145, "run_count": 19, "total_questions": 946, "avg_cost_per_task_usd": 0.0899786046511628, "avg_duration_per_task_ms": 885342.9714587738, "avg_output_tokens_per_task": 18576.52959830867, "last_run_timestamp": "2026-08-29T16:25:50.849Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.55, "accuracy_std_dev": 0.036667000000000005, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23514412, "avg_duration_per_task_ms": 166142.51, "avg_output_tokens_per_task": 13675.21, "last_run_timestamp": "2026-09-01T13:01:57.044Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.5488554020100502, "accuracy_std_dev": 0.08234461848628022, "run_count": 24, "total_questions": 1194, "avg_cost_per_task_usd": 0.07921593467336686, "avg_duration_per_task_ms": 188771.27135678392, "avg_output_tokens_per_task": 31712.71691792295, "last_run_timestamp": "2026-08-29T17:20:10.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.5379486923076923, "accuracy_std_dev": 0.2535341663450706, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.02374335076923077, "avg_duration_per_task_ms": 109221.52846153846, "avg_output_tokens_per_task": 2656.123076923077, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.5342466164383561, "accuracy_std_dev": 0.014081561995113387, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.1389923630136986, "avg_duration_per_task_ms": 68151.21917808219, "avg_output_tokens_per_task": 2403.2671232876714, "last_run_timestamp": "2026-09-01T13:01:05.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Ambient", "accuracy": 0.53, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00720358, "avg_duration_per_task_ms": 62972.84, "avg_output_tokens_per_task": 2706.77, "last_run_timestamp": "2026-05-23T16:34:17.047Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Fireworks", "accuracy": 0.53, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.004457689999999999, "avg_duration_per_task_ms": 29494.83, "avg_output_tokens_per_task": 2949.9, "last_run_timestamp": "2026-04-25T10:10:06.379Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.5213333333333333, "accuracy_std_dev": 0.019352465629876622, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.027455891999999996, "avg_duration_per_task_ms": 341646.812, "avg_output_tokens_per_task": 11097.765333333333, "last_run_timestamp": "2026-08-29T16:44:22.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Clarifai", "accuracy": 0.52, "accuracy_std_dev": 0.1, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02067781, "avg_duration_per_task_ms": 133609.62, "avg_output_tokens_per_task": 82701.09, "last_run_timestamp": "2026-04-25T09:59:33.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.51916675, "accuracy_std_dev": 0.02591380209343084, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.025645805, "avg_duration_per_task_ms": 448484.4, "avg_output_tokens_per_task": 52443.9575, "last_run_timestamp": "2026-09-19T16:00:22.539Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "WandB", "accuracy": 0.5151851666666667, "accuracy_std_dev": 0.028528172728900812, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.03173818, "avg_duration_per_task_ms": 738927.32, "avg_output_tokens_per_task": 116854.36666666667, "last_run_timestamp": "2026-07-18T16:00:10.167Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5151512828282828, "accuracy_std_dev": 0.04224499999999998, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.026153939393939396, "avg_duration_per_task_ms": 213641.58585858587, "avg_output_tokens_per_task": 54161.38383838384, "last_run_timestamp": "2026-09-01T17:31:30.190Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.513371664756447, "accuracy_std_dev": 0.024300137045049023, "run_count": 14, "total_questions": 698, "avg_cost_per_task_usd": 0.05487210888252148, "avg_duration_per_task_ms": 254698.24212034384, "avg_output_tokens_per_task": 3318.022922636103, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.5126262474747475, "accuracy_std_dev": 0.02515381026771839, "run_count": 16, "total_questions": 792, "avg_cost_per_task_usd": 0.023066161616161616, "avg_duration_per_task_ms": 1118619.621212121, "avg_output_tokens_per_task": 98415.15909090909, "last_run_timestamp": "2026-08-29T16:33:29.988Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Ionstream", "accuracy": 0.510644818430657, "accuracy_std_dev": 0.19786794978818611, "run_count": 22, "total_questions": 1096, "avg_cost_per_task_usd": 0.010844100364963506, "avg_duration_per_task_ms": 37477.74726277372, "avg_output_tokens_per_task": 1870.0903284671533, "last_run_timestamp": "2026-08-08T16:03:49.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "NextBit", "accuracy": 0.5083333333333333, "accuracy_std_dev": 0.02229848026709944, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.007687528333333332, "avg_duration_per_task_ms": 35548.63166666667, "avg_output_tokens_per_task": 2391.6783333333333, "last_run_timestamp": "2026-06-20T16:41:40.557Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.5081871543859648, "accuracy_std_dev": 0.018863102843005027, "run_count": 23, "total_questions": 1140, "avg_cost_per_task_usd": 0.037433712280701754, "avg_duration_per_task_ms": 648881.6219298246, "avg_output_tokens_per_task": 187737.9605263158, "last_run_timestamp": "2026-08-29T16:44:16.523Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Novita", "accuracy": 0.5054545454545455, "accuracy_std_dev": 0.02709029895591154, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.01571898, "avg_duration_per_task_ms": 333669.9781818182, "avg_output_tokens_per_task": 17186.79090909091, "last_run_timestamp": "2026-05-23T13:37:43.176Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.504816971098266, "accuracy_std_dev": 0.0288748445833789, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.024907991329479767, "avg_duration_per_task_ms": 563438.3439306358, "avg_output_tokens_per_task": 53889.92774566474, "last_run_timestamp": "2026-09-19T16:00:24.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "WandB", "accuracy": 0.5025926111111111, "accuracy_std_dev": 0.05174193415740047, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.10888994777777779, "avg_duration_per_task_ms": 112476.33777777778, "avg_output_tokens_per_task": 1597.5555555555557, "last_run_timestamp": "2026-07-18T08:04:08.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.5017007653061224, "accuracy_std_dev": 0.02191038312649964, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.02296841326530612, "avg_duration_per_task_ms": 1353579.7653061224, "avg_output_tokens_per_task": 120488.71173469388, "last_run_timestamp": "2026-08-29T17:16:15.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Ambient", "accuracy": 0.5, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00390972, "avg_duration_per_task_ms": 84786.05, "avg_output_tokens_per_task": 2818.58, "last_run_timestamp": "2026-06-13T17:26:55.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.49960782352941174, "accuracy_std_dev": 0.06175941664254676, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.015916549411764704, "avg_duration_per_task_ms": 33738.58705882353, "avg_output_tokens_per_task": 1680.0847058823529, "last_run_timestamp": "2026-06-30T19:02:38.080Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.49888883333333334, "accuracy_std_dev": 0.03680103829792791, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.026936693333333334, "avg_duration_per_task_ms": 564954.42, "avg_output_tokens_per_task": 52653.473333333335, "last_run_timestamp": "2026-09-19T16:00:24.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.49830913043478264, "accuracy_std_dev": 0.23149644601826339, "run_count": 14, "total_questions": 690, "avg_cost_per_task_usd": 0.005944484057971015, "avg_duration_per_task_ms": 247630.89855072464, "avg_output_tokens_per_task": 4340.623188405797, "last_run_timestamp": "2026-08-29T16:33:36.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.49666650000000007, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.13482986, "avg_duration_per_task_ms": 58783.54, "avg_output_tokens_per_task": 2896.5, "last_run_timestamp": "2026-09-01T13:00:51.538Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "CoreWeave", "accuracy": 0.49666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23109727999999996, "avg_duration_per_task_ms": 80168.37, "avg_output_tokens_per_task": 2746.03, "last_run_timestamp": "2026-08-01T17:41:27.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.49479164285714283, "accuracy_std_dev": 0.21842675350179366, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.0037404129464285716, "avg_duration_per_task_ms": 123800.87723214286, "avg_output_tokens_per_task": 3922.089285714286, "last_run_timestamp": "2026-07-11T15:34:14.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.493333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14235776, "avg_duration_per_task_ms": 443941.74, "avg_output_tokens_per_task": 8833.66, "last_run_timestamp": "2026-08-06T00:00:02.908Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.49308644444444444, "accuracy_std_dev": 0.05507886856091561, "run_count": 27, "total_questions": 1350, "avg_cost_per_task_usd": 0.04064708518518519, "avg_duration_per_task_ms": 188126.99925925926, "avg_output_tokens_per_task": 2360.5, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.48666650000000006, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12261337999999998, "avg_duration_per_task_ms": 50813.54, "avg_output_tokens_per_task": 2655.2, "last_run_timestamp": "2026-09-01T12:53:49.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.4864496585365854, "accuracy_std_dev": 0.04830717864914074, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.030671650406504065, "avg_duration_per_task_ms": 1720485.825203252, "avg_output_tokens_per_task": 131204.84552845528, "last_run_timestamp": "2026-08-29T17:10:29.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.4855528467336683, "accuracy_std_dev": 0.22212037609522364, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.011942556532663315, "avg_duration_per_task_ms": 259270.25502512563, "avg_output_tokens_per_task": 6411.776381909548, "last_run_timestamp": "2026-08-29T17:02:26.939Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.485269303030303, "accuracy_std_dev": 0.023007442468416624, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.028894598484848485, "avg_duration_per_task_ms": 389374.452020202, "avg_output_tokens_per_task": 68395.02777777778, "last_run_timestamp": "2026-09-19T16:00:24.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.48333325000000005, "accuracy_std_dev": 0.025166081671319035, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021045685, "avg_duration_per_task_ms": 161568.115, "avg_output_tokens_per_task": 3811.995, "last_run_timestamp": "2026-08-29T16:46:44.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Ambient", "accuracy": 0.48, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00732523, "avg_duration_per_task_ms": 33542.46, "avg_output_tokens_per_task": 1578.75, "last_run_timestamp": "2026-06-13T08:36:56.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Crusoe", "accuracy": 0.47916700000000007, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.04362447916666667, "avg_duration_per_task_ms": 66953.94791666667, "avg_output_tokens_per_task": 2828.96875, "last_run_timestamp": "2026-08-15T14:08:14.681Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4775, "accuracy_std_dev": 0.03857702972301004, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.0438162675, "avg_duration_per_task_ms": 100713.26625, "avg_output_tokens_per_task": 2946.27125, "last_run_timestamp": "2026-08-29T16:33:38.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.4762204823232323, "accuracy_std_dev": 0.04091233889137237, "run_count": 16, "total_questions": 792, "avg_cost_per_task_usd": 0.09208659090909091, "avg_duration_per_task_ms": 53154.96338383838, "avg_output_tokens_per_task": 3752.4343434343436, "last_run_timestamp": "2026-08-15T14:15:58.674Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.47619069387755103, "accuracy_std_dev": 0.04472200000000001, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.042756918367346936, "avg_duration_per_task_ms": 697970.0612244898, "avg_output_tokens_per_task": 99743.38775510204, "last_run_timestamp": "2026-09-01T17:27:02.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.4756506221662469, "accuracy_std_dev": 0.009147616682220762, "run_count": 16, "total_questions": 794, "avg_cost_per_task_usd": 0.11002369899244333, "avg_duration_per_task_ms": 1736618.065491184, "avg_output_tokens_per_task": 28866.458438287154, "last_run_timestamp": "2026-08-29T16:20:58.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.4749275652173913, "accuracy_std_dev": 0.036802997798296266, "run_count": 23, "total_questions": 1150, "avg_cost_per_task_usd": 0.016968564347826086, "avg_duration_per_task_ms": 57427.46, "avg_output_tokens_per_task": 1869.2747826086957, "last_run_timestamp": "2026-08-29T17:27:31.024Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.47472220833333334, "accuracy_std_dev": 0.052598362074449914, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.021700947499999998, "avg_duration_per_task_ms": 28847.52666666667, "avg_output_tokens_per_task": 2211.5291666666667, "last_run_timestamp": "2026-08-29T16:42:36.232Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.4744445, "accuracy_std_dev": 0.03557299398771489, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.023446433333333332, "avg_duration_per_task_ms": 1018917.68, "avg_output_tokens_per_task": 62981.19, "last_run_timestamp": "2026-07-18T16:00:08.888Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.47376544907407403, "accuracy_std_dev": 0.0497035553280053, "run_count": 26, "total_questions": 1296, "avg_cost_per_task_usd": 0.014925031635802471, "avg_duration_per_task_ms": 36653.99614197531, "avg_output_tokens_per_task": 2309.6319444444443, "last_run_timestamp": "2026-08-29T16:12:10.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "SiliconFlow", "accuracy": 0.4736589626436782, "accuracy_std_dev": 0.0308370259740376, "run_count": 14, "total_questions": 696, "avg_cost_per_task_usd": 0.029854971264367817, "avg_duration_per_task_ms": 1270502.7399425288, "avg_output_tokens_per_task": 22673.870689655174, "last_run_timestamp": "2026-08-15T14:12:30.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.023663, "avg_duration_per_task_ms": 175642.6, "avg_output_tokens_per_task": 45106.7, "last_run_timestamp": "2026-08-01T17:39:14.289Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "OpenAI", "accuracy": 0.473333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09545477999999999, "avg_duration_per_task_ms": 59370.42, "avg_output_tokens_per_task": 2482.38, "last_run_timestamp": "2026-08-01T08:06:06.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Cerebras", "accuracy": 0.472, "accuracy_std_dev": 0.048332183894378274, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.05911650399999999, "avg_duration_per_task_ms": 16473.438, "avg_output_tokens_per_task": 1544.504, "last_run_timestamp": "2026-05-23T08:11:43.884Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.4710981791907515, "accuracy_std_dev": 0.2776117089494547, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.01732042774566474, "avg_duration_per_task_ms": 101207.72832369943, "avg_output_tokens_per_task": 18057.771676300577, "last_run_timestamp": "2026-08-29T17:06:47.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Phala", "accuracy": 0.47, "accuracy_std_dev": 0.03, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06848815, "avg_duration_per_task_ms": 119560.68, "avg_output_tokens_per_task": 4089.64, "last_run_timestamp": "2026-04-25T09:53:01.657Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.46805773261205563, "accuracy_std_dev": 0.038430249999417805, "run_count": 26, "total_questions": 1294, "avg_cost_per_task_usd": 0.01566730989180835, "avg_duration_per_task_ms": 431720.4644513138, "avg_output_tokens_per_task": 13130.05486862442, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Together", "accuracy": 0.468, "accuracy_std_dev": 0.020396078054371138, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.17626782199999996, "avg_duration_per_task_ms": 36863.33, "avg_output_tokens_per_task": 1283.172, "last_run_timestamp": "2026-05-23T08:17:34.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.46799996, "accuracy_std_dev": 0.03661208446562965, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.0154081888, "avg_duration_per_task_ms": 64765.2344, "avg_output_tokens_per_task": 2351.1312, "last_run_timestamp": "2026-08-29T16:22:50.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.4675, "accuracy_std_dev": 0.009682458365518523, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0121487525, "avg_duration_per_task_ms": 77116.52, "avg_output_tokens_per_task": 1985.6575, "last_run_timestamp": "2026-05-16T08:26:24.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.46717946153846157, "accuracy_std_dev": 0.15927102080132052, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.02094849153846154, "avg_duration_per_task_ms": 36856.35692307692, "avg_output_tokens_per_task": 1813.543076923077, "last_run_timestamp": "2026-08-29T16:17:15.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.4660245762439807, "accuracy_std_dev": 0.04158627548980073, "run_count": 25, "total_questions": 1246, "avg_cost_per_task_usd": 0.016147356340288924, "avg_duration_per_task_ms": 152395.11235955055, "avg_output_tokens_per_task": 11304.92455858748, "last_run_timestamp": "2026-08-29T16:33:30.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "DigitalOcean", "accuracy": 0.46444433333333335, "accuracy_std_dev": 0.008314853589945066, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.033371, "avg_duration_per_task_ms": 305705.87333333335, "avg_output_tokens_per_task": 3594.2266666666665, "last_run_timestamp": "2026-08-15T08:15:42.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.464166625, "accuracy_std_dev": 0.03843292588047877, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05392122499999999, "avg_duration_per_task_ms": 149701.2725, "avg_output_tokens_per_task": 3823.17, "last_run_timestamp": "2026-08-29T17:03:47.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.46333349999999995, "accuracy_std_dev": 0.0166665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20313888, "avg_duration_per_task_ms": 52334.34, "avg_output_tokens_per_task": 2078.1, "last_run_timestamp": "2026-09-01T12:41:31.668Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.463333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26064434, "avg_duration_per_task_ms": 31467.4, "avg_output_tokens_per_task": 1819.56, "last_run_timestamp": "2026-08-01T08:06:06.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.46319996, "accuracy_std_dev": 0.0514715031235576, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.05008044400000001, "avg_duration_per_task_ms": 66373.1232, "avg_output_tokens_per_task": 1925.0056, "last_run_timestamp": "2026-08-29T16:33:30.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.46244856790123456, "accuracy_std_dev": 0.03790693205554764, "run_count": 26, "total_questions": 1296, "avg_cost_per_task_usd": 0.015229477623456784, "avg_duration_per_task_ms": 96153.46373456791, "avg_output_tokens_per_task": 2336.3819444444443, "last_run_timestamp": "2026-08-29T16:22:50.200Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.46140347368421053, "accuracy_std_dev": 0.04392064180142758, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.028559272631578946, "avg_duration_per_task_ms": 26772.692631578946, "avg_output_tokens_per_task": 1583.818947368421, "last_run_timestamp": "2026-08-29T16:58:32.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 156315.84, "avg_output_tokens_per_task": 4524, "last_run_timestamp": "2026-09-01T12:59:40.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-chat-20260303", "model_slug": "openai/gpt-5.3-chat", "model_name": "OpenAI: GPT-5.3 Chat", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09873096, "avg_duration_per_task_ms": 65244.1, "avg_output_tokens_per_task": 2748.04, "last_run_timestamp": "2026-08-01T08:05:59.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.46, "accuracy_std_dev": 0.013333000000000012, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02811872, "avg_duration_per_task_ms": 579611.05, "avg_output_tokens_per_task": 20031.27, "last_run_timestamp": "2026-09-01T13:02:37.014Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": 0.042365957548484615, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.01855554615384615, "avg_duration_per_task_ms": 109691.66846153846, "avg_output_tokens_per_task": 2329.1007692307694, "last_run_timestamp": "2026-08-29T16:33:39.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.4585964736842105, "accuracy_std_dev": 0.11550210320544337, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.08024197263157894, "avg_duration_per_task_ms": 119560.23052631579, "avg_output_tokens_per_task": 7054.113684210526, "last_run_timestamp": "2026-07-04T16:00:25.000Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.45777799999999996, "accuracy_std_dev": 0.008314764498569207, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00622334, "avg_duration_per_task_ms": 39170.17333333333, "avg_output_tokens_per_task": 1174.36, "last_run_timestamp": "2026-08-15T08:16:13.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.4570513076923077, "accuracy_std_dev": 0.04206556644533465, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.03599279999999999, "avg_duration_per_task_ms": 44357.65923076923, "avg_output_tokens_per_task": 2306.9653846153847, "last_run_timestamp": "2026-08-29T17:01:46.593Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Alibaba", "accuracy": 0.45699995, "accuracy_std_dev": 0.039635830171039686, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.016694145000000007, "avg_duration_per_task_ms": 191163.145, "avg_output_tokens_per_task": 15722.233, "last_run_timestamp": "2026-07-18T16:00:08.521Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "AtlasCloud", "accuracy": 0.45686276470588233, "accuracy_std_dev": 0.05780818724787877, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.08079224352941176, "avg_duration_per_task_ms": 50778.730588235296, "avg_output_tokens_per_task": 1461.3635294117646, "last_run_timestamp": "2026-06-30T19:13:39.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.4561111666666667, "accuracy_std_dev": 0.041137423636783854, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.01247452916666667, "avg_duration_per_task_ms": 49982.72, "avg_output_tokens_per_task": 2226.931666666667, "last_run_timestamp": "2026-08-29T16:43:11.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.45542626162790695, "accuracy_std_dev": 0.018912652487751393, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.046210372093023255, "avg_duration_per_task_ms": 174189.27034883722, "avg_output_tokens_per_task": 1898.7906976744187, "last_run_timestamp": "2026-08-29T16:23:01.234Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "AtlasCloud", "accuracy": 0.45450982352941177, "accuracy_std_dev": 0.03875415290723167, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.01727678352941177, "avg_duration_per_task_ms": 273100.5176470588, "avg_output_tokens_per_task": 7362.865882352941, "last_run_timestamp": "2026-07-01T02:31:49.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.45413336, "accuracy_std_dev": 0.03916098911506705, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.17327989679999997, "avg_duration_per_task_ms": 59985.74, "avg_output_tokens_per_task": 1913.856, "last_run_timestamp": "2026-08-29T16:12:10.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.4533333333333333, "accuracy_std_dev": 0.03771236166328253, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01162648, "avg_duration_per_task_ms": 116752.73333333334, "avg_output_tokens_per_task": 1542.3466666666666, "last_run_timestamp": "2026-05-09T11:02:11.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.45200004, "accuracy_std_dev": 0.04410092989720739, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.06653196960000002, "avg_duration_per_task_ms": 66418.1632, "avg_output_tokens_per_task": 1909.792, "last_run_timestamp": "2026-08-29T16:25:26.800Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Together", "accuracy": 0.450416625, "accuracy_std_dev": 0.05038566298794107, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.026321354999999994, "avg_duration_per_task_ms": 84986.70625, "avg_output_tokens_per_task": 1810.525, "last_run_timestamp": "2026-07-04T16:00:25.259Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.44898461069836554, "accuracy_std_dev": 0.07415219664337017, "run_count": 27, "total_questions": 1346, "avg_cost_per_task_usd": 0.020572284546805354, "avg_duration_per_task_ms": 49604.04457652303, "avg_output_tokens_per_task": 2003.719910846954, "last_run_timestamp": "2026-08-29T17:17:42.808Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.448889, "accuracy_std_dev": 0.0031423825355930196, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08242190666666667, "avg_duration_per_task_ms": 81216.82666666666, "avg_output_tokens_per_task": 4095.7466666666664, "last_run_timestamp": "2026-08-15T08:11:06.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.448811634369287, "accuracy_std_dev": 0.059025095658296695, "run_count": 11, "total_questions": 547, "avg_cost_per_task_usd": 0.030702687385740404, "avg_duration_per_task_ms": 605640.2120658135, "avg_output_tokens_per_task": 6772.7239488117, "last_run_timestamp": "2026-08-30T00:01:10.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "WandB", "accuracy": 0.44875, "accuracy_std_dev": 0.04580870550452174, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.011756012500000001, "avg_duration_per_task_ms": 28247.49, "avg_output_tokens_per_task": 1531.14625, "last_run_timestamp": "2026-06-20T16:39:11.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.4466665, "accuracy_std_dev": 0.006666499999999992, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020371870000000004, "avg_duration_per_task_ms": 172937.47, "avg_output_tokens_per_task": 42830.02, "last_run_timestamp": "2026-08-01T17:38:24.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.4466665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20957163, "avg_duration_per_task_ms": 47193.8, "avg_output_tokens_per_task": 2215.92, "last_run_timestamp": "2026-09-01T12:58:37.172Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.44642858482142855, "accuracy_std_dev": 0.022519182833890147, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.04059176116071428, "avg_duration_per_task_ms": 108020.11607142857, "avg_output_tokens_per_task": 1939.064732142857, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "auto-routing", "accuracy": 0.4452632105263158, "accuracy_std_dev": 0.057249987520641206, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.01557863684210526, "avg_duration_per_task_ms": 192621.90315789473, "avg_output_tokens_per_task": 10602.088421052631, "last_run_timestamp": "2026-08-01T17:38:32.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.445, "accuracy_std_dev": 0.001667000000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06940407, "avg_duration_per_task_ms": 279300.93, "avg_output_tokens_per_task": 3446.49, "last_run_timestamp": "2026-09-01T12:58:16.471Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.4444444444444444, "accuracy_std_dev": 0.02196410423850697, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.0480205202020202, "avg_duration_per_task_ms": 850576.6565656565, "avg_output_tokens_per_task": 104088.0202020202, "last_run_timestamp": "2026-09-01T17:31:48.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Friendli", "accuracy": 0.44375, "accuracy_std_dev": 0.04539754949333719, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.021965795, "avg_duration_per_task_ms": 36665.47375, "avg_output_tokens_per_task": 1608.6575, "last_run_timestamp": "2026-06-20T17:18:35.419Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.44, "accuracy_std_dev": 0.013332999999999984, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02857548, "avg_duration_per_task_ms": 74491.22, "avg_output_tokens_per_task": 2449.6, "last_run_timestamp": "2026-09-01T17:30:53.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "OpenAI", "accuracy": 0.44, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.7291186999999999, "avg_duration_per_task_ms": 35507.54, "avg_output_tokens_per_task": 3272.36, "last_run_timestamp": "2026-08-01T08:05:57.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.43902436585365856, "accuracy_std_dev": 0.046630122309082575, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.15270590650406504, "avg_duration_per_task_ms": 175421.88617886178, "avg_output_tokens_per_task": 6311.357723577235, "last_run_timestamp": "2026-08-30T00:01:09.486Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.4389744230769231, "accuracy_std_dev": 0.05707069678320921, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.03386180076923076, "avg_duration_per_task_ms": 88270.66923076923, "avg_output_tokens_per_task": 2280.8869230769233, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.4383965367088608, "accuracy_std_dev": 0.03960836597472182, "run_count": 8, "total_questions": 395, "avg_cost_per_task_usd": 0.05240046329113924, "avg_duration_per_task_ms": 422067.2658227848, "avg_output_tokens_per_task": 20533.427848101266, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.43711742237061774, "accuracy_std_dev": 0.06180167813928354, "run_count": 24, "total_questions": 1198, "avg_cost_per_task_usd": 0.010960462437395655, "avg_duration_per_task_ms": 96791.42904841402, "avg_output_tokens_per_task": 1832.7679465776293, "last_run_timestamp": "2026-08-29T16:11:10.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.43666675, "accuracy_std_dev": 0.011055139582452123, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.21359883, "avg_duration_per_task_ms": 47801.905, "avg_output_tokens_per_task": 2174.32, "last_run_timestamp": "2026-09-01T13:02:23.243Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.43666650000000007, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.014376880000000002, "avg_duration_per_task_ms": 93606.58, "avg_output_tokens_per_task": 10159.34, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.43621399845679015, "accuracy_std_dev": 0.04684420405366707, "run_count": 26, "total_questions": 1296, "avg_cost_per_task_usd": 0.015120972993827161, "avg_duration_per_task_ms": 167672.98225308643, "avg_output_tokens_per_task": 2265.6689814814813, "last_run_timestamp": "2026-08-29T16:17:35.870Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cerebras", "accuracy": 0.436, "accuracy_std_dev": 0.026532998322843202, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.012890084, "avg_duration_per_task_ms": 17630.374, "avg_output_tokens_per_task": 1208.354, "last_run_timestamp": "2026-05-23T08:10:41.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.43537074448897795, "accuracy_std_dev": 0.11336612641816779, "run_count": 20, "total_questions": 998, "avg_cost_per_task_usd": 0.009285377755511023, "avg_duration_per_task_ms": 54460.993987975955, "avg_output_tokens_per_task": 1751.9448897795592, "last_run_timestamp": "2026-08-29T17:44:16.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.4346106916058394, "accuracy_std_dev": 0.024005000661578763, "run_count": 22, "total_questions": 1096, "avg_cost_per_task_usd": 0.07036007481751827, "avg_duration_per_task_ms": 656963.5930656934, "avg_output_tokens_per_task": 12332.072080291971, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.43401008121827417, "accuracy_std_dev": 0.029416728833002063, "run_count": 8, "total_questions": 394, "avg_cost_per_task_usd": 0.035802939086294415, "avg_duration_per_task_ms": 50594.96446700508, "avg_output_tokens_per_task": 1953.4467005076142, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.43333334615384617, "accuracy_std_dev": 0.04355954957291871, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.025601543076923073, "avg_duration_per_task_ms": 51646.70153846154, "avg_output_tokens_per_task": 2288.9223076923076, "last_run_timestamp": "2026-08-29T16:10:58.039Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.43215811057692305, "accuracy_std_dev": 0.0625616725008659, "run_count": 25, "total_questions": 1248, "avg_cost_per_task_usd": 0.01311161698717949, "avg_duration_per_task_ms": 240895.54807692306, "avg_output_tokens_per_task": 12017.117788461539, "last_run_timestamp": "2026-08-22T16:08:38.678Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4311113333333333, "accuracy_std_dev": 0.006285236475706817, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023601506666666664, "avg_duration_per_task_ms": 216917.58666666667, "avg_output_tokens_per_task": 3146.213333333333, "last_run_timestamp": "2026-08-15T08:02:48.904Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.43, "accuracy_std_dev": 0.035355339059327376, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.02250267, "avg_duration_per_task_ms": 103645.2525, "avg_output_tokens_per_task": 2220.14375, "last_run_timestamp": "2026-06-20T17:24:59.951Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.43, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02958018, "avg_duration_per_task_ms": 73337.54, "avg_output_tokens_per_task": 2441.2, "last_run_timestamp": "2026-09-01T17:31:46.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.43, "accuracy_std_dev": 0.0033330000000000026, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.23500686999999998, "avg_duration_per_task_ms": 47776.24, "avg_output_tokens_per_task": 2493.12, "last_run_timestamp": "2026-09-01T13:00:52.000Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.42910253846153845, "accuracy_std_dev": 0.049782059536644926, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.016837386923076923, "avg_duration_per_task_ms": 118019.43153846153, "avg_output_tokens_per_task": 2356.7115384615386, "last_run_timestamp": "2026-08-29T16:50:29.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.42883213321167885, "accuracy_std_dev": 0.038713233772523226, "run_count": 22, "total_questions": 1096, "avg_cost_per_task_usd": 0.04187416332116788, "avg_duration_per_task_ms": 349556.7518248175, "avg_output_tokens_per_task": 8660.958941605839, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "Alibaba", "accuracy": 0.4281170687022901, "accuracy_std_dev": 0.06363488770219329, "run_count": 21, "total_questions": 1048, "avg_cost_per_task_usd": 0.019827515267175574, "avg_duration_per_task_ms": 147943.55916030533, "avg_output_tokens_per_task": 10525.584923664122, "last_run_timestamp": "2026-08-01T17:38:29.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.4234779606458123, "accuracy_std_dev": 0.039658085032279355, "run_count": 20, "total_questions": 991, "avg_cost_per_task_usd": 0.019681630676084763, "avg_duration_per_task_ms": 234062.66094853682, "avg_output_tokens_per_task": 2454.673057517659, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.423333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02960835, "avg_duration_per_task_ms": 524604.61, "avg_output_tokens_per_task": 20056.49, "last_run_timestamp": "2026-09-01T12:41:25.851Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.42064714021571653, "accuracy_std_dev": 0.10279858751976688, "run_count": 26, "total_questions": 1298, "avg_cost_per_task_usd": 0.015630807395993835, "avg_duration_per_task_ms": 485399.4352850539, "avg_output_tokens_per_task": 13574.13405238829, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.41933335, "accuracy_std_dev": 0.045161409534330295, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.016045132, "avg_duration_per_task_ms": 211276.496, "avg_output_tokens_per_task": 10822.63, "last_run_timestamp": "2026-08-01T17:38:35.770Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4183674081632653, "accuracy_std_dev": 0.008472500000000022, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.048346357142857135, "avg_duration_per_task_ms": 818319.5204081633, "avg_output_tokens_per_task": 111719.33673469388, "last_run_timestamp": "2026-09-01T17:32:02.570Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.41666675, "accuracy_std_dev": 0.013743584364622648, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.027539789999999998, "avg_duration_per_task_ms": 73803.845, "avg_output_tokens_per_task": 2931.665, "last_run_timestamp": "2026-09-01T17:32:02.600Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.41333329166666666, "accuracy_std_dev": 0.05206835251257009, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.025643208333333337, "avg_duration_per_task_ms": 118146.86333333333, "avg_output_tokens_per_task": 2518.2691666666665, "last_run_timestamp": "2026-08-29T16:31:33.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.413333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06003468, "avg_duration_per_task_ms": 392895.94, "avg_output_tokens_per_task": 17058.96, "last_run_timestamp": "2026-08-01T08:06:29.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.41228081754385965, "accuracy_std_dev": 0.14934019625078246, "run_count": 6, "total_questions": 285, "avg_cost_per_task_usd": 0.03923479298245614, "avg_duration_per_task_ms": 212298.04912280702, "avg_output_tokens_per_task": 88632.91228070176, "last_run_timestamp": "2026-08-29T17:06:38.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41142871428571426, "accuracy_std_dev": 0.022737753675420126, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.014496525714285714, "avg_duration_per_task_ms": 78866.04571428572, "avg_output_tokens_per_task": 9061.385714285714, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "WandB", "accuracy": 0.4103852261306533, "accuracy_std_dev": 0.0554530731629442, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.20432325502512563, "avg_duration_per_task_ms": 305226.5891959799, "avg_output_tokens_per_task": 15810.201005025127, "last_run_timestamp": "2026-07-18T16:00:08.906Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "AtlasCloud", "accuracy": 0.4086021532258065, "accuracy_std_dev": 0.04283406661478688, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.771351502016129, "avg_duration_per_task_ms": 371618.85282258067, "avg_output_tokens_per_task": 14652.147177419354, "last_run_timestamp": "2026-07-01T04:28:59.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.4074358846153846, "accuracy_std_dev": 0.045038470881379175, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.03544446692307693, "avg_duration_per_task_ms": 143373.31461538462, "avg_output_tokens_per_task": 3607.816923076923, "last_run_timestamp": "2026-08-29T16:11:10.099Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.4072934727592267, "accuracy_std_dev": 0.021976352713623176, "run_count": 23, "total_questions": 1138, "avg_cost_per_task_usd": 0.006092479789103691, "avg_duration_per_task_ms": 18758.165202108965, "avg_output_tokens_per_task": 321.83391915641477, "last_run_timestamp": "2026-08-15T14:15:11.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.40666650000000004, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06123743, "avg_duration_per_task_ms": 234369.43, "avg_output_tokens_per_task": 8023.68, "last_run_timestamp": "2026-09-01T12:41:31.874Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.40379423577235773, "accuracy_std_dev": 0.19339763406453553, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.01662922357723577, "avg_duration_per_task_ms": 1820667.5203252032, "avg_output_tokens_per_task": 111556.55284552845, "last_run_timestamp": "2026-08-29T17:12:32.860Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.40160004, "accuracy_std_dev": 0.03598909780639687, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.017235716, "avg_duration_per_task_ms": 165323.3096, "avg_output_tokens_per_task": 2670.9176, "last_run_timestamp": "2026-08-29T16:17:08.788Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.4015928839590444, "accuracy_std_dev": 0.04374660891326003, "run_count": 12, "total_questions": 586, "avg_cost_per_task_usd": 0.05105657679180887, "avg_duration_per_task_ms": 176971.67918088738, "avg_output_tokens_per_task": 3873.042662116041, "last_run_timestamp": "2026-09-19T08:13:57.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "auto-routing", "accuracy": 0.4, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 1.1315643, "avg_duration_per_task_ms": 71996.84, "avg_output_tokens_per_task": 3249.82, "last_run_timestamp": "2026-08-01T08:06:02.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-30b-a3b-instruct", "model_slug": "qwen/qwen3-coder-30b-a3b-instruct", "model_name": "Qwen: Qwen3 Coder 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.4, "accuracy_std_dev": 0.019626323972325204, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020620846666666668, "avg_duration_per_task_ms": 194089.08666666667, "avg_output_tokens_per_task": 3830.9066666666668, "last_run_timestamp": "2026-08-15T08:02:44.458Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.3997436153846154, "accuracy_std_dev": 0.01995555515307593, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.009033361538461534, "avg_duration_per_task_ms": 40294.35923076923, "avg_output_tokens_per_task": 386.8792307692308, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.39916650000000004, "accuracy_std_dev": 0.32402718893134574, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0238333725, "avg_duration_per_task_ms": 73030.4175, "avg_output_tokens_per_task": 12951.32, "last_run_timestamp": "2026-09-19T16:00:30.841Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Inceptron", "accuracy": 0.3975, "accuracy_std_dev": 0.029047375096555618, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.005109264999999999, "avg_duration_per_task_ms": 12326.42, "avg_output_tokens_per_task": 249.89, "last_run_timestamp": "2026-05-16T08:28:13.794Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.3953846153846154, "accuracy_std_dev": 0.02553822881196092, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.008089532307692306, "avg_duration_per_task_ms": 21213.126153846155, "avg_output_tokens_per_task": 369.43384615384616, "last_run_timestamp": "2026-08-29T16:10:09.208Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.39, "accuracy_std_dev": 0.014782396941407474, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.028387520000000006, "avg_duration_per_task_ms": 78569.82333333333, "avg_output_tokens_per_task": 4258.516666666666, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.3894871153846154, "accuracy_std_dev": 0.03518801188159107, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.047560097692307686, "avg_duration_per_task_ms": 76824.58384615385, "avg_output_tokens_per_task": 1990.8353846153846, "last_run_timestamp": "2026-08-29T16:33:39.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.389445249614792, "accuracy_std_dev": 0.0657706643988149, "run_count": 26, "total_questions": 1298, "avg_cost_per_task_usd": 0.035619542372881355, "avg_duration_per_task_ms": 164533.79506933744, "avg_output_tokens_per_task": 2309.8736517719567, "last_run_timestamp": "2026-08-29T16:13:54.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "AtlasCloud", "accuracy": 0.3875, "accuracy_std_dev": 0.035266839949164706, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.01950373, "avg_duration_per_task_ms": 100582.8275, "avg_output_tokens_per_task": 2448.57875, "last_run_timestamp": "2026-06-20T16:46:21.602Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.3852564230769231, "accuracy_std_dev": 0.0754143744899182, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.038005519230769245, "avg_duration_per_task_ms": 57650.9, "avg_output_tokens_per_task": 1802.6207692307692, "last_run_timestamp": "2026-08-29T16:15:12.659Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.3848532040185471, "accuracy_std_dev": 0.034396038458699964, "run_count": 26, "total_questions": 1294, "avg_cost_per_task_usd": 0.012941758887171563, "avg_duration_per_task_ms": 45655.90262751159, "avg_output_tokens_per_task": 569.8686244204018, "last_run_timestamp": "2026-08-29T16:14:38.085Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.3839215882352941, "accuracy_std_dev": 0.05204618331834641, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.01229372588235294, "avg_duration_per_task_ms": 64996.49411764706, "avg_output_tokens_per_task": 1870.1082352941175, "last_run_timestamp": "2026-06-30T19:11:05.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "WandB", "accuracy": 0.38385963157894737, "accuracy_std_dev": 0.04338409272291609, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.01576738736842105, "avg_duration_per_task_ms": 37820.52210526316, "avg_output_tokens_per_task": 2258.8694736842103, "last_run_timestamp": "2026-07-18T08:03:14.950Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.38333349999999994, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07100836, "avg_duration_per_task_ms": 283936.54, "avg_output_tokens_per_task": 7506.19, "last_run_timestamp": "2026-09-01T12:58:37.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.3822342530282638, "accuracy_std_dev": 0.06305786163883313, "run_count": 15, "total_questions": 743, "avg_cost_per_task_usd": 0.009679043068640645, "avg_duration_per_task_ms": 96270.86137281293, "avg_output_tokens_per_task": 2609.127860026918, "last_run_timestamp": "2026-08-29T16:17:15.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.3822221666666667, "accuracy_std_dev": 0.15283338826307627, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.030608421666666673, "avg_duration_per_task_ms": 214180.72166666668, "avg_output_tokens_per_task": 2714.745, "last_run_timestamp": "2026-08-29T16:33:29.143Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "WandB", "accuracy": 0.38166665, "accuracy_std_dev": 0.02484398541151358, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.038916812999999995, "avg_duration_per_task_ms": 13463.68, "avg_output_tokens_per_task": 499.736, "last_run_timestamp": "2026-07-18T16:00:09.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.38106668, "accuracy_std_dev": 0.032258962827369383, "run_count": 25, "total_questions": 1250, "avg_cost_per_task_usd": 0.0164048448, "avg_duration_per_task_ms": 118964.3272, "avg_output_tokens_per_task": 2708.0272, "last_run_timestamp": "2026-08-29T16:10:20.675Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.3793939090909091, "accuracy_std_dev": 0.05088740122422613, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.039116065454545455, "avg_duration_per_task_ms": 58462.97090909091, "avg_output_tokens_per_task": 3651.5763636363636, "last_run_timestamp": "2026-08-29T16:25:50.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.3744443333333334, "accuracy_std_dev": 0.01940465134674902, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06861020666666667, "avg_duration_per_task_ms": 17591.08, "avg_output_tokens_per_task": 793.6233333333333, "last_run_timestamp": "2026-08-29T16:18:42.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.3725, "accuracy_std_dev": 0.09312108008170508, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.040488017499999994, "avg_duration_per_task_ms": 139617.32, "avg_output_tokens_per_task": 4754.3475, "last_run_timestamp": "2026-08-29T16:11:34.922Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Venice", "accuracy": 0.368, "accuracy_std_dev": 0.04489988864128728, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.04196784600000001, "avg_duration_per_task_ms": 61894.268, "avg_output_tokens_per_task": 2505.452, "last_run_timestamp": "2026-05-23T09:30:59.805Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "model_slug": "openai/gpt-4o-2024-05-13", "model_name": "OpenAI: GPT-4o (2024-05-13)", "provider_name": "Azure", "accuracy": 0.36666699999999997, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.80181, "avg_duration_per_task_ms": 63430.68, "avg_output_tokens_per_task": 3064.6, "last_run_timestamp": "2026-08-01T08:06:07.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Alibaba", "accuracy": 0.36075754545454547, "accuracy_std_dev": 0.053009762429135535, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.019241303636363637, "avg_duration_per_task_ms": 42582.33090909091, "avg_output_tokens_per_task": 2226.5054545454545, "last_run_timestamp": "2026-08-01T17:38:37.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.35989578125, "accuracy_std_dev": 0.027985955459943063, "run_count": 13, "total_questions": 640, "avg_cost_per_task_usd": 0.06471081249999999, "avg_duration_per_task_ms": 330465.9046875, "avg_output_tokens_per_task": 6852.8765625, "last_run_timestamp": "2026-09-19T08:08:41.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.34972216666666667, "accuracy_std_dev": 0.043118179904833125, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.023779785, "avg_duration_per_task_ms": 75866.685, "avg_output_tokens_per_task": 2981.369166666667, "last_run_timestamp": "2026-08-29T16:13:32.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.34503904682274245, "accuracy_std_dev": 0.03117646397380117, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.07262195986622073, "avg_duration_per_task_ms": 212425.83779264215, "avg_output_tokens_per_task": 6685.140468227425, "last_run_timestamp": "2026-09-19T08:12:53.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "Novita", "accuracy": 0.34058, "accuracy_std_dev": null, "run_count": 1, "total_questions": 46, "avg_cost_per_task_usd": 0.11353834782608695, "avg_duration_per_task_ms": 397172.5, "avg_output_tokens_per_task": 14067.434782608696, "last_run_timestamp": "2026-08-01T08:05:57.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Parasail", "accuracy": 0.33351874638487206, "accuracy_std_dev": 0.08113788331325603, "run_count": 18, "total_questions": 899, "avg_cost_per_task_usd": 0.02762981312569522, "avg_duration_per_task_ms": 64964.9043381535, "avg_output_tokens_per_task": 3953.5528364849833, "last_run_timestamp": "2026-07-25T08:07:19.515Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.3266667692307692, "accuracy_std_dev": 0.04280749195598978, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.07343648153846152, "avg_duration_per_task_ms": 129206.12923076923, "avg_output_tokens_per_task": 6940.74, "last_run_timestamp": "2026-09-19T08:13:24.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.3263781514683153, "accuracy_std_dev": 0.11733471265135623, "run_count": 26, "total_questions": 1294, "avg_cost_per_task_usd": 0.09428212982998455, "avg_duration_per_task_ms": 283568.8972179289, "avg_output_tokens_per_task": 5857.006182380216, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "Novita", "accuracy": 0.3262745294117647, "accuracy_std_dev": 0.06326493483766375, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.010698870588235293, "avg_duration_per_task_ms": 44460.25529411765, "avg_output_tokens_per_task": 1664.0011764705882, "last_run_timestamp": "2026-07-01T01:59:56.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.3262120909090909, "accuracy_std_dev": 0.04420222727719117, "run_count": 22, "total_questions": 1100, "avg_cost_per_task_usd": 0.019649248181818187, "avg_duration_per_task_ms": 67216.11454545455, "avg_output_tokens_per_task": 2692.0563636363636, "last_run_timestamp": "2026-08-01T17:38:36.457Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.31547627678571427, "accuracy_std_dev": 0.18940222003098767, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.11378589732142856, "avg_duration_per_task_ms": 450016.14285714284, "avg_output_tokens_per_task": 4166.011160714285, "last_run_timestamp": "2026-09-19T08:15:10.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.3133333, "accuracy_std_dev": 0.067462010709065, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.006098155, "avg_duration_per_task_ms": 65881.432, "avg_output_tokens_per_task": 1526.893, "last_run_timestamp": "2026-08-22T16:04:58.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Fireworks", "accuracy": 0.29714285714285715, "accuracy_std_dev": 0.1097678440702774, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.16768704, "avg_duration_per_task_ms": 485101.77714285714, "avg_output_tokens_per_task": 17757.942857142858, "last_run_timestamp": "2026-05-09T17:11:48.691Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "model_slug": "moonshotai/kimi-k2-0905", "model_name": "MoonshotAI: Kimi K2 0905", "provider_name": "Groq", "accuracy": 0.29666649999999994, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.05697704, "avg_duration_per_task_ms": 17166.81, "avg_output_tokens_per_task": 1588.56, "last_run_timestamp": "2026-07-04T16:00:20.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.29243700336134454, "accuracy_std_dev": 0.037781726827337526, "run_count": 24, "total_questions": 1190, "avg_cost_per_task_usd": 0.012292368067226894, "avg_duration_per_task_ms": 153931.20336134452, "avg_output_tokens_per_task": 2026.116806722689, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.29205126923076924, "accuracy_std_dev": 0.03702711278089413, "run_count": 26, "total_questions": 1300, "avg_cost_per_task_usd": 0.09791408615384616, "avg_duration_per_task_ms": 63451.20538461539, "avg_output_tokens_per_task": 1227.14, "last_run_timestamp": "2026-08-29T17:05:59.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.2862745294117647, "accuracy_std_dev": 0.022107336631874436, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.006061691764705881, "avg_duration_per_task_ms": 76495.49647058823, "avg_output_tokens_per_task": 1836.8976470588236, "last_run_timestamp": "2026-07-11T08:03:53.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Chutes", "accuracy": 0.2733333333333333, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09871155999999999, "avg_duration_per_task_ms": 1053542.8, "avg_output_tokens_per_task": 16485.386666666665, "last_run_timestamp": "2026-05-02T21:11:19.074Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.27, "accuracy_std_dev": 0.03, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02140288, "avg_duration_per_task_ms": 65620.09, "avg_output_tokens_per_task": 3949.38, "last_run_timestamp": "2026-09-01T17:29:05.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.266667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02052575, "avg_duration_per_task_ms": 64186.84, "avg_output_tokens_per_task": 3663.19, "last_run_timestamp": "2026-09-01T17:31:47.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.26041658333333334, "accuracy_std_dev": 0.01347799999999999, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.07850963541666667, "avg_duration_per_task_ms": 588191.84375, "avg_output_tokens_per_task": 5667.5625, "last_run_timestamp": "2026-09-01T12:59:55.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.25, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08593323, "avg_duration_per_task_ms": 547480.87, "avg_output_tokens_per_task": 5216.43, "last_run_timestamp": "2026-09-01T12:57:03.395Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Friendli", "accuracy": 0.2375, "accuracy_std_dev": 0.030720514318611274, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0706259825, "avg_duration_per_task_ms": 47810.7625, "avg_output_tokens_per_task": 9708.8825, "last_run_timestamp": "2026-05-16T08:51:28.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Nebius", "accuracy": 0.22, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.5653728, "avg_duration_per_task_ms": 371898.36, "avg_output_tokens_per_task": 8626.18, "last_run_timestamp": "2026-04-18T22:28:10.840Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "WandB", "accuracy": 0.215, "accuracy_std_dev": 0.05361902647381805, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.08529526375, "avg_duration_per_task_ms": 35242.32625, "avg_output_tokens_per_task": 1680.2625, "last_run_timestamp": "2026-06-20T08:23:22.767Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.19841299319727893, "accuracy_std_dev": 0.016383133956060492, "run_count": 3, "total_questions": 147, "avg_cost_per_task_usd": 0.011659061224489795, "avg_duration_per_task_ms": 134707.1224489796, "avg_output_tokens_per_task": 5676.0204081632655, "last_run_timestamp": "2026-09-01T13:00:50.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "auto-routing", "accuracy": 0.18712804910714284, "accuracy_std_dev": 0.063489276040705, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.019668446428571427, "avg_duration_per_task_ms": 108649.33705357143, "avg_output_tokens_per_task": 2703.4866071428573, "last_run_timestamp": "2026-07-18T08:04:09.749Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-nemo", "model_slug": "mistralai/mistral-nemo", "model_name": "Mistral: Mistral Nemo", "provider_name": "Mistral", "accuracy": 0.17888888888888888, "accuracy_std_dev": 0.06174408353742521, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021073126666666667, "avg_duration_per_task_ms": 119786.34888888888, "avg_output_tokens_per_task": 2837.891111111111, "last_run_timestamp": "2026-07-18T08:04:25.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "Phala", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01877496, "avg_duration_per_task_ms": 63137.36, "avg_output_tokens_per_task": 2797.32, "last_run_timestamp": "2026-08-01T08:06:16.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "model_slug": "qwen/qwen-2.5-7b-instruct", "model_name": "Qwen: Qwen2.5 7B Instruct", "provider_name": "auto-routing", "accuracy": 0.166667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.016978299999999998, "avg_duration_per_task_ms": 57790.52, "avg_output_tokens_per_task": 2492.08, "last_run_timestamp": "2026-08-01T08:06:07.057Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Chutes", "accuracy": 0.155, "accuracy_std_dev": 0.043301270189221926, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015117625, "avg_duration_per_task_ms": 773376.1, "avg_output_tokens_per_task": 13813.88, "last_run_timestamp": "2026-05-02T10:35:28.048Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.1366665, "accuracy_std_dev": 0.003333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00694106, "avg_duration_per_task_ms": 56167.4, "avg_output_tokens_per_task": 2292.83, "last_run_timestamp": "2026-09-01T17:31:40.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.125, "accuracy_std_dev": 0.020750046855368784, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.00834746, "avg_duration_per_task_ms": 82234.105, "avg_output_tokens_per_task": 2517.12, "last_run_timestamp": "2026-09-01T17:31:43.416Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.1033335, "accuracy_std_dev": 0.003333499999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0070183400000000005, "avg_duration_per_task_ms": 61321.36, "avg_output_tokens_per_task": 2324.09, "last_run_timestamp": "2026-09-01T17:31:46.301Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.033333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0007856200000000001, "avg_duration_per_task_ms": 1400.1, "avg_output_tokens_per_task": 126.5, "last_run_timestamp": "2026-06-30T19:06:38.991Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.02, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19539478, "avg_duration_per_task_ms": 618160.14, "avg_output_tokens_per_task": 1815.22, "last_run_timestamp": "2026-04-18T10:55:30.513Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.013333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00098206, "avg_duration_per_task_ms": 2461.54, "avg_output_tokens_per_task": 61.9, "last_run_timestamp": "2026-07-01T01:58:01.413Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "AtlasCloud", "accuracy": 0.006667000000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00013588, "avg_duration_per_task_ms": 446, "avg_output_tokens_per_task": 9.72, "last_run_timestamp": "2026-06-30T19:02:54.924Z" } ] }