{ "dataset": "benchmark-leaderboard", "title": "OpenRouter benchmark leaderboard", "description": "One row per benchmark, model version, and provider: the aggregate values displayed on the OpenRouter leaderboard pages (GPQA Diamond, tau2-bench airline). Derived from the benchmark-runs dataset.", "schema_version": "v1", "snapshot_date": "2026-11-14", "retrieved_at": "2026-11-14T01:46:03.748Z", "update_cadence": "daily", "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "attribution": "Source: OpenRouter (openrouter.ai), licensed under CC BY 4.0: reuse and republish with attribution to OpenRouter.", "harness": { "name": "OpenRouter benchmark harness", "url": "https://github.com/OpenRouterTeam/benchmark-harness" }, "source_pages": [ "https://openrouter.ai/benchmarks/gpqa-diamond", "https://openrouter.ai/benchmarks/tau2-bench-airline" ], "methodology": "Rows aggregate the benchmark-runs dataset exactly as the leaderboard pages do, over each benchmark's non-zero-score runs from the newest 90 days: accuracy is the task-weighted mean of per-run accuracy, accuracy_std_dev is the population standard deviation of per-run accuracy (null with a single run), and the per-task averages divide summed cost, generation time, and output tokens by summed tasks. Rows are ordered by accuracy descending, then model name.", "readme_url": "https://openrouter.ai/api/v1/datasets/exports/README.md", "columns": [ { "name": "benchmark", "type": "string", "nullable": false, "description": "Benchmark identifier." }, { "name": "benchmark_page_url", "type": "string", "nullable": false, "description": "openrouter.ai leaderboard page that displays this row." }, { "name": "model_permaslug", "type": "string", "nullable": false, "description": "Immutable OpenRouter model version identifier." }, { "name": "model_slug", "type": "string", "nullable": true, "description": "Current OpenRouter model slug, null when the version is no longer listed." }, { "name": "model_name", "type": "string", "nullable": false, "description": "Display name as shown on the leaderboard page." }, { "name": "provider_name", "type": "string", "nullable": false, "description": "Inference provider that served the aggregated runs." }, { "name": "accuracy", "type": "number", "nullable": false, "description": "Task-weighted mean accuracy across runs, 0 to 1." }, { "name": "accuracy_std_dev", "type": "number", "nullable": true, "description": "Population standard deviation of per-run accuracy; null when only one run exists." }, { "name": "run_count", "type": "integer", "nullable": false, "description": "Number of runs aggregated." }, { "name": "total_questions", "type": "integer", "nullable": false, "description": "Tasks attempted across all aggregated runs (sample size)." }, { "name": "avg_cost_per_task_usd", "type": "number", "nullable": true, "description": "Summed run cost divided by summed tasks, in USD." }, { "name": "avg_duration_per_task_ms", "type": "number", "nullable": true, "description": "Summed generation time divided by summed tasks, in milliseconds." }, { "name": "avg_output_tokens_per_task", "type": "number", "nullable": true, "description": "Summed completion tokens divided by summed tasks." }, { "name": "last_run_timestamp", "type": "string", "nullable": false, "description": "ISO 8601 UTC timestamp of the most recent aggregated run." } ], "row_count": 2275, "rows": [ { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.964646, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.09468125757575757, "avg_duration_per_task_ms": 101980.28282828283, "avg_output_tokens_per_task": 8439.414141414141, "last_run_timestamp": "2026-10-14T15:00:03.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.95637, "accuracy_std_dev": null, "run_count": 1, "total_questions": 191, "avg_cost_per_task_usd": 0.06822796858638744, "avg_duration_per_task_ms": 132197.8062827225, "avg_output_tokens_per_task": 18049.068062827224, "last_run_timestamp": "2026-10-01T08:31:33.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.954545, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.332555398989899, "avg_duration_per_task_ms": 75446.5505050505, "avg_output_tokens_per_task": 4530.388888888889, "last_run_timestamp": "2026-10-01T08:27:35.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.952862, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07423223737373738, "avg_duration_per_task_ms": 86748.28787878787, "avg_output_tokens_per_task": 19646.409090909092, "last_run_timestamp": "2026-10-01T08:29:11.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.95202, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20364760606060606, "avg_duration_per_task_ms": 135706.18686868687, "avg_output_tokens_per_task": 16847.588383838385, "last_run_timestamp": "2026-10-01T19:14:26.157Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.9511785, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.33620659090909094, "avg_duration_per_task_ms": 70544.52272727272, "avg_output_tokens_per_task": 4588.830808080808, "last_run_timestamp": "2026-10-02T23:23:01.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.2-20261109", "model_slug": "anthropic/claude-fable-5.2", "model_name": "Anthropic: Claude Fable 5.2", "provider_name": "Anthropic", "accuracy": 0.9511780000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08733781818181818, "avg_duration_per_task_ms": 25865.449494949495, "avg_output_tokens_per_task": 1942.8484848484848, "last_run_timestamp": "2026-11-10T18:01:45.629Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.9511779999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20247136363636364, "avg_duration_per_task_ms": 137578.30555555556, "avg_output_tokens_per_task": 16748.56818181818, "last_run_timestamp": "2026-10-01T19:12:42.155Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.2-20261109", "model_slug": "anthropic/claude-fable-5.2", "model_name": "Anthropic: Claude Fable 5.2", "provider_name": "Azure", "accuracy": 0.949495, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08434266666666666, "avg_duration_per_task_ms": 25783.79797979798, "avg_output_tokens_per_task": 1867.969696969697, "last_run_timestamp": "2026-11-10T18:03:17.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.949495, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.35495546717171716, "avg_duration_per_task_ms": 72102.17676767676, "avg_output_tokens_per_task": 4622.60101010101, "last_run_timestamp": "2026-10-01T08:29:03.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.9482614885496183, "accuracy_std_dev": 0.0012429999999999941, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.07536056743002545, "avg_duration_per_task_ms": 132814.94147582698, "avg_output_tokens_per_task": 18975.51145038168, "last_run_timestamp": "2026-10-15T21:00:04.244Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.947917, "accuracy_std_dev": null, "run_count": 1, "total_questions": 96, "avg_cost_per_task_usd": 0.009331666666666667, "avg_duration_per_task_ms": 232878.71875, "avg_output_tokens_per_task": 10278.75, "last_run_timestamp": "2026-10-14T18:00:04.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.947811, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.021484267676767678, "avg_duration_per_task_ms": 55623.856060606064, "avg_output_tokens_per_task": 1890.328282828283, "last_run_timestamp": "2026-10-30T00:03:33.414Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.9472503333333333, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11162585016835018, "avg_duration_per_task_ms": 48234.978114478115, "avg_output_tokens_per_task": 2076.191919191919, "last_run_timestamp": "2026-10-02T23:16:05.967Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.2-20261109", "model_slug": "anthropic/claude-fable-5.2", "model_name": "Anthropic: Claude Fable 5.2", "provider_name": "Google", "accuracy": 0.9461279999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08508327272727273, "avg_duration_per_task_ms": 22398.060606060608, "avg_output_tokens_per_task": 1886.4848484848485, "last_run_timestamp": "2026-11-10T18:01:46.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.9461276666666666, "accuracy_std_dev": 0.001374572014200141, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.1181814292929293, "avg_duration_per_task_ms": 36670.56565656565, "avg_output_tokens_per_task": 2059.811447811448, "last_run_timestamp": "2026-10-30T00:02:55.917Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.945286, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20203182828282829, "avg_duration_per_task_ms": 133595.55303030304, "avg_output_tokens_per_task": 16712.858585858587, "last_run_timestamp": "2026-10-01T19:16:14.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.9450355212765958, "accuracy_std_dev": 0.00181198405683448, "run_count": 3, "total_questions": 564, "avg_cost_per_task_usd": 0.030139015957446804, "avg_duration_per_task_ms": 60727.07978723404, "avg_output_tokens_per_task": 7608.468085106383, "last_run_timestamp": "2026-10-15T21:01:11.190Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11333201010101009, "avg_duration_per_task_ms": 57056.792929292926, "avg_output_tokens_per_task": 2108.121212121212, "last_run_timestamp": "2026-10-01T08:25:16.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.944444, "accuracy_std_dev": 0.0027491439879836154, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020607563973063972, "avg_duration_per_task_ms": 47519.37542087542, "avg_output_tokens_per_task": 1902.2373737373737, "last_run_timestamp": "2026-10-02T23:20:22.253Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.944444, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02136650505050505, "avg_duration_per_task_ms": 52411.32323232323, "avg_output_tokens_per_task": 1978.1313131313132, "last_run_timestamp": "2026-10-01T08:24:51.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.9431713315363881, "accuracy_std_dev": 0.006147978549897494, "run_count": 4, "total_questions": 742, "avg_cost_per_task_usd": 0.10982969407008086, "avg_duration_per_task_ms": 117613.10242587602, "avg_output_tokens_per_task": 1949.2371967654988, "last_run_timestamp": "2026-10-01T08:26:36.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google AI Studio", "accuracy": 0.941077, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.030745075757575756, "avg_duration_per_task_ms": 30416.38888888889, "avg_output_tokens_per_task": 8050.770202020202, "last_run_timestamp": "2026-10-01T16:16:03.631Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.941077, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.15125092424242426, "avg_duration_per_task_ms": 97603.77272727272, "avg_output_tokens_per_task": 11559.585858585859, "last_run_timestamp": "2026-10-01T16:10:33.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Nebius", "accuracy": 0.939394, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06875348484848484, "avg_duration_per_task_ms": 279532.6212121212, "avg_output_tokens_per_task": 6669.813131313132, "last_run_timestamp": "2026-11-04T21:00:08.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.939394, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.31252995454545457, "avg_duration_per_task_ms": 150832.55050505052, "avg_output_tokens_per_task": 10288.712121212122, "last_run_timestamp": "2026-10-01T19:12:11.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.939394, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.32760574494949496, "avg_duration_per_task_ms": 78455.45202020202, "avg_output_tokens_per_task": 11977.780303030304, "last_run_timestamp": "2026-10-01T16:36:19.657Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.9393935, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.15067959090909092, "avg_duration_per_task_ms": 95544.60353535354, "avg_output_tokens_per_task": 11515.699494949495, "last_run_timestamp": "2026-10-02T23:21:48.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Sail Research", "accuracy": 0.9387350223978919, "accuracy_std_dev": 0.0014398627538762172, "run_count": 4, "total_questions": 759, "avg_cost_per_task_usd": 0.14429597496706192, "avg_duration_per_task_ms": 118270.40579710146, "avg_output_tokens_per_task": 6705.238471673254, "last_run_timestamp": "2026-11-06T22:04:38.207Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "auto-routing", "accuracy": 0.938528, "accuracy_std_dev": 0.004357499999999959, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.0293307974025974, "avg_duration_per_task_ms": 59248.22337662338, "avg_output_tokens_per_task": 7683.532467532467, "last_run_timestamp": "2026-10-01T16:17:32.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.9382713333333333, "accuracy_std_dev": 0.004627357405210382, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.022108563973063967, "avg_duration_per_task_ms": 32928.50084175084, "avg_output_tokens_per_task": 1914.9974747474748, "last_run_timestamp": "2026-10-01T08:40:50.203Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.9368683333333334, "accuracy_std_dev": 0.0040948247690089186, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.3309691094276094, "avg_duration_per_task_ms": 165124.49915824915, "avg_output_tokens_per_task": 10209.419191919193, "last_run_timestamp": "2026-10-01T19:14:07.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "River", "accuracy": 0.936027, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08390751515151515, "avg_duration_per_task_ms": 94274.73232323233, "avg_output_tokens_per_task": 8182.020202020202, "last_run_timestamp": "2026-11-13T06:01:36.932Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.9360266666666668, "accuracy_std_dev": 0.0036369299629708854, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.3141816919191919, "avg_duration_per_task_ms": 142474.1195286195, "avg_output_tokens_per_task": 10342.412457912458, "last_run_timestamp": "2026-10-02T23:25:41.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Io Net", "accuracy": 0.9348024214876033, "accuracy_std_dev": 0.0015875000000000195, "run_count": 2, "total_questions": 363, "avg_cost_per_task_usd": 0.11736377685950414, "avg_duration_per_task_ms": 170976.24517906335, "avg_output_tokens_per_task": 7983.9201101928375, "last_run_timestamp": "2026-11-11T05:45:32.564Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.9335015000000001, "accuracy_std_dev": 0.000841499999999995, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.04714142045454545, "avg_duration_per_task_ms": 67600.7297979798, "avg_output_tokens_per_task": 11818.68813131313, "last_run_timestamp": "2026-10-01T19:12:42.153Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Decart", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.04747170707070707, "avg_duration_per_task_ms": 97339.0303030303, "avg_output_tokens_per_task": 7164.717171717171, "last_run_timestamp": "2026-11-11T15:01:28.174Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "DigitalOcean", "accuracy": 0.93266, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06645131313131314, "avg_duration_per_task_ms": 52649.732323232325, "avg_output_tokens_per_task": 6433.60101010101, "last_run_timestamp": "2026-11-07T03:00:36.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Modal", "accuracy": 0.931818, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07295092424242425, "avg_duration_per_task_ms": 39086.47474747475, "avg_output_tokens_per_task": 7118.949494949495, "last_run_timestamp": "2026-11-04T21:00:08.524Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Parasail", "accuracy": 0.930976, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07401931818181819, "avg_duration_per_task_ms": 70607.26262626263, "avg_output_tokens_per_task": 7194.131313131313, "last_run_timestamp": "2026-11-04T21:00:08.540Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "InferenceNet", "accuracy": 0.9306844829721364, "accuracy_std_dev": 0.007658821616932981, "run_count": 5, "total_questions": 969, "avg_cost_per_task_usd": 0.0836984004127967, "avg_duration_per_task_ms": 71158.75335397317, "avg_output_tokens_per_task": 7439.702786377709, "last_run_timestamp": "2026-11-05T00:00:09.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.9301345000000001, "accuracy_std_dev": 0.006832688313053163, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.026123152076318747, "avg_duration_per_task_ms": 18695.293490460157, "avg_output_tokens_per_task": 2221.0524691358023, "last_run_timestamp": "2026-10-31T08:07:18.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9301345, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.044492378787878796, "avg_duration_per_task_ms": 69170.04040404041, "avg_output_tokens_per_task": 11715.780303030304, "last_run_timestamp": "2026-10-01T19:12:42.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.929854, "accuracy_std_dev": 0.006348890821762996, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.025367877946127944, "avg_duration_per_task_ms": 20154.14898989899, "avg_output_tokens_per_task": 2304.1734006734005, "last_run_timestamp": "2026-10-31T08:33:04.486Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Moonshot AI", "accuracy": 0.9292930000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10512464646464648, "avg_duration_per_task_ms": 151923.42676767678, "avg_output_tokens_per_task": 6756.267676767677, "last_run_timestamp": "2026-11-04T18:03:37.503Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "AkashML", "accuracy": 0.9292929999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08810357575757576, "avg_duration_per_task_ms": 85562.65151515152, "avg_output_tokens_per_task": 8602.494949494949, "last_run_timestamp": "2026-11-05T06:00:43.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Morph", "accuracy": 0.9292929999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11162276262626263, "avg_duration_per_task_ms": 129833.19191919192, "avg_output_tokens_per_task": 9183.469696969696, "last_run_timestamp": "2026-11-05T06:00:06.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.9290121666666666, "accuracy_std_dev": 0.0044978590987515684, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024844954545454545, "avg_duration_per_task_ms": 19502.118686868685, "avg_output_tokens_per_task": 2244.3215488215487, "last_run_timestamp": "2026-10-31T09:02:13.238Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.9287315, "accuracy_std_dev": 0.0045244150174359715, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.09770303535353536, "avg_duration_per_task_ms": 54704.919191919194, "avg_output_tokens_per_task": 3506.051346801347, "last_run_timestamp": "2026-10-01T19:42:48.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.928451, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.3476244823232323, "avg_duration_per_task_ms": 112460.52777777778, "avg_output_tokens_per_task": 10405.60101010101, "last_run_timestamp": "2026-10-01T19:13:06.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.928451, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12326839393939394, "avg_duration_per_task_ms": 361758.7878787879, "avg_output_tokens_per_task": 20322.222222222223, "last_run_timestamp": "2026-10-01T16:09:08.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.9282826, "accuracy_std_dev": 0.00626296053316642, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025195250505050507, "avg_duration_per_task_ms": 19633.72121212121, "avg_output_tokens_per_task": 2281.29898989899, "last_run_timestamp": "2026-10-31T08:09:30.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Tenstorrent", "accuracy": 0.9281247875, "accuracy_std_dev": 0.003304499999999988, "run_count": 2, "total_questions": 160, "avg_cost_per_task_usd": 0.01232809375, "avg_duration_per_task_ms": 251386.81875, "avg_output_tokens_per_task": 10178.75625, "last_run_timestamp": "2026-11-04T18:00:19.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.9276095000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04436604545454545, "avg_duration_per_task_ms": 61670.28282828283, "avg_output_tokens_per_task": 11683.290404040405, "last_run_timestamp": "2026-10-01T19:11:04.522Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Fireworks", "accuracy": 0.927609, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07610462626262626, "avg_duration_per_task_ms": 98167.92929292929, "avg_output_tokens_per_task": 7400.540404040404, "last_run_timestamp": "2026-11-05T15:00:03.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.9275846571787361, "accuracy_std_dev": 0.010407425048021134, "run_count": 13, "total_questions": 2389, "avg_cost_per_task_usd": 0.13203736626203433, "avg_duration_per_task_ms": 128986.54792800335, "avg_output_tokens_per_task": 8711.713269150272, "last_run_timestamp": "2026-10-31T08:39:06.851Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "PrimaLabs", "accuracy": 0.927326, "accuracy_std_dev": null, "run_count": 1, "total_questions": 172, "avg_cost_per_task_usd": 0.04753577906976744, "avg_duration_per_task_ms": 65225.773255813954, "avg_output_tokens_per_task": 5210.343023255814, "last_run_timestamp": "2026-11-06T15:43:35.998Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Phala", "accuracy": 0.9267675, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07679769191919193, "avg_duration_per_task_ms": 175516.4292929293, "avg_output_tokens_per_task": 7530.295454545455, "last_run_timestamp": "2026-11-05T18:00:48.183Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.9264870000000001, "accuracy_std_dev": 0.007815979699734826, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03737200673400674, "avg_duration_per_task_ms": 60770.02693602694, "avg_output_tokens_per_task": 3580.875420875421, "last_run_timestamp": "2026-10-02T23:20:36.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.9264075634856123, "accuracy_std_dev": 0.007984700291764953, "run_count": 33, "total_questions": 6151, "avg_cost_per_task_usd": 0.1738332020809624, "avg_duration_per_task_ms": 167157.2952365469, "avg_output_tokens_per_task": 9377.01739554544, "last_run_timestamp": "2026-10-31T08:52:06.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9260940999999999, "accuracy_std_dev": 0.00286181765491794, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.026339350505050502, "avg_duration_per_task_ms": 19433.036868686868, "avg_output_tokens_per_task": 2294.320202020202, "last_run_timestamp": "2026-10-31T08:17:29.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.9259260000000001, "accuracy_std_dev": 0.004610436378478734, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.10788488484848485, "avg_duration_per_task_ms": 187765.80707070706, "avg_output_tokens_per_task": 9499.632323232323, "last_run_timestamp": "2026-10-24T08:37:09.424Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "KT", "accuracy": 0.925926, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13830230303030303, "avg_duration_per_task_ms": 115013.06313131313, "avg_output_tokens_per_task": 9092.70202020202, "last_run_timestamp": "2026-11-11T01:44:59.803Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.9255047500000002, "accuracy_std_dev": 0.0055038172832589756, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.14877371590909091, "avg_duration_per_task_ms": 85367.24242424243, "avg_output_tokens_per_task": 15632.513888888889, "last_run_timestamp": "2026-10-01T19:13:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.9252448891402715, "accuracy_std_dev": 0.0069132275216017836, "run_count": 9, "total_questions": 1768, "avg_cost_per_task_usd": 0.10858752149321266, "avg_duration_per_task_ms": 188184.25339366516, "avg_output_tokens_per_task": 9458.231334841628, "last_run_timestamp": "2026-10-31T09:20:20.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Darkbloom", "accuracy": 0.925214, "accuracy_std_dev": null, "run_count": 1, "total_questions": 78, "avg_cost_per_task_usd": 0.0012695128205128204, "avg_duration_per_task_ms": 237789.78205128206, "avg_output_tokens_per_task": 6844.423076923077, "last_run_timestamp": "2026-10-30T01:38:27.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.9246375695652174, "accuracy_std_dev": 0.006863146440316182, "run_count": 7, "total_questions": 1380, "avg_cost_per_task_usd": 0.12292555942028985, "avg_duration_per_task_ms": 340480.1579710145, "avg_output_tokens_per_task": 19662.684057971015, "last_run_timestamp": "2026-10-01T16:09:31.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.924599700084246, "accuracy_std_dev": 0.006819818440072705, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.02751449789385004, "avg_duration_per_task_ms": 24957.139848357205, "avg_output_tokens_per_task": 2420.7127211457455, "last_run_timestamp": "2026-10-01T08:24:52.927Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.9243430301944727, "accuracy_std_dev": 0.011215611086765923, "run_count": 11, "total_questions": 1954, "avg_cost_per_task_usd": 0.14577470982599794, "avg_duration_per_task_ms": 113023.06345957011, "avg_output_tokens_per_task": 9570.704708290687, "last_run_timestamp": "2026-10-31T08:14:43.086Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.9243079168138753, "accuracy_std_dev": 0.02419971284017385, "run_count": 32, "total_questions": 6227, "avg_cost_per_task_usd": 0.1328945098763449, "avg_duration_per_task_ms": 158997.73036775333, "avg_output_tokens_per_task": 10041.62630480167, "last_run_timestamp": "2026-11-04T21:14:13.300Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Together", "accuracy": 0.9242420000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.09054113636363637, "avg_duration_per_task_ms": 151399.2323232323, "avg_output_tokens_per_task": 8895.065656565657, "last_run_timestamp": "2026-11-04T18:00:08.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.9242420000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03863523484848485, "avg_duration_per_task_ms": 63161.229797979795, "avg_output_tokens_per_task": 3708.2954545454545, "last_run_timestamp": "2026-10-01T19:12:28.573Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.9240912983587339, "accuracy_std_dev": 0.015687164131543977, "run_count": 10, "total_questions": 1706, "avg_cost_per_task_usd": 0.1119299531066823, "avg_duration_per_task_ms": 198230.14243845252, "avg_output_tokens_per_task": 8354.249120750294, "last_run_timestamp": "2026-10-31T08:14:43.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "River", "accuracy": 0.9236109999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 192, "avg_cost_per_task_usd": 0.15688300520833334, "avg_duration_per_task_ms": 120831.05208333333, "avg_output_tokens_per_task": 10253.458333333334, "last_run_timestamp": "2026-11-14T00:02:16.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Near AI", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02620280303030303, "avg_duration_per_task_ms": 592141.2247474748, "avg_output_tokens_per_task": 21737.315656565657, "last_run_timestamp": "2026-10-31T10:06:11.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.9234005000000001, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1436656515151515, "avg_duration_per_task_ms": 89621.648989899, "avg_output_tokens_per_task": 15839.800505050505, "last_run_timestamp": "2026-10-01T19:18:40.706Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.9232600924690181, "accuracy_std_dev": 0.007367183577716027, "run_count": 11, "total_questions": 2098, "avg_cost_per_task_usd": 0.13289319971401337, "avg_duration_per_task_ms": 111823.89990467111, "avg_output_tokens_per_task": 8724.990943755958, "last_run_timestamp": "2026-10-31T08:14:19.970Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DeepInfra", "accuracy": 0.9228069789473685, "accuracy_std_dev": 0.011126928806578514, "run_count": 3, "total_questions": 570, "avg_cost_per_task_usd": 0.00580749649122807, "avg_duration_per_task_ms": 218614.69122807018, "avg_output_tokens_per_task": 19521.97543859649, "last_run_timestamp": "2026-10-31T08:40:34.158Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "ParetoAlpha", "accuracy": 0.92256925, "accuracy_std_dev": 0.005618592528382903, "run_count": 5, "total_questions": 960, "avg_cost_per_task_usd": 0.148641165625, "avg_duration_per_task_ms": 385822.7, "avg_output_tokens_per_task": 9697.144791666668, "last_run_timestamp": "2026-10-31T08:38:36.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.9225589999999999, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14588472727272728, "avg_duration_per_task_ms": 63933.833333333336, "avg_output_tokens_per_task": 16085.368686868687, "last_run_timestamp": "2026-10-01T19:12:44.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Io Net", "accuracy": 0.9221647956015523, "accuracy_std_dev": 0.01297397580697607, "run_count": 5, "total_questions": 773, "avg_cost_per_task_usd": 0.013588241914618368, "avg_duration_per_task_ms": 479472.9379042691, "avg_output_tokens_per_task": 29097.7063389392, "last_run_timestamp": "2026-11-11T06:30:35.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9219976666666666, "accuracy_std_dev": 0.0028614651647170097, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.08309612457912459, "avg_duration_per_task_ms": 49364.952861952865, "avg_output_tokens_per_task": 3748.658249158249, "last_run_timestamp": "2026-10-30T00:00:06.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.921964588235294, "accuracy_std_dev": 0.007325561540883652, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.12449768538324421, "avg_duration_per_task_ms": 152742.1247771836, "avg_output_tokens_per_task": 11205.551396316103, "last_run_timestamp": "2026-11-06T22:04:48.119Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DigitalOcean", "accuracy": 0.9208753333333334, "accuracy_std_dev": 0.003636621356271357, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.019656516835016836, "avg_duration_per_task_ms": 90437.9696969697, "avg_output_tokens_per_task": 20830.57239057239, "last_run_timestamp": "2026-10-31T08:09:30.885Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.9208750000000001, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.026268675084175085, "avg_duration_per_task_ms": 25466.75420875421, "avg_output_tokens_per_task": 2308.39898989899, "last_run_timestamp": "2026-10-30T00:00:05.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.9208749999999999, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025590255892255893, "avg_duration_per_task_ms": 33502.22222222222, "avg_output_tokens_per_task": 2400.506734006734, "last_run_timestamp": "2026-10-02T23:17:42.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.9205386, "accuracy_std_dev": 0.004178401589124734, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.14947429494949496, "avg_duration_per_task_ms": 120588.33434343434, "avg_output_tokens_per_task": 9732.809090909092, "last_run_timestamp": "2026-10-31T08:09:30.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.9202018, "accuracy_std_dev": 0.01007849159150319, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.13160375303030303, "avg_duration_per_task_ms": 237248.802020202, "avg_output_tokens_per_task": 10658.989393939393, "last_run_timestamp": "2026-10-31T08:21:51.280Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.9199569999999999, "accuracy_std_dev": 0.006554291127331654, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.17177113406795225, "avg_duration_per_task_ms": 299167.57943067036, "avg_output_tokens_per_task": 9764.863636363636, "last_run_timestamp": "2026-10-31T08:33:15.782Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Wafer", "accuracy": 0.9191920000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07685085101010102, "avg_duration_per_task_ms": 125614, "avg_output_tokens_per_task": 7560.146464646465, "last_run_timestamp": "2026-11-05T21:03:19.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Relace", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.062148696969696965, "avg_duration_per_task_ms": 92710.41414141415, "avg_output_tokens_per_task": 7696.424242424242, "last_run_timestamp": "2026-11-05T00:00:09.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02528019191919192, "avg_duration_per_task_ms": 32477.28787878788, "avg_output_tokens_per_task": 2369.5, "last_run_timestamp": "2026-10-01T08:24:24.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.7-20260916", "model_slug": "x-ai/grok-4.7", "model_name": "SpaceXAI: Grok 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.9191919999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.21825907575757575, "avg_duration_per_task_ms": 803866.0202020202, "avg_output_tokens_per_task": 36106.61616161616, "last_run_timestamp": "2026-11-03T03:00:04.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.918981375, "accuracy_std_dev": 0.00679620392089402, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12805398863636364, "avg_duration_per_task_ms": 541652.8712121212, "avg_output_tokens_per_task": 37116.541666666664, "last_run_timestamp": "2026-10-03T09:05:48.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.9186305, "accuracy_std_dev": 0.0030220059425267055, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.012602135521885521, "avg_duration_per_task_ms": 20976.548821548822, "avg_output_tokens_per_task": 2132.777777777778, "last_run_timestamp": "2026-11-02T17:19:33.587Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "SiliconFlow", "accuracy": 0.9186303333333333, "accuracy_std_dev": 0.010408091158111372, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.025292042087542085, "avg_duration_per_task_ms": 160402.3451178451, "avg_output_tokens_per_task": 20957.713804713803, "last_run_timestamp": "2026-10-31T08:50:50.713Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.9177144177922683, "accuracy_std_dev": 0.013751154902061724, "run_count": 11, "total_questions": 2147, "avg_cost_per_task_usd": 0.2460820894271076, "avg_duration_per_task_ms": 409070.95482068, "avg_output_tokens_per_task": 37154.30647414998, "last_run_timestamp": "2026-10-31T08:34:22.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimaLabs", "accuracy": 0.9175825384615385, "accuracy_std_dev": 0.009135000000000004, "run_count": 2, "total_questions": 182, "avg_cost_per_task_usd": 0.03656871978021978, "avg_duration_per_task_ms": 44231.42857142857, "avg_output_tokens_per_task": 10074.60989010989, "last_run_timestamp": "2026-11-06T15:33:21.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.9175082499999999, "accuracy_std_dev": 0.003571399938516188, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.025937152356902356, "avg_duration_per_task_ms": 19174.19191919192, "avg_output_tokens_per_task": 2235.470117845118, "last_run_timestamp": "2026-10-31T08:20:35.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.9174755201839551, "accuracy_std_dev": 0.007025002493949755, "run_count": 10, "total_questions": 1957, "avg_cost_per_task_usd": 0.14208481297904957, "avg_duration_per_task_ms": 283983.31374552887, "avg_output_tokens_per_task": 9268.438426162493, "last_run_timestamp": "2026-10-24T08:44:42.687Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.916877387341772, "accuracy_std_dev": 0.005308941032719149, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.10198255443037975, "avg_duration_per_task_ms": 806625.7974683545, "avg_output_tokens_per_task": 39607.979746835445, "last_run_timestamp": "2026-10-31T16:24:40.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.9168302048835251, "accuracy_std_dev": 0.02695337897854574, "run_count": 18, "total_questions": 3563, "avg_cost_per_task_usd": 0.14734509009261856, "avg_duration_per_task_ms": 382336.6034240808, "avg_output_tokens_per_task": 9501.996070726915, "last_run_timestamp": "2026-10-31T09:12:10.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.9165900909090909, "accuracy_std_dev": 0.009424357296191685, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0340643388429752, "avg_duration_per_task_ms": 223570.00459136823, "avg_output_tokens_per_task": 28227.440771349862, "last_run_timestamp": "2026-10-31T16:05:09.530Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Chutes", "accuracy": 0.9164236554907678, "accuracy_std_dev": 0.014292148644484252, "run_count": 11, "total_questions": 2058, "avg_cost_per_task_usd": 0.1620518138969874, "avg_duration_per_task_ms": 492245.379494655, "avg_output_tokens_per_task": 11216.908163265307, "last_run_timestamp": "2026-10-31T08:37:30.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.9159186669815778, "accuracy_std_dev": 0.007589099806154146, "run_count": 11, "total_questions": 2117, "avg_cost_per_task_usd": 0.09760643316013226, "avg_duration_per_task_ms": 645928.4369390648, "avg_output_tokens_per_task": 39652.34246575343, "last_run_timestamp": "2026-11-10T18:49:16.069Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.9158379972375691, "accuracy_std_dev": 0.012441503983078046, "run_count": 13, "total_questions": 1810, "avg_cost_per_task_usd": 0.11735693314917128, "avg_duration_per_task_ms": 179712.15801104973, "avg_output_tokens_per_task": 8118.493370165746, "last_run_timestamp": "2026-10-31T08:07:17.446Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DekaLLM", "accuracy": 0.915825, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014663378787878788, "avg_duration_per_task_ms": 96926.35858585859, "avg_output_tokens_per_task": 21419.76767676768, "last_run_timestamp": "2026-11-04T09:00:09.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "DeepInfra", "accuracy": 0.915825, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06562552525252525, "avg_duration_per_task_ms": 514778.64646464644, "avg_output_tokens_per_task": 6703.651515151515, "last_run_timestamp": "2026-11-05T06:03:33.391Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.9153655454545453, "accuracy_std_dev": 0.009766163789176628, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1584063576675849, "avg_duration_per_task_ms": 611741.2534435262, "avg_output_tokens_per_task": 39713.49908172635, "last_run_timestamp": "2026-10-31T08:28:10.440Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.9149830000000001, "accuracy_std_dev": 0.009335323856192691, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.15949898674242427, "avg_duration_per_task_ms": 579778.1357323233, "avg_output_tokens_per_task": 39960.98863636364, "last_run_timestamp": "2026-10-31T08:49:58.581Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.9149829999999999, "accuracy_std_dev": 0.006208509031965745, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.035431615151515154, "avg_duration_per_task_ms": 289878.5762626263, "avg_output_tokens_per_task": 29385.477272727272, "last_run_timestamp": "2026-10-31T16:17:15.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "CoreWeave", "accuracy": 0.9149660000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.022535153061224493, "avg_duration_per_task_ms": 106164.53571428571, "avg_output_tokens_per_task": 18639.61224489796, "last_run_timestamp": "2026-11-04T18:00:08.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "Together", "accuracy": 0.9141410000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.03261640909090909, "avg_duration_per_task_ms": 98347.65151515152, "avg_output_tokens_per_task": 32384.540404040403, "last_run_timestamp": "2026-11-13T00:01:54.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "Nebius", "accuracy": 0.9141410000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.030628545454545457, "avg_duration_per_task_ms": 320020.58585858584, "avg_output_tokens_per_task": 30417.116161616163, "last_run_timestamp": "2026-11-13T00:01:53.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.9140508968609865, "accuracy_std_dev": 0.01852693073581796, "run_count": 7, "total_questions": 892, "avg_cost_per_task_usd": 0.02261122085201794, "avg_duration_per_task_ms": 825583.0426008969, "avg_output_tokens_per_task": 20985.829596412557, "last_run_timestamp": "2026-10-31T16:03:21.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.9133203374352331, "accuracy_std_dev": 0.03022029270209009, "run_count": 8, "total_questions": 1544, "avg_cost_per_task_usd": 0.04059680051813471, "avg_duration_per_task_ms": 413773.9656735751, "avg_output_tokens_per_task": 12145.259715025906, "last_run_timestamp": "2026-10-31T16:08:56.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Baidu", "accuracy": 0.91311096, "accuracy_std_dev": 0.009099497963486788, "run_count": 4, "total_questions": 750, "avg_cost_per_task_usd": 0.01717951733333333, "avg_duration_per_task_ms": 130330.4, "avg_output_tokens_per_task": 20403.964, "last_run_timestamp": "2026-11-10T18:06:00.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.9129648851570964, "accuracy_std_dev": 0.008195106272424223, "run_count": 14, "total_questions": 2769, "avg_cost_per_task_usd": 0.0985225175153485, "avg_duration_per_task_ms": 283575.7085590466, "avg_output_tokens_per_task": 25018.89021307331, "last_run_timestamp": "2026-10-31T08:40:32.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Alibaba", "accuracy": 0.91245775, "accuracy_std_dev": 0.005709247427419856, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.027895760101010105, "avg_duration_per_task_ms": 177535.13888888888, "avg_output_tokens_per_task": 21414.281565656565, "last_run_timestamp": "2026-10-31T08:15:24.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.911926105263158, "accuracy_std_dev": 0.013578898854258508, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.1411968139287613, "avg_duration_per_task_ms": 616719.149122807, "avg_output_tokens_per_task": 40116.86735778841, "last_run_timestamp": "2026-10-31T08:31:11.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "River", "accuracy": 0.911616, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009611237373737374, "avg_duration_per_task_ms": 119490.68686868687, "avg_output_tokens_per_task": 19858.734848484848, "last_run_timestamp": "2026-11-12T20:19:21.746Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Venice", "accuracy": 0.911616, "accuracy_std_dev": 0.004209000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.024379722222222223, "avg_duration_per_task_ms": 102982.82575757576, "avg_output_tokens_per_task": 20129.815656565657, "last_run_timestamp": "2026-10-31T08:34:49.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "GMICloud", "accuracy": 0.9114477999999999, "accuracy_std_dev": 0.0039265801609033885, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.018160035353535352, "avg_duration_per_task_ms": 129702.60808080809, "avg_output_tokens_per_task": 22106.812121212122, "last_run_timestamp": "2026-11-10T06:01:40.563Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "OpenAI", "accuracy": 0.910774, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.011462444444444444, "avg_duration_per_task_ms": 27557.444444444445, "avg_output_tokens_per_task": 2133.969696969697, "last_run_timestamp": "2026-10-28T00:00:04.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "PrimeIntellect", "accuracy": 0.910774, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01510519191919192, "avg_duration_per_task_ms": 254029.7676767677, "avg_output_tokens_per_task": 30029.237373737375, "last_run_timestamp": "2026-11-13T03:01:57.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "Cloudflare", "accuracy": 0.910774, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.030777131313131315, "avg_duration_per_task_ms": 94527.24747474748, "avg_output_tokens_per_task": 30575.68686868687, "last_run_timestamp": "2026-11-13T00:01:53.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.9106891772151898, "accuracy_std_dev": 0.008562817793226714, "run_count": 5, "total_questions": 948, "avg_cost_per_task_usd": 0.14278316033755276, "avg_duration_per_task_ms": 333972.91455696203, "avg_output_tokens_per_task": 37406.36181434599, "last_run_timestamp": "2026-10-31T08:31:11.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Azure", "accuracy": 0.9096519999999999, "accuracy_std_dev": 0.0075707498968068145, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.012606855218855219, "avg_duration_per_task_ms": 20423.260942760942, "avg_output_tokens_per_task": 2135.06734006734, "last_run_timestamp": "2026-11-03T15:00:04.730Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Modal", "accuracy": 0.909632, "accuracy_std_dev": 0.0055275000000000185, "run_count": 2, "total_questions": 308, "avg_cost_per_task_usd": 0.05270355519480519, "avg_duration_per_task_ms": 55285.37662337662, "avg_output_tokens_per_task": 12143.461038961039, "last_run_timestamp": "2026-10-31T08:15:03.291Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.9095115, "accuracy_std_dev": 0.002759842885020819, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0416229191919192, "avg_duration_per_task_ms": 138307.48484848486, "avg_output_tokens_per_task": 27969.503787878788, "last_run_timestamp": "2026-10-01T16:11:23.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.9094272, "accuracy_std_dev": 0.00547072746168187, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.025930728282828285, "avg_duration_per_task_ms": 177886.11212121212, "avg_output_tokens_per_task": 28121.050505050505, "last_run_timestamp": "2026-10-03T09:34:35.533Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StreamLake", "accuracy": 0.9090910000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018701613636363638, "avg_duration_per_task_ms": 137003.63383838383, "avg_output_tokens_per_task": 22059.484848484848, "last_run_timestamp": "2026-10-31T08:36:23.829Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.1305062777777778, "avg_duration_per_task_ms": 540279.6767676767, "avg_output_tokens_per_task": 38448.40404040404, "last_run_timestamp": "2026-10-01T09:14:23.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2221867676767677, "avg_duration_per_task_ms": 298883.5202020202, "avg_output_tokens_per_task": 29352.99494949495, "last_run_timestamp": "2026-09-27T00:06:19.757Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "Modal", "accuracy": 0.9090909999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02941411111111111, "avg_duration_per_task_ms": 293246.38383838383, "avg_output_tokens_per_task": 29214.535353535353, "last_run_timestamp": "2026-11-13T06:01:37.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.909090857142857, "accuracy_std_dev": 0.0075824593339510055, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.038180986291486294, "avg_duration_per_task_ms": 147767.6544011544, "avg_output_tokens_per_task": 28692.146464646463, "last_run_timestamp": "2026-11-10T06:07:30.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StepFun", "accuracy": 0.9090908333333333, "accuracy_std_dev": 0.007076201038614481, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01709530134680135, "avg_duration_per_task_ms": 84455.94696969698, "avg_output_tokens_per_task": 21725.516835016835, "last_run_timestamp": "2026-11-02T17:24:31.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.9083779032258064, "accuracy_std_dev": 0.022205862232752862, "run_count": 12, "total_questions": 2232, "avg_cost_per_task_usd": 0.1474674775985663, "avg_duration_per_task_ms": 555743.3785842294, "avg_output_tokens_per_task": 37926.89829749104, "last_run_timestamp": "2026-10-31T08:44:11.309Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.908249, "accuracy_std_dev": 0.006574437713599543, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.009681809974747475, "avg_duration_per_task_ms": 167110.30050505052, "avg_output_tokens_per_task": 18361.854166666668, "last_run_timestamp": "2026-11-04T21:05:58.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.9081554444444445, "accuracy_std_dev": 0.008876832231665518, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.033613874298540965, "avg_duration_per_task_ms": 210737.89562289562, "avg_output_tokens_per_task": 27019.509539842875, "last_run_timestamp": "2026-10-31T08:40:16.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.9080193636363636, "accuracy_std_dev": 0.0188735967823871, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.037270338383838385, "avg_duration_per_task_ms": 248010.31542699726, "avg_output_tokens_per_task": 31082.94582185491, "last_run_timestamp": "2026-10-31T16:09:19.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.9078681221719456, "accuracy_std_dev": 0.005280935005338756, "run_count": 7, "total_questions": 1326, "avg_cost_per_task_usd": 0.023841498491704374, "avg_duration_per_task_ms": 170556.5128205128, "avg_output_tokens_per_task": 27449.11161387632, "last_run_timestamp": "2026-11-10T18:05:49.922Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Mancer 2", "accuracy": 0.9078013574468085, "accuracy_std_dev": 0.012928926018815361, "run_count": 3, "total_questions": 470, "avg_cost_per_task_usd": 0.01807536170212766, "avg_duration_per_task_ms": 357786.7425531915, "avg_output_tokens_per_task": 17947.382978723403, "last_run_timestamp": "2026-10-31T09:05:48.501Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.9076879166666667, "accuracy_std_dev": 0.026811471259078443, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.036969556397306404, "avg_duration_per_task_ms": 266792.257996633, "avg_output_tokens_per_task": 30839.63425925926, "last_run_timestamp": "2026-10-31T16:23:41.132Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cohere", "accuracy": 0.9074074999999998, "accuracy_std_dev": 0.008417500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014508287878787877, "avg_duration_per_task_ms": 90086.95707070707, "avg_output_tokens_per_task": 21790.61111111111, "last_run_timestamp": "2026-10-31T08:13:11.036Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Nebius", "accuracy": 0.9074072499999999, "accuracy_std_dev": 0.004610545920766882, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.025605553030303027, "avg_duration_per_task_ms": 96285.60858585859, "avg_output_tokens_per_task": 21160.45580808081, "last_run_timestamp": "2026-10-31T09:42:32.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.9073293664700928, "accuracy_std_dev": 0.008220232321331707, "run_count": 12, "total_questions": 2374, "avg_cost_per_task_usd": 0.030960309604043806, "avg_duration_per_task_ms": 400875.2552653749, "avg_output_tokens_per_task": 32028.098146588036, "last_run_timestamp": "2026-10-31T16:32:36.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.9071887349785409, "accuracy_std_dev": 0.014345789366918764, "run_count": 5, "total_questions": 932, "avg_cost_per_task_usd": 0.12532361909871245, "avg_duration_per_task_ms": 1379721.1190987125, "avg_output_tokens_per_task": 49889.66094420601, "last_run_timestamp": "2026-10-31T08:27:43.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.9070474558880683, "accuracy_std_dev": 0.028977319190080154, "run_count": 13, "total_questions": 2573, "avg_cost_per_task_usd": 0.0985262642829382, "avg_duration_per_task_ms": 564503.555771473, "avg_output_tokens_per_task": 39384.385542168675, "last_run_timestamp": "2026-10-31T08:39:28.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Modal", "accuracy": 0.9070292653061225, "accuracy_std_dev": 0.005723067708842884, "run_count": 3, "total_questions": 588, "avg_cost_per_task_usd": 0.021270954081632654, "avg_duration_per_task_ms": 117754.6887755102, "avg_output_tokens_per_task": 21116.049319727892, "last_run_timestamp": "2026-10-31T08:51:31.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.9062928608176387, "accuracy_std_dev": 0.051292972174679034, "run_count": 11, "total_questions": 2177, "avg_cost_per_task_usd": 0.13471541892512628, "avg_duration_per_task_ms": 314455.0349104272, "avg_output_tokens_per_task": 10081.463481855764, "last_run_timestamp": "2026-10-31T09:37:23.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Fireworks", "accuracy": 0.9062632644787644, "accuracy_std_dev": 0.007463126815174401, "run_count": 8, "total_questions": 1554, "avg_cost_per_task_usd": 0.024727387387387386, "avg_duration_per_task_ms": 89200.08815958817, "avg_output_tokens_per_task": 20548.897683397685, "last_run_timestamp": "2026-10-31T08:46:47.079Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.9061779082526508, "accuracy_std_dev": 0.011623858653648528, "run_count": 11, "total_questions": 2169, "avg_cost_per_task_usd": 0.026847682342093126, "avg_duration_per_task_ms": 336788.01567542646, "avg_output_tokens_per_task": 22212.46196403873, "last_run_timestamp": "2026-10-31T16:12:28.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.9061447499999998, "accuracy_std_dev": 0.005102889738912664, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.007892248737373738, "avg_duration_per_task_ms": 123987.29040404041, "avg_output_tokens_per_task": 16306.166666666666, "last_run_timestamp": "2026-10-31T08:53:36.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.9060041666666666, "accuracy_std_dev": 0.006179339377222198, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.032878036195286196, "avg_duration_per_task_ms": 809018.2592592592, "avg_output_tokens_per_task": 37448.21212121212, "last_run_timestamp": "2026-10-31T08:23:27.402Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.9059826923076922, "accuracy_std_dev": 0.02531585573200691, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.03357483760683761, "avg_duration_per_task_ms": 222969.2094017094, "avg_output_tokens_per_task": 27670.721056721057, "last_run_timestamp": "2026-11-10T06:30:46.502Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.9059684472128378, "accuracy_std_dev": 0.029240642852196717, "run_count": 12, "total_questions": 2368, "avg_cost_per_task_usd": 0.15467415033783785, "avg_duration_per_task_ms": 833998.046875, "avg_output_tokens_per_task": 38756.0152027027, "last_run_timestamp": "2026-10-31T08:15:03.301Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Decart", "accuracy": 0.9059342499999998, "accuracy_std_dev": 0.010910172663874526, "run_count": 3, "total_questions": 528, "avg_cost_per_task_usd": 0.008819143939393941, "avg_duration_per_task_ms": 163195.54924242425, "avg_output_tokens_per_task": 22179.03787878788, "last_run_timestamp": "2026-10-31T08:49:09.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.9057238888888889, "accuracy_std_dev": 0.009356618102764875, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03432713860830528, "avg_duration_per_task_ms": 196757.96408529743, "avg_output_tokens_per_task": 28444.87317620651, "last_run_timestamp": "2026-10-31T08:29:56.453Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.9057234999999999, "accuracy_std_dev": 0.0029159075345422127, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013535670454545456, "avg_duration_per_task_ms": 88017.25757575757, "avg_output_tokens_per_task": 21462.573232323233, "last_run_timestamp": "2026-11-01T12:00:08.725Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.9055437090526317, "accuracy_std_dev": 0.022030260791818957, "run_count": 12, "total_questions": 2375, "avg_cost_per_task_usd": 0.12441664378947369, "avg_duration_per_task_ms": 324658.0492631579, "avg_output_tokens_per_task": 8521.422736842105, "last_run_timestamp": "2026-10-31T08:25:00.194Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.9054404524137931, "accuracy_std_dev": 0.015780138865932054, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.10642273195402298, "avg_duration_per_task_ms": 537964.4225287356, "avg_output_tokens_per_task": 37458.18114942529, "last_run_timestamp": "2026-10-31T08:11:08.576Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Makora", "accuracy": 0.9054373465045593, "accuracy_std_dev": 0.0037554479359991244, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.026207787234042552, "avg_duration_per_task_ms": 127141.36068895644, "avg_output_tokens_per_task": 21741.29989868288, "last_run_timestamp": "2026-10-31T08:31:11.848Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.9053869333333332, "accuracy_std_dev": 0.006878297322900644, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.07875497138047138, "avg_duration_per_task_ms": 227261.18787878787, "avg_output_tokens_per_task": 31017.457912457914, "last_run_timestamp": "2026-10-31T16:10:09.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.905376, "accuracy_std_dev": null, "run_count": 1, "total_questions": 155, "avg_cost_per_task_usd": 0.09219292258064517, "avg_duration_per_task_ms": 892432.9419354838, "avg_output_tokens_per_task": 36790.02580645162, "last_run_timestamp": "2026-10-02T06:44:44.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.9053496666666667, "accuracy_std_dev": 0.008612696635394351, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.02068693490460157, "avg_duration_per_task_ms": 208793.50953984287, "avg_output_tokens_per_task": 28854.00224466891, "last_run_timestamp": "2026-10-31T09:01:06.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Sail Research", "accuracy": 0.9053028750000003, "accuracy_std_dev": 0.00682546092651441, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0031826717171717163, "avg_duration_per_task_ms": 242276.39330808082, "avg_output_tokens_per_task": 19735.85101010101, "last_run_timestamp": "2026-11-06T21:57:38.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.9049724513274336, "accuracy_std_dev": 0.008906991898924113, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.020146505689001264, "avg_duration_per_task_ms": 644404.02528445, "avg_output_tokens_per_task": 42703.32616940582, "last_run_timestamp": "2026-10-28T03:59:43.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Wafer", "accuracy": 0.9048818749999998, "accuracy_std_dev": 0.008666358814945001, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.008252799242424242, "avg_duration_per_task_ms": 82042.59659090909, "avg_output_tokens_per_task": 16383.162878787878, "last_run_timestamp": "2026-10-31T08:56:01.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Makora", "accuracy": 0.9048634068157615, "accuracy_std_dev": 0.012266915351464668, "run_count": 5, "total_questions": 939, "avg_cost_per_task_usd": 0.06399653567625134, "avg_duration_per_task_ms": 75580.11395101172, "avg_output_tokens_per_task": 14917.831735889244, "last_run_timestamp": "2026-10-31T08:13:57.643Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.9046013333333334, "accuracy_std_dev": 0.009152126540986068, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02685950505050505, "avg_duration_per_task_ms": 136981.2441077441, "avg_output_tokens_per_task": 22236.84175084175, "last_run_timestamp": "2026-10-31T08:31:11.499Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "AtlasCloud", "accuracy": 0.9046013333333334, "accuracy_std_dev": 0.008285549120138134, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01933321717171717, "avg_duration_per_task_ms": 139538.02861952863, "avg_output_tokens_per_task": 20900.63973063973, "last_run_timestamp": "2026-10-31T08:20:08.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.9044004522003033, "accuracy_std_dev": 0.00856689234728672, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.014848743045017704, "avg_duration_per_task_ms": 370376.5558927668, "avg_output_tokens_per_task": 28855.582195245323, "last_run_timestamp": "2026-10-31T08:03:09.643Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Mistral", "accuracy": 0.904321, "accuracy_std_dev": 0.006024476685433623, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.025462516835016838, "avg_duration_per_task_ms": 81189.70454545454, "avg_output_tokens_per_task": 21063.152356902356, "last_run_timestamp": "2026-11-11T02:59:16.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.9042355953038675, "accuracy_std_dev": 0.04114912403373615, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.1376939341620626, "avg_duration_per_task_ms": 212484.72605893185, "avg_output_tokens_per_task": 9252.047421731124, "last_run_timestamp": "2026-10-31T08:40:21.641Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.9042316703786192, "accuracy_std_dev": 0.0031656583896560934, "run_count": 5, "total_questions": 898, "avg_cost_per_task_usd": 0.22350682405345215, "avg_duration_per_task_ms": 441723.66035634745, "avg_output_tokens_per_task": 42286.14922048998, "last_run_timestamp": "2026-10-31T08:44:55.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.9041981924485772, "accuracy_std_dev": 0.01113877499359892, "run_count": 18, "total_questions": 3549, "avg_cost_per_task_usd": 0.027435362073823615, "avg_duration_per_task_ms": 185052.6246830093, "avg_output_tokens_per_task": 25932.813750352212, "last_run_timestamp": "2026-10-31T08:28:13.141Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.9040405, "accuracy_std_dev": 0.011784499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.13174327272727274, "avg_duration_per_task_ms": 249918.9898989899, "avg_output_tokens_per_task": 43791.378787878784, "last_run_timestamp": "2026-10-01T19:18:35.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.9037997142857144, "accuracy_std_dev": 0.016282944378479363, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.12103858297258295, "avg_duration_per_task_ms": 424148.33405483403, "avg_output_tokens_per_task": 37524.782106782106, "last_run_timestamp": "2026-10-31T16:10:36.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.903508802631579, "accuracy_std_dev": 0.0984941779678147, "run_count": 3, "total_questions": 456, "avg_cost_per_task_usd": 0.11285829166666667, "avg_duration_per_task_ms": 423954.7916666667, "avg_output_tokens_per_task": 16846.184210526317, "last_run_timestamp": "2026-10-01T16:04:59.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.903428, "accuracy_std_dev": 0.020115933056342978, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.12324611707988982, "avg_duration_per_task_ms": 520523.87649219466, "avg_output_tokens_per_task": 38776.62029384757, "last_run_timestamp": "2026-10-31T08:17:28.484Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimeIntellect", "accuracy": 0.9033668, "accuracy_std_dev": 0.005061737504059227, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.026716864646464646, "avg_duration_per_task_ms": 106368.8292929293, "avg_output_tokens_per_task": 22085.938383838384, "last_run_timestamp": "2026-10-31T08:13:57.548Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Particle AI", "accuracy": 0.9031985, "accuracy_std_dev": 0.0075759444460740165, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01504592297979798, "avg_duration_per_task_ms": 86737.82702020202, "avg_output_tokens_per_task": 17203.179292929293, "last_run_timestamp": "2026-11-03T01:49:06.293Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.9031985, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14226843434343436, "avg_duration_per_task_ms": 149341.0782828283, "avg_output_tokens_per_task": 9356.287878787878, "last_run_timestamp": "2026-10-01T19:14:55.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Together", "accuracy": 0.9029179999999999, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.023974107744107742, "avg_duration_per_task_ms": 50011.877104377105, "avg_output_tokens_per_task": 19809.430976430976, "last_run_timestamp": "2026-10-31T08:51:04.442Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.9028377142857142, "accuracy_std_dev": 0.0030797912134375885, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.13983470995670993, "avg_duration_per_task_ms": 1849163.816017316, "avg_output_tokens_per_task": 58254.32756132756, "last_run_timestamp": "2026-10-31T16:13:09.866Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.9027775833333331, "accuracy_std_dev": 0.009146948712169298, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.004449537878787879, "avg_duration_per_task_ms": 380489.8497474748, "avg_output_tokens_per_task": 24420.810185185186, "last_run_timestamp": "2026-10-31T08:39:08.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.9027699539584503, "accuracy_std_dev": 0.006295075818094274, "run_count": 9, "total_questions": 1781, "avg_cost_per_task_usd": 0.028251830432341377, "avg_duration_per_task_ms": 172539.25996631107, "avg_output_tokens_per_task": 28841.043234138124, "last_run_timestamp": "2026-10-31T08:31:10.643Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.9026157692307694, "accuracy_std_dev": 0.012508586614825496, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.022228931623931622, "avg_duration_per_task_ms": 201340.69813519812, "avg_output_tokens_per_task": 26432.28243978244, "last_run_timestamp": "2026-11-10T06:09:57.337Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.902357, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1437863712121212, "avg_duration_per_task_ms": 149585.63636363635, "avg_output_tokens_per_task": 9456.126262626263, "last_run_timestamp": "2026-10-01T19:14:23.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "DeepInfra", "accuracy": 0.902357, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.027153818181818182, "avg_duration_per_task_ms": 429418.7121212121, "avg_output_tokens_per_task": 26931.89898989899, "last_run_timestamp": "2026-11-13T12:01:34.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.9023568, "accuracy_std_dev": 0.003838970664123373, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0695745484848485, "avg_duration_per_task_ms": 344238.6484848485, "avg_output_tokens_per_task": 16158.315151515151, "last_run_timestamp": "2026-10-31T08:52:05.708Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.9020761666666667, "accuracy_std_dev": 0.005271695534223824, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05407119865319866, "avg_duration_per_task_ms": 30482.292087542086, "avg_output_tokens_per_task": 2495.6936026936028, "last_run_timestamp": "2026-10-31T08:27:47.532Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.9020202237762238, "accuracy_std_dev": 0.008897393508385223, "run_count": 12, "total_questions": 2145, "avg_cost_per_task_usd": 0.029685459207459214, "avg_duration_per_task_ms": 182816.28065268067, "avg_output_tokens_per_task": 23066.490442890445, "last_run_timestamp": "2026-11-06T06:01:51.918Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "PrimeIntellect", "accuracy": 0.9018877466467959, "accuracy_std_dev": 0.01830165892678099, "run_count": 6, "total_questions": 671, "avg_cost_per_task_usd": 0.0035507228017883754, "avg_duration_per_task_ms": 96273.04172876304, "avg_output_tokens_per_task": 15221.761549925484, "last_run_timestamp": "2026-11-02T17:22:52.620Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.9017445454545455, "accuracy_std_dev": 0.018612573415759153, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03461236868686868, "avg_duration_per_task_ms": 310679.8080808081, "avg_output_tokens_per_task": 30520.936639118456, "last_run_timestamp": "2026-10-31T16:18:48.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.9015653066016837, "accuracy_std_dev": 0.007303971162529, "run_count": 12, "total_questions": 2257, "avg_cost_per_task_usd": 0.013315422241914046, "avg_duration_per_task_ms": 151646.0602569783, "avg_output_tokens_per_task": 26073.0053167922, "last_run_timestamp": "2026-11-10T18:29:20.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "auto-routing", "accuracy": 0.9015150000000001, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0736977297979798, "avg_duration_per_task_ms": 138589.0126262626, "avg_output_tokens_per_task": 17125.45707070707, "last_run_timestamp": "2026-10-31T08:09:26.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "BaseTen", "accuracy": 0.901515, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0720575808080808, "avg_duration_per_task_ms": 100990.45707070707, "avg_output_tokens_per_task": 16730.29797979798, "last_run_timestamp": "2026-10-31T08:17:04.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.901515, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.16319313383838385, "avg_duration_per_task_ms": 88415.74242424243, "avg_output_tokens_per_task": 9762.232323232323, "last_run_timestamp": "2026-10-01T19:12:25.243Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.9015149999999998, "accuracy_std_dev": 0.006314360868830512, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.20703088957759413, "avg_duration_per_task_ms": 97686.47543617999, "avg_output_tokens_per_task": 7837.382690541782, "last_run_timestamp": "2026-10-31T08:46:46.997Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.9012853636363636, "accuracy_std_dev": 0.01164766215538338, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.03467139577594123, "avg_duration_per_task_ms": 250666.62396694216, "avg_output_tokens_per_task": 27670.98209366391, "last_run_timestamp": "2026-10-31T08:36:32.971Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.9012343333333332, "accuracy_std_dev": 0.0037221826780652403, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03380553956228956, "avg_duration_per_task_ms": 826187.4343434343, "avg_output_tokens_per_task": 38504.42255892256, "last_run_timestamp": "2026-10-31T08:14:20.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.9011541428571429, "accuracy_std_dev": 0.004473489479416358, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.053932944444444436, "avg_duration_per_task_ms": 26662.532467532466, "avg_output_tokens_per_task": 2480.9805194805194, "last_run_timestamp": "2026-10-31T08:25:18.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Crusoe", "accuracy": 0.9010944999999999, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00798360101010101, "avg_duration_per_task_ms": 317918.4797979798, "avg_output_tokens_per_task": 28057.032828282827, "last_run_timestamp": "2026-10-31T16:08:33.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.900953777777778, "accuracy_std_dev": 0.018721426879248864, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.07315864141414141, "avg_duration_per_task_ms": 540347.3451178451, "avg_output_tokens_per_task": 36755.94332210999, "last_run_timestamp": "2026-11-10T07:51:59.421Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.9006730000000001, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.010972851010101009, "avg_duration_per_task_ms": 117479.04797979798, "avg_output_tokens_per_task": 19415.13888888889, "last_run_timestamp": "2026-10-31T08:11:46.712Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.9005650372881355, "accuracy_std_dev": 0.0005644881654108368, "run_count": 3, "total_questions": 590, "avg_cost_per_task_usd": 0.013123520338983051, "avg_duration_per_task_ms": 181977.62711864407, "avg_output_tokens_per_task": 21777.491525423728, "last_run_timestamp": "2026-10-31T08:31:11.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "BaseTen", "accuracy": 0.9005043975240714, "accuracy_std_dev": 0.00897585252218416, "run_count": 4, "total_questions": 727, "avg_cost_per_task_usd": 0.027938022008253093, "avg_duration_per_task_ms": 73305.65749656121, "avg_output_tokens_per_task": 18908.090784044016, "last_run_timestamp": "2026-11-05T21:01:47.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.900418212121212, "accuracy_std_dev": 0.006022033356174897, "run_count": 33, "total_questions": 6534, "avg_cost_per_task_usd": 0.21616242209978576, "avg_duration_per_task_ms": 92587.19115396388, "avg_output_tokens_per_task": 7884.330272421182, "last_run_timestamp": "2026-10-31T08:23:44.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Nebius", "accuracy": 0.9002525, "accuracy_std_dev": 0.004014983343676526, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.014062290404040403, "avg_duration_per_task_ms": 87702.04797979798, "avg_output_tokens_per_task": 13895.555555555555, "last_run_timestamp": "2026-10-31T08:28:22.736Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.9002466549491212, "accuracy_std_dev": 0.01549038271192753, "run_count": 11, "total_questions": 2162, "avg_cost_per_task_usd": 0.16790182839962997, "avg_duration_per_task_ms": 387266.13876040705, "avg_output_tokens_per_task": 42094.61147086031, "last_run_timestamp": "2026-10-31T08:09:30.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.9001631212121214, "accuracy_std_dev": 0.005552397340584662, "run_count": 33, "total_questions": 6534, "avg_cost_per_task_usd": 0.22268922956841136, "avg_duration_per_task_ms": 93998.23614937252, "avg_output_tokens_per_task": 8126.572849709213, "last_run_timestamp": "2026-10-31T08:31:12.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.9000319047619048, "accuracy_std_dev": 0.005076517459315892, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.05824434295334296, "avg_duration_per_task_ms": 35780.01707551708, "avg_output_tokens_per_task": 2517.5358345358345, "last_run_timestamp": "2026-10-31T09:01:04.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.8999309479616755, "accuracy_std_dev": 0.123858969413793, "run_count": 27, "total_questions": 5323, "avg_cost_per_task_usd": 0.16280099473980839, "avg_duration_per_task_ms": 193274.2459139583, "avg_output_tokens_per_task": 10715.915085478115, "last_run_timestamp": "2026-10-31T09:24:16.791Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Cohere", "accuracy": 0.8996625716694773, "accuracy_std_dev": 0.0020759481367959394, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.11324433389544686, "avg_duration_per_task_ms": 858665.3389544688, "avg_output_tokens_per_task": 45077.612141652615, "last_run_timestamp": "2026-10-31T08:28:18.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8995687234401349, "accuracy_std_dev": 0.007447984730328704, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.0197627712197864, "avg_duration_per_task_ms": 382016.3063518831, "avg_output_tokens_per_task": 22859.64811691962, "last_run_timestamp": "2026-10-31T08:33:02.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.8995508333333332, "accuracy_std_dev": 0.0038470676025888335, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.15600541077441077, "avg_duration_per_task_ms": 138877.1792929293, "avg_output_tokens_per_task": 9623.925925925925, "last_run_timestamp": "2026-10-01T19:18:24.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.8994372877637131, "accuracy_std_dev": 0.01977995405637288, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.03544181097046414, "avg_duration_per_task_ms": 152645.21687763714, "avg_output_tokens_per_task": 26597.288607594935, "last_run_timestamp": "2026-10-03T09:28:39.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.8993638888888887, "accuracy_std_dev": 0.004029367042778795, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.030411617283950615, "avg_duration_per_task_ms": 237553.6509539843, "avg_output_tokens_per_task": 26633.1290684624, "last_run_timestamp": "2026-10-03T10:19:34.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.8993264, "accuracy_std_dev": 0.006511664598242117, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.13052546969696968, "avg_duration_per_task_ms": 30874.333333333332, "avg_output_tokens_per_task": 2403.130303030303, "last_run_timestamp": "2026-10-01T08:24:30.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.8992302857142856, "accuracy_std_dev": 0.0048260090792136764, "run_count": 21, "total_questions": 4158, "avg_cost_per_task_usd": 0.05735562409812409, "avg_duration_per_task_ms": 25146.343674843676, "avg_output_tokens_per_task": 2479.3174603174602, "last_run_timestamp": "2026-10-31T09:06:56.339Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.8991100000000001, "accuracy_std_dev": 0.005600276243186554, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.05768347799422799, "avg_duration_per_task_ms": 26053.75036075036, "avg_output_tokens_per_task": 2530.0901875901877, "last_run_timestamp": "2026-10-31T08:22:04.658Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.8989131363636366, "accuracy_std_dev": 0.008207886901663264, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.1284000422405877, "avg_duration_per_task_ms": 58962.32116620753, "avg_output_tokens_per_task": 4769.593434343435, "last_run_timestamp": "2026-10-31T08:31:13.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.8985305454545454, "accuracy_std_dev": 0.018596178744147863, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.023964589072543615, "avg_duration_per_task_ms": 168611.35950413224, "avg_output_tokens_per_task": 27930.285583103763, "last_run_timestamp": "2026-11-10T06:04:30.952Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.8981922631578948, "accuracy_std_dev": 0.013259923107827002, "run_count": 19, "total_questions": 3762, "avg_cost_per_task_usd": 0.007911754917597024, "avg_duration_per_task_ms": 156354.3657628921, "avg_output_tokens_per_task": 24268.396331738437, "last_run_timestamp": "2026-10-31T09:24:59.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.8981735454545454, "accuracy_std_dev": 0.007167908105686761, "run_count": 33, "total_questions": 6534, "avg_cost_per_task_usd": 0.2089634925007652, "avg_duration_per_task_ms": 71375.81910009183, "avg_output_tokens_per_task": 7612.6602387511475, "last_run_timestamp": "2026-10-31T08:51:35.523Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.8981480833333332, "accuracy_std_dev": 0.007606843601415013, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.19857798400673404, "avg_duration_per_task_ms": 95247.20033670033, "avg_output_tokens_per_task": 7731.943181818182, "last_run_timestamp": "2026-10-31T08:27:45.200Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.898148, "accuracy_std_dev": 0.012012972737836369, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03667170117845118, "avg_duration_per_task_ms": 157797.89814814815, "avg_output_tokens_per_task": 27519.53787878788, "last_run_timestamp": "2026-10-03T09:28:24.400Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.8981478333333334, "accuracy_std_dev": 0.007386487717823356, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.12543263804713806, "avg_duration_per_task_ms": 965889.989057239, "avg_output_tokens_per_task": 49917.90404040404, "last_run_timestamp": "2026-10-31T08:45:01.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.89814775, "accuracy_std_dev": 0.0055197238778311765, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.11915268055555557, "avg_duration_per_task_ms": 825703.6161616162, "avg_output_tokens_per_task": 47450.94823232323, "last_run_timestamp": "2026-10-31T08:07:17.850Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.8980186153846152, "accuracy_std_dev": 0.03138066286630978, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.11786784654234653, "avg_duration_per_task_ms": 508070.64024864027, "avg_output_tokens_per_task": 36980.76418026418, "last_run_timestamp": "2026-11-10T08:27:28.930Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.8980075833333333, "accuracy_std_dev": 0.004784274352123693, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.23032522264309765, "avg_duration_per_task_ms": 110445.48653198653, "avg_output_tokens_per_task": 8998.862373737375, "last_run_timestamp": "2026-10-31T08:24:37.252Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Ionstream", "accuracy": 0.8979796, "accuracy_std_dev": 0.008689115757083655, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.014924614141414142, "avg_duration_per_task_ms": 134095.19090909092, "avg_output_tokens_per_task": 23160.81515151515, "last_run_timestamp": "2026-11-10T16:47:27.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "StreamLake", "accuracy": 0.8978673333333335, "accuracy_std_dev": 0.007570651065499923, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03724634175084175, "avg_duration_per_task_ms": 745311.5892255892, "avg_output_tokens_per_task": 37370.69696969697, "last_run_timestamp": "2026-10-31T08:12:46.423Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8973063333333333, "accuracy_std_dev": 0.004955863351178606, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03703221212121212, "avg_duration_per_task_ms": 137593.69696969696, "avg_output_tokens_per_task": 26012.949494949495, "last_run_timestamp": "2026-10-02T23:26:10.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Crusoe", "accuracy": 0.897306, "accuracy_std_dev": 0.004761657064510171, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.026144708754208754, "avg_duration_per_task_ms": 176560.02693602693, "avg_output_tokens_per_task": 21638.170033670034, "last_run_timestamp": "2026-10-31T08:57:38.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Together", "accuracy": 0.8973059999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07477493686868687, "avg_duration_per_task_ms": 142684.398989899, "avg_output_tokens_per_task": 17437.89393939394, "last_run_timestamp": "2026-10-31T08:49:18.270Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.8971531818181817, "accuracy_std_dev": 0.011450877590966798, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.20187068044077136, "avg_duration_per_task_ms": 96528.3347107438, "avg_output_tokens_per_task": 7855.07943067034, "last_run_timestamp": "2026-10-31T08:32:06.814Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Nebius", "accuracy": 0.8970256666666665, "accuracy_std_dev": 0.00667522615513675, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06776180639730639, "avg_duration_per_task_ms": 222686.2255892256, "avg_output_tokens_per_task": 15778.796296296296, "last_run_timestamp": "2026-10-31T08:27:47.368Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.8968690856353589, "accuracy_std_dev": 0.010670089513748447, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.018374484806629836, "avg_duration_per_task_ms": 362424.26427255984, "avg_output_tokens_per_task": 28910.183241252304, "last_run_timestamp": "2026-10-31T09:16:30.257Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.8967014333929941, "accuracy_std_dev": 0.03454965314283198, "run_count": 20, "total_questions": 3911, "avg_cost_per_task_usd": 0.37683498056762965, "avg_duration_per_task_ms": 497556.26566095627, "avg_output_tokens_per_task": 57557.85144464331, "last_run_timestamp": "2026-10-31T16:18:08.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.896694090909091, "accuracy_std_dev": 0.006667655625269636, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.208940303030303, "avg_duration_per_task_ms": 100740.17171717172, "avg_output_tokens_per_task": 8143.670798898072, "last_run_timestamp": "2026-10-31T08:27:47.283Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.8963318181818182, "accuracy_std_dev": 0.010896439994787305, "run_count": 5, "total_questions": 836, "avg_cost_per_task_usd": 0.0957473456937799, "avg_duration_per_task_ms": 189939.80023923446, "avg_output_tokens_per_task": 25436.38875598086, "last_run_timestamp": "2026-09-26T09:19:13.012Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.8962463164155431, "accuracy_std_dev": 0.019589876018192762, "run_count": 7, "total_questions": 1261, "avg_cost_per_task_usd": 0.15202800396510704, "avg_duration_per_task_ms": 619590.4591593973, "avg_output_tokens_per_task": 49768.916732751786, "last_run_timestamp": "2026-10-31T16:34:03.928Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.8962348181818182, "accuracy_std_dev": 0.007295060767622528, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.12272349724517907, "avg_duration_per_task_ms": 58904.73461891644, "avg_output_tokens_per_task": 4673.36593204775, "last_run_timestamp": "2026-10-31T08:31:11.765Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Google", "accuracy": 0.896184, "accuracy_std_dev": 0.007570749896806769, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0237254696969697, "avg_duration_per_task_ms": 319850.3367003367, "avg_output_tokens_per_task": 19569.10606060606, "last_run_timestamp": "2026-10-31T08:20:35.720Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.8961838333333332, "accuracy_std_dev": 0.011153184947458816, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1947589175084175, "avg_duration_per_task_ms": 1704111.0387205386, "avg_output_tokens_per_task": 57112.030303030304, "last_run_timestamp": "2026-10-24T16:20:08.208Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8961037142857144, "accuracy_std_dev": 0.004737329618927418, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.054530834776334774, "avg_duration_per_task_ms": 30476.945887445887, "avg_output_tokens_per_task": 2521.178932178932, "last_run_timestamp": "2026-10-31T08:40:20.874Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.895623, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03841230555555556, "avg_duration_per_task_ms": 144059.0782828283, "avg_output_tokens_per_task": 27039.489898989897, "last_run_timestamp": "2026-10-01T16:12:34.647Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "GMICloud", "accuracy": 0.8956226666666667, "accuracy_std_dev": 0.0059917140749167225, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.11581659259259257, "avg_duration_per_task_ms": 1591244.2491582492, "avg_output_tokens_per_task": 46146.09090909091, "last_run_timestamp": "2026-11-10T07:04:59.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.895412375, "accuracy_std_dev": 0.020051295631065218, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02414115151515151, "avg_duration_per_task_ms": 210274.94823232322, "avg_output_tokens_per_task": 27810.135101010103, "last_run_timestamp": "2026-10-31T08:27:49.932Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.8952905937499999, "accuracy_std_dev": 0.016969367452854572, "run_count": 8, "total_questions": 1536, "avg_cost_per_task_usd": 0.032397834635416664, "avg_duration_per_task_ms": 236218.794921875, "avg_output_tokens_per_task": 32195.716796875, "last_run_timestamp": "2026-10-31T09:17:17.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.8951635454545456, "accuracy_std_dev": 0.01007786883462638, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.2307558640955005, "avg_duration_per_task_ms": 109981.68457300276, "avg_output_tokens_per_task": 9006.644168962352, "last_run_timestamp": "2026-10-31T08:13:10.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.8951417142857143, "accuracy_std_dev": 0.009679824699036726, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008688588023088023, "avg_duration_per_task_ms": 171107.75108225108, "avg_output_tokens_per_task": 30552.396103896102, "last_run_timestamp": "2026-10-31T09:09:17.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.8951417142857143, "accuracy_std_dev": 0.01785557157315559, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.2036656197691198, "avg_duration_per_task_ms": 957488.6248196248, "avg_output_tokens_per_task": 59748.26406926407, "last_run_timestamp": "2026-10-31T16:24:42.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.8950656277001271, "accuracy_std_dev": 0.016150328135211358, "run_count": 16, "total_questions": 3148, "avg_cost_per_task_usd": 0.007430722681067345, "avg_duration_per_task_ms": 254760.92153748413, "avg_output_tokens_per_task": 28431.650254129607, "last_run_timestamp": "2026-11-02T17:25:00.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.894573530232558, "accuracy_std_dev": 0.042752448470087655, "run_count": 12, "total_questions": 2365, "avg_cost_per_task_usd": 0.07040914122621564, "avg_duration_per_task_ms": 595191.9403805496, "avg_output_tokens_per_task": 39081.489217758986, "last_run_timestamp": "2026-10-31T16:51:33.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "ModelRun", "accuracy": 0.8943087542213883, "accuracy_std_dev": 0.03942204850926279, "run_count": 3, "total_questions": 533, "avg_cost_per_task_usd": 0.019450917448405252, "avg_duration_per_task_ms": 177927.2420262664, "avg_output_tokens_per_task": 28174.943714821762, "last_run_timestamp": "2026-10-28T05:00:14.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.8942835463539759, "accuracy_std_dev": 0.01749872529716121, "run_count": 14, "total_questions": 2729, "avg_cost_per_task_usd": 0.1169677999267131, "avg_duration_per_task_ms": 555475.2238915353, "avg_output_tokens_per_task": 43652.794796628805, "last_run_timestamp": "2026-11-06T22:44:36.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.894276, "accuracy_std_dev": 0.008869991905295046, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.11497575656565656, "avg_duration_per_task_ms": 654340.1272727273, "avg_output_tokens_per_task": 45820.55656565657, "last_run_timestamp": "2026-10-31T08:45:35.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.8941497500000002, "accuracy_std_dev": 0.012326495474687834, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.010706986111111112, "avg_duration_per_task_ms": 265829.2607323232, "avg_output_tokens_per_task": 20748.16351010101, "last_run_timestamp": "2026-10-31T08:33:46.211Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "CoreWeave", "accuracy": 0.893939358490566, "accuracy_std_dev": 0.004245539018258487, "run_count": 3, "total_questions": 583, "avg_cost_per_task_usd": 0.023210332761578047, "avg_duration_per_task_ms": 116069.80960548884, "avg_output_tokens_per_task": 13628.193825042881, "last_run_timestamp": "2026-10-31T08:17:29.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Phala", "accuracy": 0.8939393333333333, "accuracy_std_dev": 0.012217692780917704, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02448665488215488, "avg_duration_per_task_ms": 139886.7744107744, "avg_output_tokens_per_task": 20249.15824915825, "last_run_timestamp": "2026-10-31T09:02:19.878Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.8936585000000001, "accuracy_std_dev": 0.017069648744189203, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.23770161616161617, "avg_duration_per_task_ms": 925404.0513468013, "avg_output_tokens_per_task": 59269.02946127946, "last_run_timestamp": "2026-10-24T16:07:10.471Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.8934709999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 194, "avg_cost_per_task_usd": 0.11686864948453608, "avg_duration_per_task_ms": 605687.7474226804, "avg_output_tokens_per_task": 46496.701030927834, "last_run_timestamp": "2026-10-13T03:00:07.016Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.8934035909090907, "accuracy_std_dev": 0.009447194358206847, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.24392887970615243, "avg_duration_per_task_ms": 85202.9074839302, "avg_output_tokens_per_task": 9060.956611570247, "last_run_timestamp": "2026-10-31T08:12:47.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "BaseTen", "accuracy": 0.8933780000000001, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03135734848484849, "avg_duration_per_task_ms": 31339.205387205388, "avg_output_tokens_per_task": 13928.784511784512, "last_run_timestamp": "2026-10-31T08:28:17.755Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "DigitalOcean", "accuracy": 0.8933779999999999, "accuracy_std_dev": 0.0034590473640392213, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01648433501683502, "avg_duration_per_task_ms": 93266.93602693603, "avg_output_tokens_per_task": 13659.838383838383, "last_run_timestamp": "2026-10-31T08:40:23.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.8931983140809443, "accuracy_std_dev": 0.01619141679512122, "run_count": 12, "total_questions": 2372, "avg_cost_per_task_usd": 0.1531594173693086, "avg_duration_per_task_ms": 667713.1500843171, "avg_output_tokens_per_task": 37442.99325463744, "last_run_timestamp": "2026-11-05T00:00:09.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "PrimeIntellect", "accuracy": 0.8931621743589744, "accuracy_std_dev": 0.0023230000000000195, "run_count": 2, "total_questions": 390, "avg_cost_per_task_usd": 0.020242015384615383, "avg_duration_per_task_ms": 141496.24102564104, "avg_output_tokens_per_task": 40323.36666666667, "last_run_timestamp": "2026-10-31T08:52:13.905Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.8930975833333333, "accuracy_std_dev": 0.02298558731559965, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.08767985311447812, "avg_duration_per_task_ms": 506727.319023569, "avg_output_tokens_per_task": 39812.03493265993, "last_run_timestamp": "2026-10-31T16:16:10.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.8930975, "accuracy_std_dev": 0.010942499999999966, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04334855050505051, "avg_duration_per_task_ms": 40520.42929292929, "avg_output_tokens_per_task": 3151.881313131313, "last_run_timestamp": "2026-10-01T19:11:43.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "AtlasCloud", "accuracy": 0.8930975, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11303334343434343, "avg_duration_per_task_ms": 814053.6843434343, "avg_output_tokens_per_task": 45048.43686868687, "last_run_timestamp": "2026-10-31T08:27:45.248Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.8928929372738239, "accuracy_std_dev": 0.015585617718297187, "run_count": 17, "total_questions": 3316, "avg_cost_per_task_usd": 0.014895325693606755, "avg_duration_per_task_ms": 187175.30186972255, "avg_output_tokens_per_task": 21447.96381182147, "last_run_timestamp": "2026-10-31T08:57:07.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.892885651478817, "accuracy_std_dev": 0.023028232143902604, "run_count": 7, "total_questions": 1251, "avg_cost_per_task_usd": 0.21098625099920065, "avg_duration_per_task_ms": 1517055.7913669064, "avg_output_tokens_per_task": 52600.646682653874, "last_run_timestamp": "2026-10-31T16:04:29.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.8928570861678005, "accuracy_std_dev": 0.015340394005370288, "run_count": 5, "total_questions": 882, "avg_cost_per_task_usd": 0.12683532766439912, "avg_duration_per_task_ms": 285532.1031746032, "avg_output_tokens_per_task": 28366.77097505669, "last_run_timestamp": "2026-09-19T09:14:09.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.8928297457392905, "accuracy_std_dev": 0.010017422802463, "run_count": 11, "total_questions": 2171, "avg_cost_per_task_usd": 0.008073876554583142, "avg_duration_per_task_ms": 317611.8940580378, "avg_output_tokens_per_task": 28391.647167204053, "last_run_timestamp": "2026-10-31T08:16:11.302Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Novita", "accuracy": 0.892817, "accuracy_std_dev": 0.00915257617650169, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007166784511784513, "avg_duration_per_task_ms": 121588.36531986532, "avg_output_tokens_per_task": 11764.878787878788, "last_run_timestamp": "2026-10-31T08:24:44.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.892817, "accuracy_std_dev": 0.0067805750493597225, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.11382627104377104, "avg_duration_per_task_ms": 836209.1372053872, "avg_output_tokens_per_task": 45359.85101010101, "last_run_timestamp": "2026-10-31T09:36:23.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8926684364569961, "accuracy_std_dev": 0.021409854620919034, "run_count": 12, "total_questions": 2337, "avg_cost_per_task_usd": 0.01926024176294395, "avg_duration_per_task_ms": 249786.02053915276, "avg_output_tokens_per_task": 30225.52802738554, "last_run_timestamp": "2026-11-04T21:06:11.704Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.8922560000000002, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.14220060606060608, "avg_duration_per_task_ms": 188559.2904040404, "avg_output_tokens_per_task": 47276.156565656565, "last_run_timestamp": "2026-10-01T19:12:30.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.8922558333333332, "accuracy_std_dev": 0.010423127464388435, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.04560339057239057, "avg_duration_per_task_ms": 43217.19528619529, "avg_output_tokens_per_task": 3431.844276094276, "last_run_timestamp": "2026-10-01T19:13:06.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.8919497272727274, "accuracy_std_dev": 0.012304710177894609, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.20576803719008263, "avg_duration_per_task_ms": 98251.07162534435, "avg_output_tokens_per_task": 8002.224517906336, "last_run_timestamp": "2026-10-31T08:39:17.193Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "OpenInference", "accuracy": 0.8916946666666665, "accuracy_std_dev": 0.009944030347678707, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.011122356902356903, "avg_duration_per_task_ms": 302424.50673400675, "avg_output_tokens_per_task": 25444.905723905726, "last_run_timestamp": "2026-10-31T09:01:06.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.8913374545454544, "accuracy_std_dev": 0.018875016904814263, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0336423507805326, "avg_duration_per_task_ms": 191376.6694214876, "avg_output_tokens_per_task": 25231.550505050505, "last_run_timestamp": "2026-10-31T08:53:51.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.8912864848484849, "accuracy_std_dev": 0.009755318270195008, "run_count": 33, "total_questions": 6534, "avg_cost_per_task_usd": 0.13280288047138045, "avg_duration_per_task_ms": 59185.807315580045, "avg_output_tokens_per_task": 4745.987909397, "last_run_timestamp": "2026-10-31T08:46:09.151Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8910532857142858, "accuracy_std_dev": 0.008331061289180467, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02299151948051948, "avg_duration_per_task_ms": 246585.4624819625, "avg_output_tokens_per_task": 30738.57647907648, "last_run_timestamp": "2026-10-31T08:09:30.604Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.8904192727272727, "accuracy_std_dev": 0.008657537410634094, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.205998660697888, "avg_duration_per_task_ms": 99123.53581267217, "avg_output_tokens_per_task": 8011.966483011937, "last_run_timestamp": "2026-10-31T09:23:04.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.8900113333333334, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008230473063973063, "avg_duration_per_task_ms": 80128.80303030302, "avg_output_tokens_per_task": 12935.998316498317, "last_run_timestamp": "2026-10-02T23:23:43.005Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.889876224111283, "accuracy_std_dev": 0.005998909101508626, "run_count": 7, "total_questions": 1294, "avg_cost_per_task_usd": 0.030587522411128287, "avg_duration_per_task_ms": 772887.7256568779, "avg_output_tokens_per_task": 36034.0115919629, "last_run_timestamp": "2026-10-31T08:09:29.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.8895201249999999, "accuracy_std_dev": 0.013120057749849085, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.10712794823232324, "avg_duration_per_task_ms": 473198.4646464646, "avg_output_tokens_per_task": 37189.18434343435, "last_run_timestamp": "2026-10-31T08:45:16.520Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.02853141346198763, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.136280962962963, "avg_duration_per_task_ms": 40331.19781144781, "avg_output_tokens_per_task": 2516.2390572390573, "last_run_timestamp": "2026-10-01T08:24:17.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.8894498333333334, "accuracy_std_dev": 0.009896244159893304, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.008765856902356902, "avg_duration_per_task_ms": 59664.589225589225, "avg_output_tokens_per_task": 12901.691919191919, "last_run_timestamp": "2026-10-01T08:29:53.989Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.8893538140969164, "accuracy_std_dev": 0.014720288133832686, "run_count": 12, "total_questions": 2270, "avg_cost_per_task_usd": 0.037593757268722466, "avg_duration_per_task_ms": 339622.79823788546, "avg_output_tokens_per_task": 28197.38370044053, "last_run_timestamp": "2026-10-31T08:10:35.431Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.8893479090909092, "accuracy_std_dev": 0.0107458728521871, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.029853909090909094, "avg_duration_per_task_ms": 337392.91643709823, "avg_output_tokens_per_task": 27657.61432506887, "last_run_timestamp": "2026-10-31T09:11:16.308Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.8891708512988952, "accuracy_std_dev": 0.008107170901595804, "run_count": 17, "total_questions": 3349, "avg_cost_per_task_usd": 0.015164630934607346, "avg_duration_per_task_ms": 200146.0895789788, "avg_output_tokens_per_task": 27111.550313526426, "last_run_timestamp": "2026-10-31T09:08:32.372Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.8890757777777778, "accuracy_std_dev": 0.019894746820749636, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0056425325476992146, "avg_duration_per_task_ms": 237464.28787878787, "avg_output_tokens_per_task": 30270.649831649833, "last_run_timestamp": "2026-11-10T06:21:53.777Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.889041818181818, "accuracy_std_dev": 0.012149455384862324, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.11870665610651976, "avg_duration_per_task_ms": 546946.9178145088, "avg_output_tokens_per_task": 30914.909090909092, "last_run_timestamp": "2026-10-31T16:23:41.134Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.8888890000000002, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0431288872053872, "avg_duration_per_task_ms": 52519.67676767677, "avg_output_tokens_per_task": 3463.8030303030305, "last_run_timestamp": "2026-10-02T23:21:06.556Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.888889, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1979060606060606, "avg_duration_per_task_ms": 95702.44696969698, "avg_output_tokens_per_task": 7691.454545454545, "last_run_timestamp": "2026-10-01T19:11:18.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Nebius", "accuracy": 0.888889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13065590404040406, "avg_duration_per_task_ms": 663129.7929292929, "avg_output_tokens_per_task": 36983.707070707074, "last_run_timestamp": "2026-11-03T21:00:06.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.8887622653758542, "accuracy_std_dev": 0.006290540123073671, "run_count": 5, "total_questions": 878, "avg_cost_per_task_usd": 0.01762260478359909, "avg_duration_per_task_ms": 219383.70842824603, "avg_output_tokens_per_task": 26459.921412300682, "last_run_timestamp": "2026-09-19T08:54:58.652Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.8886065933593749, "accuracy_std_dev": 0.01380537387006623, "run_count": 13, "total_questions": 2560, "avg_cost_per_task_usd": 0.10022955078125, "avg_duration_per_task_ms": 460846.6203125, "avg_output_tokens_per_task": 34095.671484375, "last_run_timestamp": "2026-10-31T16:08:08.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.8882767272727271, "accuracy_std_dev": 0.018777918676858473, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.013537202938475667, "avg_duration_per_task_ms": 349207.04545454547, "avg_output_tokens_per_task": 23738.730945821855, "last_run_timestamp": "2026-10-31T08:24:31.558Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Crusoe", "accuracy": 0.8881691813471503, "accuracy_std_dev": 0.0030120000000000147, "run_count": 2, "total_questions": 386, "avg_cost_per_task_usd": 0.1504010103626943, "avg_duration_per_task_ms": 651195.4274611399, "avg_output_tokens_per_task": 42839.41709844559, "last_run_timestamp": "2026-10-31T16:08:08.539Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.8881671428571429, "accuracy_std_dev": 0.01839453545740899, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008335702741702741, "avg_duration_per_task_ms": 417723.5966810967, "avg_output_tokens_per_task": 29344.838383838385, "last_run_timestamp": "2026-10-31T16:48:29.809Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.8880907953321364, "accuracy_std_dev": 0.02107123389837437, "run_count": 3, "total_questions": 557, "avg_cost_per_task_usd": 0.004236710951526032, "avg_duration_per_task_ms": 379493.5960502693, "avg_output_tokens_per_task": 16317.405745062837, "last_run_timestamp": "2026-09-05T08:24:32.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.8876718896377422, "accuracy_std_dev": 0.0038308239938977078, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.032938989890480205, "avg_duration_per_task_ms": 750389.2257792755, "avg_output_tokens_per_task": 38026.47177759057, "last_run_timestamp": "2026-10-31T08:03:09.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.8875500348763478, "accuracy_std_dev": 0.013802209238356553, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.1294895707038681, "avg_duration_per_task_ms": 1720933.2999365884, "avg_output_tokens_per_task": 32195.95688015219, "last_run_timestamp": "2026-10-31T16:35:08.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.8875475545108005, "accuracy_std_dev": 0.011882850320945717, "run_count": 20, "total_questions": 3935, "avg_cost_per_task_usd": 0.192012674205845, "avg_duration_per_task_ms": 351514.216772554, "avg_output_tokens_per_task": 31670.305209656926, "last_run_timestamp": "2026-10-24T16:13:46.475Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.887537924012158, "accuracy_std_dev": 0.020250680116973837, "run_count": 5, "total_questions": 987, "avg_cost_per_task_usd": 0.12033997061803446, "avg_duration_per_task_ms": 580755.4113475177, "avg_output_tokens_per_task": 34357.5663627153, "last_run_timestamp": "2026-10-03T16:44:56.199Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.8874829878542512, "accuracy_std_dev": 0.0253227022698781, "run_count": 15, "total_questions": 2964, "avg_cost_per_task_usd": 0.14755026484480432, "avg_duration_per_task_ms": 305870.1987179487, "avg_output_tokens_per_task": 10905.781039136302, "last_run_timestamp": "2026-10-31T09:14:35.378Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "River", "accuracy": 0.8872055, "accuracy_std_dev": 0.011784499999999976, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.016706742424242426, "avg_duration_per_task_ms": 144417.60353535353, "avg_output_tokens_per_task": 34626.61363636364, "last_run_timestamp": "2026-11-12T20:24:33.604Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.8872054285714285, "accuracy_std_dev": 0.032307641642970675, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.26962912914862913, "avg_duration_per_task_ms": 1116880.2828282828, "avg_output_tokens_per_task": 58910.568542568544, "last_run_timestamp": "2026-10-31T16:10:09.220Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.8872051666666666, "accuracy_std_dev": 0.024761493125527695, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1434989882154882, "avg_duration_per_task_ms": 42121.643939393936, "avg_output_tokens_per_task": 2660.465488215488, "last_run_timestamp": "2026-10-01T08:24:34.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.8872049999999999, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004174421717171718, "avg_duration_per_task_ms": 214515.8207070707, "avg_output_tokens_per_task": 22838.073232323233, "last_run_timestamp": "2026-08-29T10:57:26.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.8871015424486148, "accuracy_std_dev": 0.007433711649565692, "run_count": 6, "total_questions": 1119, "avg_cost_per_task_usd": 0.100128890974084, "avg_duration_per_task_ms": 379563.4110813226, "avg_output_tokens_per_task": 36832.25379803396, "last_run_timestamp": "2026-10-31T16:50:08.743Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.8870104150464918, "accuracy_std_dev": 0.040746454673654994, "run_count": 6, "total_questions": 1183, "avg_cost_per_task_usd": 0.19607871766694843, "avg_duration_per_task_ms": 1537768.7954353339, "avg_output_tokens_per_task": 56311.583262890956, "last_run_timestamp": "2026-10-31T16:04:29.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.8869948749999998, "accuracy_std_dev": 0.02052275922261364, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.1362751912878788, "avg_duration_per_task_ms": 824252.7323232323, "avg_output_tokens_per_task": 63166.344696969696, "last_run_timestamp": "2026-11-10T18:52:31.004Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.8867051248554914, "accuracy_std_dev": 0.021208571037200977, "run_count": 5, "total_questions": 865, "avg_cost_per_task_usd": 0.16137210289017342, "avg_duration_per_task_ms": 430828.78150289017, "avg_output_tokens_per_task": 40500.787283237, "last_run_timestamp": "2026-09-19T08:53:57.162Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.8866902897526502, "accuracy_std_dev": 0.024113592079662773, "run_count": 8, "total_questions": 1415, "avg_cost_per_task_usd": 0.035600583745583045, "avg_duration_per_task_ms": 349687.4614840989, "avg_output_tokens_per_task": 26740.03038869258, "last_run_timestamp": "2026-10-31T16:15:26.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.8865318000000001, "accuracy_std_dev": 0.007110736653821441, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.030907244444444443, "avg_duration_per_task_ms": 199105.52626262626, "avg_output_tokens_per_task": 25534.58484848485, "last_run_timestamp": "2026-10-31T08:28:17.984Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.8863635000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.22157512626262627, "avg_duration_per_task_ms": 106335.13383838384, "avg_output_tokens_per_task": 8638.217171717171, "last_run_timestamp": "2026-10-01T19:10:42.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "BaseTen", "accuracy": 0.886083, "accuracy_std_dev": 0.012320026406898109, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.022550329966329965, "avg_duration_per_task_ms": 149365.32154882155, "avg_output_tokens_per_task": 44803.96127946128, "last_run_timestamp": "2026-10-31T08:13:10.777Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.8856471629778674, "accuracy_std_dev": 0.009013068317726206, "run_count": 8, "total_questions": 1491, "avg_cost_per_task_usd": 0.010023977867203217, "avg_duration_per_task_ms": 270739.96177062375, "avg_output_tokens_per_task": 28220.264922870556, "last_run_timestamp": "2026-10-24T09:01:43.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.8856098161888702, "accuracy_std_dev": 0.01054693657998484, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.010296545531197302, "avg_duration_per_task_ms": 75118.94940978078, "avg_output_tokens_per_task": 7923.973018549747, "last_run_timestamp": "2026-10-01T19:12:42.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.885566721955478, "accuracy_std_dev": 0.009041573880860321, "run_count": 12, "total_questions": 2291, "avg_cost_per_task_usd": 0.09590346355303361, "avg_duration_per_task_ms": 473697.19292885205, "avg_output_tokens_per_task": 28730.71977302488, "last_run_timestamp": "2026-10-31T16:12:28.231Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.885522, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02146120202020202, "avg_duration_per_task_ms": 674351.9393939395, "avg_output_tokens_per_task": 45464.17676767677, "last_run_timestamp": "2026-10-01T08:31:57.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Together", "accuracy": 0.8855216666666669, "accuracy_std_dev": 0.0023805928299947096, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.016851747474747474, "avg_duration_per_task_ms": 62951.132996632994, "avg_output_tokens_per_task": 13874.079124579124, "last_run_timestamp": "2026-10-31T08:03:10.178Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.8852179872935196, "accuracy_std_dev": 0.008384701841581625, "run_count": 12, "total_questions": 2361, "avg_cost_per_task_usd": 0.007895573909360442, "avg_duration_per_task_ms": 349145.3604404913, "avg_output_tokens_per_task": 20775.56925031766, "last_run_timestamp": "2026-10-31T08:07:18.580Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.8851849999999999, "accuracy_std_dev": 0.05937202157919166, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.008028678282828283, "avg_duration_per_task_ms": 579179.9328282828, "avg_output_tokens_per_task": 32745.30505050505, "last_run_timestamp": "2026-10-31T08:07:18.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.8850689656218401, "accuracy_std_dev": 0.014411932672615443, "run_count": 5, "total_questions": 989, "avg_cost_per_task_usd": 0.026149528816986855, "avg_duration_per_task_ms": 355797.47320525785, "avg_output_tokens_per_task": 26995.41152679474, "last_run_timestamp": "2026-10-10T08:22:45.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.88468, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1319599772727273, "avg_duration_per_task_ms": 249775.19949494948, "avg_output_tokens_per_task": 43863.606060606064, "last_run_timestamp": "2026-10-01T19:15:31.426Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.8846799999999999, "accuracy_std_dev": 0.021044000000000007, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.08896683838383838, "avg_duration_per_task_ms": 416558.4267676768, "avg_output_tokens_per_task": 22204.20202020202, "last_run_timestamp": "2026-10-31T08:57:00.457Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.8844494345509893, "accuracy_std_dev": 0.01262210148905483, "run_count": 7, "total_questions": 1314, "avg_cost_per_task_usd": 0.18848695585996955, "avg_duration_per_task_ms": 741484.9490106545, "avg_output_tokens_per_task": 53820.38508371385, "last_run_timestamp": "2026-10-31T16:35:50.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.8840456666666667, "accuracy_std_dev": 0.005144110505120299, "run_count": 3, "total_questions": 585, "avg_cost_per_task_usd": 0.02067068717948718, "avg_duration_per_task_ms": 91684.32991452991, "avg_output_tokens_per_task": 41136.99487179487, "last_run_timestamp": "2026-10-31T08:35:13.435Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.8839239383667181, "accuracy_std_dev": 0.011423232412500406, "run_count": 8, "total_questions": 1298, "avg_cost_per_task_usd": 0.00878981278890601, "avg_duration_per_task_ms": 844366.6594761171, "avg_output_tokens_per_task": 29702.687981510015, "last_run_timestamp": "2026-10-31T08:34:49.723Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Together", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11300773737373737, "avg_duration_per_task_ms": 418159.2121212121, "avg_output_tokens_per_task": 32149.79797979798, "last_run_timestamp": "2026-08-22T16:49:15.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Inferact vLLM", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008843217171717172, "avg_duration_per_task_ms": 77278.5505050505, "avg_output_tokens_per_task": 29168.68181818182, "last_run_timestamp": "2026-11-13T21:02:01.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04134767424242424, "avg_duration_per_task_ms": 54936.305555555555, "avg_output_tokens_per_task": 3316.282828282828, "last_run_timestamp": "2026-10-01T19:13:07.806Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008301368686868687, "avg_duration_per_task_ms": 72985.27777777778, "avg_output_tokens_per_task": 13098.969696969696, "last_run_timestamp": "2026-10-01T08:24:38.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "auto-routing", "accuracy": 0.883838, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014708931818181817, "avg_duration_per_task_ms": 71351.09343434343, "avg_output_tokens_per_task": 13550.636363636364, "last_run_timestamp": "2026-10-31T08:27:52.629Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "StreamLake", "accuracy": 0.8836312915601023, "accuracy_std_dev": 0.003620000000000012, "run_count": 2, "total_questions": 391, "avg_cost_per_task_usd": 0.006277416879795397, "avg_duration_per_task_ms": 104811.05115089515, "avg_output_tokens_per_task": 10905.734015345268, "last_run_timestamp": "2026-10-31T08:31:24.328Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.8833571428571427, "accuracy_std_dev": 0.012029934703878998, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.2045760606060606, "avg_duration_per_task_ms": 640185.823953824, "avg_output_tokens_per_task": 59868.3759018759, "last_run_timestamp": "2026-10-31T16:09:16.278Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.8831221012658229, "accuracy_std_dev": 0.01342904913771097, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.1860454797468354, "avg_duration_per_task_ms": 320097.15611814347, "avg_output_tokens_per_task": 46697.94894514768, "last_run_timestamp": "2026-10-31T08:25:00.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Minimax", "accuracy": 0.8829965, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006819977272727273, "avg_duration_per_task_ms": 114306.47727272728, "avg_output_tokens_per_task": 11182.35606060606, "last_run_timestamp": "2026-10-31T08:07:19.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8829694193548384, "accuracy_std_dev": 0.009541162889069374, "run_count": 31, "total_questions": 6138, "avg_cost_per_task_usd": 0.1423835060280222, "avg_duration_per_task_ms": 63221.74942978169, "avg_output_tokens_per_task": 5118.9055066797, "last_run_timestamp": "2026-10-31T08:31:11.082Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.8829290000000001, "accuracy_std_dev": 0.012699110785799135, "run_count": 4, "total_questions": 783, "avg_cost_per_task_usd": 0.027103435504469985, "avg_duration_per_task_ms": 200899.3601532567, "avg_output_tokens_per_task": 30539.469987228607, "last_run_timestamp": "2026-10-31T08:15:03.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.882792129057187, "accuracy_std_dev": 0.012193080659094465, "run_count": 7, "total_questions": 1294, "avg_cost_per_task_usd": 0.06525332148377125, "avg_duration_per_task_ms": 302195.9992272025, "avg_output_tokens_per_task": 18028.40030911901, "last_run_timestamp": "2026-10-31T16:03:21.450Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.8826767013977129, "accuracy_std_dev": 0.009892996415464268, "run_count": 12, "total_questions": 2361, "avg_cost_per_task_usd": 0.10956175815332488, "avg_duration_per_task_ms": 529359.7882253282, "avg_output_tokens_per_task": 31450.83905124947, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.882365125, "accuracy_std_dev": 0.0043279540904883765, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.029606650252525252, "avg_duration_per_task_ms": 78746.81123737374, "avg_output_tokens_per_task": 24519.371843434343, "last_run_timestamp": "2026-10-31T09:02:59.407Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.8821789256756756, "accuracy_std_dev": 0.01706691482241463, "run_count": 6, "total_questions": 1184, "avg_cost_per_task_usd": 0.04157157263513513, "avg_duration_per_task_ms": 164106.9873310811, "avg_output_tokens_per_task": 34476.61317567567, "last_run_timestamp": "2026-10-31T08:31:11.691Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "ModelRun", "accuracy": 0.8821550000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.026309151515151515, "avg_duration_per_task_ms": 202247.18686868687, "avg_output_tokens_per_task": 39119.5505050505, "last_run_timestamp": "2026-10-29T03:00:03.996Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.8821550000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.129083601010101, "avg_duration_per_task_ms": 388751.6590909091, "avg_output_tokens_per_task": 32112.176767676767, "last_run_timestamp": "2026-10-17T16:07:32.047Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.8821546666666666, "accuracy_std_dev": 0.015120291935747206, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007597324915824916, "avg_duration_per_task_ms": 210520.51515151514, "avg_output_tokens_per_task": 26812.25925925926, "last_run_timestamp": "2026-10-03T16:57:09.267Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Reka", "accuracy": 0.881818, "accuracy_std_dev": 0.03925680632450885, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.010723755555555554, "avg_duration_per_task_ms": 145330.13333333333, "avg_output_tokens_per_task": 16217.381818181819, "last_run_timestamp": "2026-10-31T09:01:13.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8816956363636362, "accuracy_std_dev": 0.019816803931249533, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.12889897842056933, "avg_duration_per_task_ms": 63188.49035812672, "avg_output_tokens_per_task": 4922.145546372819, "last_run_timestamp": "2026-10-31T08:31:10.506Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.8816457669902912, "accuracy_std_dev": 0.0330676479779465, "run_count": 11, "total_questions": 2163, "avg_cost_per_task_usd": 0.11147017290799817, "avg_duration_per_task_ms": 672376.8964401295, "avg_output_tokens_per_task": 27712.78640776699, "last_run_timestamp": "2026-10-31T16:08:08.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.8815936666666665, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002981575757575758, "avg_duration_per_task_ms": 52993.44276094276, "avg_output_tokens_per_task": 5804.632996632997, "last_run_timestamp": "2026-10-02T23:20:31.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.8811927142857143, "accuracy_std_dev": 0.007888829783113527, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.015891973304473304, "avg_duration_per_task_ms": 279407.10173160175, "avg_output_tokens_per_task": 24276.554834054834, "last_run_timestamp": "2026-10-08T22:18:51.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.8810325, "accuracy_std_dev": 0.016020429392393534, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03953295538720539, "avg_duration_per_task_ms": 203574.52356902356, "avg_output_tokens_per_task": 32800.2398989899, "last_run_timestamp": "2026-10-31T08:47:44.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.8809522857142856, "accuracy_std_dev": 0.031683881867663896, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.12096548845598847, "avg_duration_per_task_ms": 28374.095238095237, "avg_output_tokens_per_task": 2176.287878787879, "last_run_timestamp": "2026-11-11T21:00:44.509Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.8806113333333333, "accuracy_std_dev": 0.012070796855312507, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.006245051767676768, "avg_duration_per_task_ms": 409383.2984006734, "avg_output_tokens_per_task": 30642.196969696968, "last_run_timestamp": "2026-10-31T08:19:56.454Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.8802082288602942, "accuracy_std_dev": 0.029715678021281344, "run_count": 11, "total_questions": 2176, "avg_cost_per_task_usd": 0.17839703079044117, "avg_duration_per_task_ms": 356421.5234375, "avg_output_tokens_per_task": 40400.66819852941, "last_run_timestamp": "2026-10-31T16:32:36.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.8801906666666667, "accuracy_std_dev": 0.011226983784119793, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.0227881835016835, "avg_duration_per_task_ms": 234700.68434343435, "avg_output_tokens_per_task": 25264.190235690236, "last_run_timestamp": "2026-10-03T09:30:18.263Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.8800631670886074, "accuracy_std_dev": 0.006743341790240229, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.033778830379746834, "avg_duration_per_task_ms": 152045.89430379748, "avg_output_tokens_per_task": 27961.480379746834, "last_run_timestamp": "2026-10-31T09:42:46.430Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.8799569142550911, "accuracy_std_dev": 0.006792606115475872, "run_count": 5, "total_questions": 933, "avg_cost_per_task_usd": 0.11584198606645231, "avg_duration_per_task_ms": 484616.05144694535, "avg_output_tokens_per_task": 33104.76848874598, "last_run_timestamp": "2026-10-17T16:07:16.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.879872690530519, "accuracy_std_dev": 0.053065810239372686, "run_count": 18, "total_questions": 3506, "avg_cost_per_task_usd": 0.00976165202509983, "avg_duration_per_task_ms": 414485.4175698802, "avg_output_tokens_per_task": 26831.222760981174, "last_run_timestamp": "2026-10-31T08:47:06.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.87984, "accuracy_std_dev": 0.018052164925570553, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.036141941919191914, "avg_duration_per_task_ms": 139656.41224747474, "avg_output_tokens_per_task": 28081.721590909092, "last_run_timestamp": "2026-10-31T09:14:35.460Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.8796295000000002, "accuracy_std_dev": 0.017716787991901916, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.02372089204545455, "avg_duration_per_task_ms": 758589.1104797979, "avg_output_tokens_per_task": 35952.905303030304, "last_run_timestamp": "2026-10-31T16:46:00.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "OpenInference", "accuracy": 0.879386, "accuracy_std_dev": null, "run_count": 1, "total_questions": 76, "avg_cost_per_task_usd": 0.03120878947368421, "avg_duration_per_task_ms": 274583.63157894736, "avg_output_tokens_per_task": 9741.526315789473, "last_run_timestamp": "2026-10-09T00:00:06.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.8793102638453499, "accuracy_std_dev": 0.020773557275777312, "run_count": 10, "total_questions": 1914, "avg_cost_per_task_usd": 0.09333590334378267, "avg_duration_per_task_ms": 520471.578892372, "avg_output_tokens_per_task": 28040.05590386625, "last_run_timestamp": "2026-10-31T08:31:12.870Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.8792469090909092, "accuracy_std_dev": 0.01899374060269975, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.12746250872359963, "avg_duration_per_task_ms": 62359.71349862259, "avg_output_tokens_per_task": 4869.864095500459, "last_run_timestamp": "2026-10-31T08:12:45.681Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.8792467272727271, "accuracy_std_dev": 0.029423779211845618, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1929496730945822, "avg_duration_per_task_ms": 46902.871441689625, "avg_output_tokens_per_task": 3649.5376492194673, "last_run_timestamp": "2026-10-31T08:26:18.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.8792337006896551, "accuracy_std_dev": 0.0059956127348350235, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.11865828321839081, "avg_duration_per_task_ms": 795322.059770115, "avg_output_tokens_per_task": 31057.051954022987, "last_run_timestamp": "2026-10-31T16:19:28.502Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.8792084999999998, "accuracy_std_dev": 0.013918588167746535, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.009092756734006733, "avg_duration_per_task_ms": 214671.74326599325, "avg_output_tokens_per_task": 32045.307239057238, "last_run_timestamp": "2026-10-31T08:31:11.360Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.8789951917808219, "accuracy_std_dev": 0.028062210901257694, "run_count": 3, "total_questions": 511, "avg_cost_per_task_usd": 0.12092838356164383, "avg_duration_per_task_ms": 556595.4500978474, "avg_output_tokens_per_task": 50889.26614481409, "last_run_timestamp": "2026-10-03T17:01:24.374Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.8788738633776092, "accuracy_std_dev": 0.01184394360000081, "run_count": 8, "total_questions": 1581, "avg_cost_per_task_usd": 0.05087817014547753, "avg_duration_per_task_ms": 522785.04807084124, "avg_output_tokens_per_task": 36266.31309297913, "last_run_timestamp": "2026-11-10T18:38:17.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8787880000000001, "accuracy_std_dev": 0.007273551310513108, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.009556249158249158, "avg_duration_per_task_ms": 52070.99494949495, "avg_output_tokens_per_task": 7331.292929292929, "last_run_timestamp": "2026-10-30T00:00:06.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.878788, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.043995565656565656, "avg_duration_per_task_ms": 588519.7272727273, "avg_output_tokens_per_task": 34166.82828282828, "last_run_timestamp": "2026-10-01T08:47:14.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.8783067142857143, "accuracy_std_dev": 0.019872992683641802, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.12211109668109668, "avg_duration_per_task_ms": 631289.5389610389, "avg_output_tokens_per_task": 35901.76046176046, "last_run_timestamp": "2026-10-31T16:25:53.821Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.8782263333333332, "accuracy_std_dev": 0.004199386807089298, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.020176612794612796, "avg_duration_per_task_ms": 120195.73063973064, "avg_output_tokens_per_task": 42971.59090909091, "last_run_timestamp": "2026-10-31T08:09:30.418Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.8781846784586228, "accuracy_std_dev": 0.00428175396303899, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.00286223562855338, "avg_duration_per_task_ms": 91558.78395451674, "avg_output_tokens_per_task": 5494.553379658875, "last_run_timestamp": "2026-11-03T03:00:05.640Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.8778809359190558, "accuracy_std_dev": 0.007512559071684942, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.12372368465430017, "avg_duration_per_task_ms": 878302.9047217538, "avg_output_tokens_per_task": 43322.24536256324, "last_run_timestamp": "2026-10-24T16:19:29.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "GMICloud", "accuracy": 0.8777777999999999, "accuracy_std_dev": 0.016888770131658508, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.009364964646464646, "avg_duration_per_task_ms": 177367.14141414143, "avg_output_tokens_per_task": 18449.226262626264, "last_run_timestamp": "2026-11-10T06:10:46.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.8777164545454544, "accuracy_std_dev": 0.03678752360482025, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.18499438292011017, "avg_duration_per_task_ms": 44694.47474747475, "avg_output_tokens_per_task": 3489.084940312213, "last_run_timestamp": "2026-10-31T08:13:57.565Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.8774407999999999, "accuracy_std_dev": 0.00954717737134911, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.03921924343434343, "avg_duration_per_task_ms": 242470.04242424242, "avg_output_tokens_per_task": 37354.009090909094, "last_run_timestamp": "2026-10-21T11:23:15.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.8772070529695024, "accuracy_std_dev": 0.03893659390197737, "run_count": 4, "total_questions": 623, "avg_cost_per_task_usd": 0.09425666452648475, "avg_duration_per_task_ms": 461375.2568218299, "avg_output_tokens_per_task": 23922.329052969504, "last_run_timestamp": "2026-09-30T11:37:23.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.8771043333333335, "accuracy_std_dev": 0.007896528133018712, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.003143374579124579, "avg_duration_per_task_ms": 35450.33585858586, "avg_output_tokens_per_task": 5733.669191919192, "last_run_timestamp": "2026-10-01T08:25:15.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.877104, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07509321212121212, "avg_duration_per_task_ms": 263175.32828282827, "avg_output_tokens_per_task": 20738.459595959597, "last_run_timestamp": "2026-08-29T19:12:43.653Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.876730111111111, "accuracy_std_dev": 0.012390967395149362, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.03705083277216611, "avg_duration_per_task_ms": 153849.39562289562, "avg_output_tokens_per_task": 30722.645342312007, "last_run_timestamp": "2026-11-05T00:00:09.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.8767260400808354, "accuracy_std_dev": 0.008082492995494786, "run_count": 15, "total_questions": 2969, "avg_cost_per_task_usd": 0.024447516335466487, "avg_duration_per_task_ms": 220112.43718423712, "avg_output_tokens_per_task": 34286.534523408554, "last_run_timestamp": "2026-11-02T17:21:00.007Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.876645090909091, "accuracy_std_dev": 0.013199311025094829, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.00596057116620753, "avg_duration_per_task_ms": 273115.2906336088, "avg_output_tokens_per_task": 18439.58402203857, "last_run_timestamp": "2026-10-31T08:36:26.194Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.8763216813793104, "accuracy_std_dev": 0.01096372812565248, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.0099918183908046, "avg_duration_per_task_ms": 425649.0579310345, "avg_output_tokens_per_task": 22880.83264367816, "last_run_timestamp": "2026-10-31T08:32:56.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "auto-routing", "accuracy": 0.8762625, "accuracy_std_dev": 0.009259500000000032, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014013469696969696, "avg_duration_per_task_ms": 171317.10353535353, "avg_output_tokens_per_task": 26009.79292929293, "last_run_timestamp": "2026-10-31T08:09:25.873Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.8762625, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.009382047979797978, "avg_duration_per_task_ms": 83118.84595959596, "avg_output_tokens_per_task": 7689.015151515152, "last_run_timestamp": "2026-10-01T19:14:22.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.8759833172804533, "accuracy_std_dev": 0.02383166285579381, "run_count": 6, "total_questions": 1059, "avg_cost_per_task_usd": 0.006660402266288952, "avg_duration_per_task_ms": 451840.41265344666, "avg_output_tokens_per_task": 36610.579792256845, "last_run_timestamp": "2026-10-31T08:15:03.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Novita", "accuracy": 0.8759819999999999, "accuracy_std_dev": 0.04446362050785635, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.019776964646464647, "avg_duration_per_task_ms": 142117.47643097644, "avg_output_tokens_per_task": 20178.936026936026, "last_run_timestamp": "2026-10-31T09:01:06.393Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.8758864791489362, "accuracy_std_dev": 0.029346183036394313, "run_count": 6, "total_questions": 1175, "avg_cost_per_task_usd": 0.13275879148936168, "avg_duration_per_task_ms": 778330.9634042553, "avg_output_tokens_per_task": 39857.91574468085, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.8757797045454545, "accuracy_std_dev": 0.012408993441628312, "run_count": 8, "total_questions": 1496, "avg_cost_per_task_usd": 0.008467116978609626, "avg_duration_per_task_ms": 224989.5588235294, "avg_output_tokens_per_task": 29168.869652406418, "last_run_timestamp": "2026-10-28T03:34:12.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.8755709520547945, "accuracy_std_dev": 0.007533567739715949, "run_count": 3, "total_questions": 584, "avg_cost_per_task_usd": 0.031521625, "avg_duration_per_task_ms": 247947.07705479453, "avg_output_tokens_per_task": 62802.890410958906, "last_run_timestamp": "2026-10-31T08:14:43.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Nebius", "accuracy": 0.875421, "accuracy_std_dev": 0.010101000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.05678013888888889, "avg_duration_per_task_ms": 453449.72222222225, "avg_output_tokens_per_task": 62821.7904040404, "last_run_timestamp": "2026-10-31T08:23:57.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.8754208571428571, "accuracy_std_dev": 0.009220961422271589, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1371511341991342, "avg_duration_per_task_ms": 439856.0331890332, "avg_output_tokens_per_task": 35614.08585858586, "last_run_timestamp": "2026-10-31T16:30:53.502Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Together", "accuracy": 0.8754206666666668, "accuracy_std_dev": 0.004955863351178644, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01787970538720539, "avg_duration_per_task_ms": 181946.53367003368, "avg_output_tokens_per_task": 29504.771043771045, "last_run_timestamp": "2026-10-31T08:09:31.246Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.8754205, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04156198232323233, "avg_duration_per_task_ms": 309570.8888888889, "avg_output_tokens_per_task": 39159.9494949495, "last_run_timestamp": "2026-10-17T08:26:18.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.8752677272727273, "accuracy_std_dev": 0.0335202236895631, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.19080393296602388, "avg_duration_per_task_ms": 47299.57070707071, "avg_output_tokens_per_task": 3603.7240587695132, "last_run_timestamp": "2026-10-31T08:09:25.606Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.8750631623672231, "accuracy_std_dev": 0.10207384869887097, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.02678827718765807, "avg_duration_per_task_ms": 188464.65857359636, "avg_output_tokens_per_task": 22634.93171471927, "last_run_timestamp": "2026-10-31T16:04:29.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.8746991428571429, "accuracy_std_dev": 0.005681157161518922, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.08339443073593074, "avg_duration_per_task_ms": 446738.63275613275, "avg_output_tokens_per_task": 23715.052669552668, "last_run_timestamp": "2026-10-31T16:15:54.664Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.8746888260869566, "accuracy_std_dev": 0.03857164733172279, "run_count": 23, "total_questions": 4554, "avg_cost_per_task_usd": 0.19603994861660076, "avg_duration_per_task_ms": 42405.70619235837, "avg_output_tokens_per_task": 3518.768115942029, "last_run_timestamp": "2026-11-11T21:00:44.482Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.8746087667870037, "accuracy_std_dev": 0.007592482877484693, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.2179415407942238, "avg_duration_per_task_ms": 1239989.4288808664, "avg_output_tokens_per_task": 48485.83321299639, "last_run_timestamp": "2026-10-24T16:18:35.554Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.8745317312013828, "accuracy_std_dev": 0.011186512454241005, "run_count": 6, "total_questions": 1157, "avg_cost_per_task_usd": 0.0043274416594641305, "avg_duration_per_task_ms": 185028.52031114954, "avg_output_tokens_per_task": 21870.51080380294, "last_run_timestamp": "2026-09-26T08:53:59.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.8744105999999999, "accuracy_std_dev": 0.014149500063253112, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.09050255303030302, "avg_duration_per_task_ms": 196310.64444444445, "avg_output_tokens_per_task": 14775.43686868687, "last_run_timestamp": "2026-10-31T08:07:27.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.8743250351006383, "accuracy_std_dev": 0.026296453097608783, "run_count": 21, "total_questions": 4074, "avg_cost_per_task_usd": 0.03975532817869414, "avg_duration_per_task_ms": 80921.18360333824, "avg_output_tokens_per_task": 25880.549828178693, "last_run_timestamp": "2026-10-31T09:03:20.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.8742938723163841, "accuracy_std_dev": 0.008148155125586256, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.009214897175141243, "avg_duration_per_task_ms": 375895.7553672316, "avg_output_tokens_per_task": 32530.198305084745, "last_run_timestamp": "2026-10-17T08:23:51.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.8740494840344654, "accuracy_std_dev": 0.0038017229265163467, "run_count": 10, "total_questions": 1973, "avg_cost_per_task_usd": 0.04434169640141917, "avg_duration_per_task_ms": 202197.7790167258, "avg_output_tokens_per_task": 38209.482513938165, "last_run_timestamp": "2026-10-31T08:10:37.345Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.8737693295358652, "accuracy_std_dev": 0.014113612302391693, "run_count": 12, "total_questions": 2370, "avg_cost_per_task_usd": 0.09247935569620254, "avg_duration_per_task_ms": 203634.65358649788, "avg_output_tokens_per_task": 23372.77805907173, "last_run_timestamp": "2026-10-31T08:40:26.156Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Nebius", "accuracy": 0.873737, "accuracy_std_dev": 0.007273551310513108, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01830383333333333, "avg_duration_per_task_ms": 381305.08585858584, "avg_output_tokens_per_task": 30305.045454545456, "last_run_timestamp": "2026-10-31T08:27:46.342Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0029526111111111116, "avg_duration_per_task_ms": 49722.71717171717, "avg_output_tokens_per_task": 5746.707070707071, "last_run_timestamp": "2026-10-01T08:23:37.659Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.873737, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.045554454545454544, "avg_duration_per_task_ms": 605480.6616161616, "avg_output_tokens_per_task": 35385.22727272727, "last_run_timestamp": "2026-10-01T08:40:06.225Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.8735040000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.0022876666666666666, "avg_duration_per_task_ms": 331580.09743589745, "avg_output_tokens_per_task": 16588.984615384616, "last_run_timestamp": "2026-08-22T10:20:26.941Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.8727751428571429, "accuracy_std_dev": 0.01839478573966448, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.047217443722943724, "avg_duration_per_task_ms": 671531.1060606061, "avg_output_tokens_per_task": 35962.7987012987, "last_run_timestamp": "2026-10-31T16:16:55.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.8726149999999999, "accuracy_std_dev": 0.0020998715833752102, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017651676767676767, "avg_duration_per_task_ms": 95564.90235690236, "avg_output_tokens_per_task": 29293.20202020202, "last_run_timestamp": "2026-10-31T08:33:05.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.8723940682248895, "accuracy_std_dev": 0.00799901581348451, "run_count": 8, "total_questions": 1583, "avg_cost_per_task_usd": 0.0495316885660139, "avg_duration_per_task_ms": 303242.33101705625, "avg_output_tokens_per_task": 21030.309538850284, "last_run_timestamp": "2026-10-31T16:14:42.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.8722941428571429, "accuracy_std_dev": 0.007658356601565021, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01657262121212121, "avg_duration_per_task_ms": 289911.9790764791, "avg_output_tokens_per_task": 29391.78354978355, "last_run_timestamp": "2026-10-31T16:04:48.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.8722639999999998, "accuracy_std_dev": 0.01391911622553672, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.01943295707070707, "avg_duration_per_task_ms": 314686.08207070705, "avg_output_tokens_per_task": 35123.857954545456, "last_run_timestamp": "2026-10-31T08:09:31.159Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.8722068181818181, "accuracy_std_dev": 0.028274856431813575, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.16051083654729112, "avg_duration_per_task_ms": 154457.4214876033, "avg_output_tokens_per_task": 15411.82598714417, "last_run_timestamp": "2026-10-31T08:57:30.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.8720540000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.12898378535353536, "avg_duration_per_task_ms": 597048.5454545454, "avg_output_tokens_per_task": 42966.707070707074, "last_run_timestamp": "2026-08-29T18:25:54.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.8720540000000001, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.09837352525252525, "avg_duration_per_task_ms": 729156.0252525252, "avg_output_tokens_per_task": 28490.656565656565, "last_run_timestamp": "2026-08-29T11:31:24.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Morph", "accuracy": 0.871875, "accuracy_std_dev": null, "run_count": 1, "total_questions": 160, "avg_cost_per_task_usd": 0.04507655, "avg_duration_per_task_ms": 1504372.61875, "avg_output_tokens_per_task": 18693.5375, "last_run_timestamp": "2026-08-27T15:00:05.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8718365483870967, "accuracy_std_dev": 0.025673440848897774, "run_count": 31, "total_questions": 6138, "avg_cost_per_task_usd": 0.17548565819485176, "avg_duration_per_task_ms": 159833.02085369828, "avg_output_tokens_per_task": 16263.557673509287, "last_run_timestamp": "2026-10-31T08:33:01.888Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.8714925000000001, "accuracy_std_dev": 0.007251768353305299, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.007901208754208754, "avg_duration_per_task_ms": 584978.2028619528, "avg_output_tokens_per_task": 44690.53282828283, "last_run_timestamp": "2026-10-31T16:48:05.771Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.8714859264426126, "accuracy_std_dev": 0.00752103815972237, "run_count": 8, "total_questions": 1577, "avg_cost_per_task_usd": 0.06945589727330374, "avg_duration_per_task_ms": 276858.71528218134, "avg_output_tokens_per_task": 11308.629676601142, "last_run_timestamp": "2026-10-10T08:22:14.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.8713324285714285, "accuracy_std_dev": 0.009851679854683843, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07396083838383838, "avg_duration_per_task_ms": 301937.24242424243, "avg_output_tokens_per_task": 21002.70779220779, "last_run_timestamp": "2026-10-31T16:09:19.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8712119166666668, "accuracy_std_dev": 0.038220154043075064, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1817093308080808, "avg_duration_per_task_ms": 45516.91035353536, "avg_output_tokens_per_task": 3430.526515151515, "last_run_timestamp": "2026-10-31T08:20:29.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8708191333333333, "accuracy_std_dev": 0.005011630059061523, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.004255378451178452, "avg_duration_per_task_ms": 39486.23198653199, "avg_output_tokens_per_task": 7745.265656565656, "last_run_timestamp": "2026-10-31T08:49:45.152Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.8707621944670464, "accuracy_std_dev": 0.005695171399115766, "run_count": 7, "total_questions": 1229, "avg_cost_per_task_usd": 0.005226334418226199, "avg_duration_per_task_ms": 743725.4109031733, "avg_output_tokens_per_task": 28735.099267697315, "last_run_timestamp": "2026-10-31T16:22:14.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.870748394132653, "accuracy_std_dev": 0.011444362026233704, "run_count": 4, "total_questions": 784, "avg_cost_per_task_usd": 0.02509891071428572, "avg_duration_per_task_ms": 366798.57908163266, "avg_output_tokens_per_task": 20671.150510204083, "last_run_timestamp": "2026-09-12T08:54:29.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.8703704, "accuracy_std_dev": 0.00583193054142453, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004018268686868687, "avg_duration_per_task_ms": 37454.91717171717, "avg_output_tokens_per_task": 7796.7444444444445, "last_run_timestamp": "2026-10-31T08:14:20.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "BaseTen", "accuracy": 0.8703701666666666, "accuracy_std_dev": 0.006941288747215641, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.024583196127946128, "avg_duration_per_task_ms": 150981.7457912458, "avg_output_tokens_per_task": 20283.477272727272, "last_run_timestamp": "2026-09-19T08:49:51.735Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.8703605355249205, "accuracy_std_dev": 0.03387763198941155, "run_count": 10, "total_questions": 1886, "avg_cost_per_task_usd": 0.014264545599151641, "avg_duration_per_task_ms": 348340.7725344645, "avg_output_tokens_per_task": 28651.488865323437, "last_run_timestamp": "2026-10-31T08:28:24.326Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.870112333932584, "accuracy_std_dev": 0.07630745338957075, "run_count": 23, "total_questions": 4450, "avg_cost_per_task_usd": 0.21229808741573034, "avg_duration_per_task_ms": 491047.8352808989, "avg_output_tokens_per_task": 61310.42764044944, "last_run_timestamp": "2026-10-31T09:02:18.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.8700336, "accuracy_std_dev": 0.008343089106559986, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.07480730858585859, "avg_duration_per_task_ms": 106906.41464646465, "avg_output_tokens_per_task": 12161.163636363637, "last_run_timestamp": "2026-10-31T08:03:09.289Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.8699400591341078, "accuracy_std_dev": 0.01681928135474285, "run_count": 10, "total_questions": 1894, "avg_cost_per_task_usd": 0.07171590390707498, "avg_duration_per_task_ms": 408524.5543822598, "avg_output_tokens_per_task": 23628.092397043296, "last_run_timestamp": "2026-10-31T08:45:16.438Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "PrimaLabs", "accuracy": 0.8697916770833333, "accuracy_std_dev": 0.045953881606091436, "run_count": 3, "total_questions": 576, "avg_cost_per_task_usd": 0.32188484548611107, "avg_duration_per_task_ms": 172479.83506944444, "avg_output_tokens_per_task": 40101.135416666664, "last_run_timestamp": "2026-11-06T16:32:49.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8697338870630417, "accuracy_std_dev": 0.011260986943785035, "run_count": 17, "total_questions": 3347, "avg_cost_per_task_usd": 0.04833613654018524, "avg_duration_per_task_ms": 200583.7881685091, "avg_output_tokens_per_task": 37254.27158649537, "last_run_timestamp": "2026-10-31T08:35:39.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.8696521878754779, "accuracy_std_dev": 0.02976323739044529, "run_count": 10, "total_questions": 1831, "avg_cost_per_task_usd": 0.022690866193336975, "avg_duration_per_task_ms": 451948.2801747679, "avg_output_tokens_per_task": 21491.623702894594, "last_run_timestamp": "2026-10-17T08:18:04.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.8695517762430939, "accuracy_std_dev": 0.027759107412948937, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.10362995395948434, "avg_duration_per_task_ms": 356439.69521178637, "avg_output_tokens_per_task": 23309.89779005525, "last_run_timestamp": "2026-10-31T08:27:45.609Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.8695284999999999, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2202375707070707, "avg_duration_per_task_ms": 264875.98484848486, "avg_output_tokens_per_task": 21926.78787878788, "last_run_timestamp": "2026-10-01T16:12:49.000Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "ModelRun", "accuracy": 0.8695284999999998, "accuracy_std_dev": 0.01157224020015143, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.018078708333333332, "avg_duration_per_task_ms": 141813.27146464647, "avg_output_tokens_per_task": 29935.99873737374, "last_run_timestamp": "2026-10-31T08:24:37.341Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.8694082857142857, "accuracy_std_dev": 0.006664936667029211, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008169142135642135, "avg_duration_per_task_ms": 374429.6991341991, "avg_output_tokens_per_task": 28832.51875901876, "last_run_timestamp": "2026-10-31T16:18:48.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8690236000000001, "accuracy_std_dev": 0.0056741455250989255, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004051724242424242, "avg_duration_per_task_ms": 38666.534343434345, "avg_output_tokens_per_task": 7872.538383838384, "last_run_timestamp": "2026-10-31T08:35:55.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.8687761417721519, "accuracy_std_dev": 0.03267208970972012, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.03197368734177215, "avg_duration_per_task_ms": 624455.1689873418, "avg_output_tokens_per_task": 38087.244303797466, "last_run_timestamp": "2026-11-10T07:01:38.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.868687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005480646464646464, "avg_duration_per_task_ms": 745039.691919192, "avg_output_tokens_per_task": 30273.38888888889, "last_run_timestamp": "2026-08-22T16:48:48.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.8686867500000001, "accuracy_std_dev": 0.009143975622643609, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.2541581641414141, "avg_duration_per_task_ms": 251149.7840909091, "avg_output_tokens_per_task": 24116.972222222223, "last_run_timestamp": "2026-10-01T16:14:35.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.8682058571428571, "accuracy_std_dev": 0.007673379706651383, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07688844300144301, "avg_duration_per_task_ms": 83582.13492063493, "avg_output_tokens_per_task": 12574.14357864358, "last_run_timestamp": "2026-10-10T08:21:46.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.8681285334935416, "accuracy_std_dev": 0.03709653553275389, "run_count": 17, "total_questions": 3329, "avg_cost_per_task_usd": 0.01566810363472514, "avg_duration_per_task_ms": 433897.7128266747, "avg_output_tokens_per_task": 40485.41213577651, "last_run_timestamp": "2026-11-04T21:50:43.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.8681253333333332, "accuracy_std_dev": 0.00965467639137749, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.021851893939393942, "avg_duration_per_task_ms": 524499.1397306398, "avg_output_tokens_per_task": 46276.513468013465, "last_run_timestamp": "2026-10-31T08:03:09.464Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "SambaNova", "accuracy": 0.8678450000000001, "accuracy_std_dev": 0.012168917536083462, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02358163005050505, "avg_duration_per_task_ms": 164224.13383838383, "avg_output_tokens_per_task": 28381.034090909092, "last_run_timestamp": "2026-11-11T19:09:33.977Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.8676650854638421, "accuracy_std_dev": 0.009102300097295443, "run_count": 7, "total_questions": 1369, "avg_cost_per_task_usd": 0.07595047918188458, "avg_duration_per_task_ms": 297193.17823228636, "avg_output_tokens_per_task": 20961.7319211103, "last_run_timestamp": "2026-10-31T16:05:36.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.8676153636363637, "accuracy_std_dev": 0.004014278793432891, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0702307419651056, "avg_duration_per_task_ms": 231760.75160697888, "avg_output_tokens_per_task": 11437.757575757576, "last_run_timestamp": "2026-10-31T08:17:29.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.8675645333333333, "accuracy_std_dev": 0.00826267276262484, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.004299042424242424, "avg_duration_per_task_ms": 36217.24208754209, "avg_output_tokens_per_task": 7829.657575757576, "last_run_timestamp": "2026-10-31T08:38:41.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.8675643333333334, "accuracy_std_dev": 0.0028614651647170092, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.009448984848484849, "avg_duration_per_task_ms": 79377.02525252526, "avg_output_tokens_per_task": 7744.79797979798, "last_run_timestamp": "2026-10-02T23:19:10.742Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.867521, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.08994571282051282, "avg_duration_per_task_ms": 589733.9384615384, "avg_output_tokens_per_task": 26825.066666666666, "last_run_timestamp": "2026-08-22T09:51:24.721Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.8674240000000001, "accuracy_std_dev": 0.0048169925783625705, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004064253787878788, "avg_duration_per_task_ms": 38624.37373737374, "avg_output_tokens_per_task": 7893.444444444444, "last_run_timestamp": "2026-10-31T08:27:45.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.8671716000000002, "accuracy_std_dev": 0.006408346388890055, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.0041782994949494945, "avg_duration_per_task_ms": 35977.948484848486, "avg_output_tokens_per_task": 7724.74494949495, "last_run_timestamp": "2026-10-31T09:34:38.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.867147968231047, "accuracy_std_dev": 0.04215195953321502, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.16483890613718413, "avg_duration_per_task_ms": 954961.5436823105, "avg_output_tokens_per_task": 56075.11624548736, "last_run_timestamp": "2026-10-31T16:11:41.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.8670032857142856, "accuracy_std_dev": 0.007740900274594989, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02579789898989899, "avg_duration_per_task_ms": 254191.68181818182, "avg_output_tokens_per_task": 21252.31313131313, "last_run_timestamp": "2026-10-08T22:14:16.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.867003, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.22265377272727274, "avg_duration_per_task_ms": 265427.9646464646, "avg_output_tokens_per_task": 22169.171717171717, "last_run_timestamp": "2026-10-01T16:12:54.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.8668349000000002, "accuracy_std_dev": 0.02737903360401897, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.008139828787878788, "avg_duration_per_task_ms": 416338.91818181815, "avg_output_tokens_per_task": 26892.52676767677, "last_run_timestamp": "2026-10-31T08:17:29.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.8665222857142858, "accuracy_std_dev": 0.007726115189302267, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10016765367965368, "avg_duration_per_task_ms": 312867.60533910536, "avg_output_tokens_per_task": 22150.73520923521, "last_run_timestamp": "2026-10-31T16:15:29.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.866391, "accuracy_std_dev": 0.031399399620780255, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1546900968778696, "avg_duration_per_task_ms": 152674.89577594123, "avg_output_tokens_per_task": 15252.516069788797, "last_run_timestamp": "2026-10-31T08:27:53.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.866331762553802, "accuracy_std_dev": 0.010391402837802988, "run_count": 8, "total_questions": 1394, "avg_cost_per_task_usd": 0.05799772740315638, "avg_duration_per_task_ms": 435430.42898134864, "avg_output_tokens_per_task": 19221.758967001435, "last_run_timestamp": "2026-10-31T16:05:14.702Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.86633, "accuracy_std_dev": 0.01754735666133223, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.10058035606060606, "avg_duration_per_task_ms": 457394.16717171716, "avg_output_tokens_per_task": 26933.21414141414, "last_run_timestamp": "2026-10-31T16:03:21.454Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.8661615, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.09752832323232324, "avg_duration_per_task_ms": 1426009.4393939395, "avg_output_tokens_per_task": 43223.868686868685, "last_run_timestamp": "2026-08-29T18:36:10.818Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.8657789090909093, "accuracy_std_dev": 0.014068643001399471, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.7534191322314051, "avg_duration_per_task_ms": 505384.1258034894, "avg_output_tokens_per_task": 29965.463269054177, "last_run_timestamp": "2026-10-31T08:23:25.857Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.8656005, "accuracy_std_dev": 0.00803956708515244, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.02525451262626263, "avg_duration_per_task_ms": 233486.3627946128, "avg_output_tokens_per_task": 20811.0765993266, "last_run_timestamp": "2026-10-01T16:08:04.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.86532, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.02509249494949495, "avg_duration_per_task_ms": 884843.8888888889, "avg_output_tokens_per_task": 25370.131313131315, "last_run_timestamp": "2026-08-22T10:03:34.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Relace", "accuracy": 0.86532, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.01495640909090909, "avg_duration_per_task_ms": 120441.0101010101, "avg_output_tokens_per_task": 29795.70707070707, "last_run_timestamp": "2026-10-24T15:00:02.228Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.8653197142857143, "accuracy_std_dev": 0.018961344600561016, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.16908653823953823, "avg_duration_per_task_ms": 1092542.5418470418, "avg_output_tokens_per_task": 53671.56421356421, "last_run_timestamp": "2026-10-31T16:05:14.701Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.8653195, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008214638888888888, "avg_duration_per_task_ms": 536604.0606060605, "avg_output_tokens_per_task": 28981.24494949495, "last_run_timestamp": "2026-08-29T18:49:16.416Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.8653195, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008587179292929294, "avg_duration_per_task_ms": 240899.30303030304, "avg_output_tokens_per_task": 30387.573232323233, "last_run_timestamp": "2026-08-29T18:38:15.907Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cohere", "accuracy": 0.8647586666666667, "accuracy_std_dev": 0.008943734131906087, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05048964983164983, "avg_duration_per_task_ms": 169537.7323232323, "avg_output_tokens_per_task": 41902.75589225589, "last_run_timestamp": "2026-10-31T08:15:03.392Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.8647076363636366, "accuracy_std_dev": 0.01038443121629445, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8432962465564738, "avg_duration_per_task_ms": 409882.468778696, "avg_output_tokens_per_task": 33540.75895316804, "last_run_timestamp": "2026-10-31T08:13:57.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.864685303613644, "accuracy_std_dev": 0.016464492614107477, "run_count": 15, "total_questions": 2961, "avg_cost_per_task_usd": 0.08223853157716987, "avg_duration_per_task_ms": 179300.8497129348, "avg_output_tokens_per_task": 24017.497804795676, "last_run_timestamp": "2026-11-04T21:43:38.767Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.8645034242424242, "accuracy_std_dev": 0.03391179125947315, "run_count": 33, "total_questions": 6534, "avg_cost_per_task_usd": 0.1657385367309458, "avg_duration_per_task_ms": 152502.7294153658, "avg_output_tokens_per_task": 15317.695592286502, "last_run_timestamp": "2026-10-31T08:51:08.192Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.8642484545454546, "accuracy_std_dev": 0.007188163178747238, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.7696710606060606, "avg_duration_per_task_ms": 439496.80716253445, "avg_output_tokens_per_task": 30614.35123966942, "last_run_timestamp": "2026-10-31T08:15:24.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.8640248461538463, "accuracy_std_dev": 0.04767222403655736, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.11144916588966587, "avg_duration_per_task_ms": 473283.12665112666, "avg_output_tokens_per_task": 28048.23271173271, "last_run_timestamp": "2026-11-10T06:16:52.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.863852179419525, "accuracy_std_dev": 0.02754414592685712, "run_count": 10, "total_questions": 1895, "avg_cost_per_task_usd": 0.012207302902374668, "avg_duration_per_task_ms": 440151.53984168865, "avg_output_tokens_per_task": 28225.067018469657, "last_run_timestamp": "2026-10-31T08:36:30.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.8636363333333335, "accuracy_std_dev": 0.008473463964374614, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.056405106902356894, "avg_duration_per_task_ms": 429778.9991582492, "avg_output_tokens_per_task": 55743.61616161616, "last_run_timestamp": "2026-10-31T16:10:09.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.86321525, "accuracy_std_dev": 0.03865413810560907, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0056033421717171725, "avg_duration_per_task_ms": 440745.6477272727, "avg_output_tokens_per_task": 17145.257575757576, "last_run_timestamp": "2026-11-05T09:00:05.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.86321525, "accuracy_std_dev": 0.010127171825712258, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.022006194444444443, "avg_duration_per_task_ms": 265798.13636363635, "avg_output_tokens_per_task": 33108.22601010101, "last_run_timestamp": "2026-09-19T08:51:58.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.8630751666666665, "accuracy_std_dev": 0.009048483766478349, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0646409191919192, "avg_duration_per_task_ms": 213655.1308922559, "avg_output_tokens_per_task": 10505.142255892257, "last_run_timestamp": "2026-10-31T08:13:57.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.8630240000000001, "accuracy_std_dev": 0.009717986304692007, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8468043273645547, "avg_duration_per_task_ms": 463572.84573002753, "avg_output_tokens_per_task": 33681.08218549128, "last_run_timestamp": "2026-10-31T08:07:51.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.8630240000000001, "accuracy_std_dev": 0.019570785500099618, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.7589622681359046, "avg_duration_per_task_ms": 512637.1014692378, "avg_output_tokens_per_task": 30220.091827364555, "last_run_timestamp": "2026-10-31T08:30:25.667Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.8629476002066115, "accuracy_std_dev": 0.031092890107547094, "run_count": 10, "total_questions": 1936, "avg_cost_per_task_usd": 0.13384704958677685, "avg_duration_per_task_ms": 479346.4189049587, "avg_output_tokens_per_task": 43979.465392561986, "last_run_timestamp": "2026-10-31T16:16:35.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.862870909090909, "accuracy_std_dev": 0.016717786165173394, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.7582240289256198, "avg_duration_per_task_ms": 509993.8374655647, "avg_output_tokens_per_task": 30186.383379247014, "last_run_timestamp": "2026-10-31T08:09:30.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.8627943333333332, "accuracy_std_dev": 0.06508388926010969, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1701148493265993, "avg_duration_per_task_ms": 2045429.0942760943, "avg_output_tokens_per_task": 60250.10101010101, "last_run_timestamp": "2026-10-24T16:14:24.978Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.8627287304492512, "accuracy_std_dev": 0.07377561754013702, "run_count": 7, "total_questions": 1202, "avg_cost_per_task_usd": 0.045114597337770374, "avg_duration_per_task_ms": 123217.97254575707, "avg_output_tokens_per_task": 27990.334442595675, "last_run_timestamp": "2026-10-31T16:08:08.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.8624118181818182, "accuracy_std_dev": 0.010959488075454655, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0702179696969697, "avg_duration_per_task_ms": 231211.1772268136, "avg_output_tokens_per_task": 11433.751147842057, "last_run_timestamp": "2026-10-31T08:01:54.465Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.862393, "accuracy_std_dev": null, "run_count": 1, "total_questions": 195, "avg_cost_per_task_usd": 0.11643089743589745, "avg_duration_per_task_ms": 352801.641025641, "avg_output_tokens_per_task": 29357.646153846155, "last_run_timestamp": "2026-08-22T10:19:36.962Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.8622894, "accuracy_std_dev": 0.02509268697130698, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0055417101010101, "avg_duration_per_task_ms": 936355.7888888889, "avg_output_tokens_per_task": 29101.413131313133, "last_run_timestamp": "2026-10-31T16:46:54.774Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.8622588181818182, "accuracy_std_dev": 0.01689329864617211, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.8046746588613406, "avg_duration_per_task_ms": 489059.52777777775, "avg_output_tokens_per_task": 30498.26652892562, "last_run_timestamp": "2026-10-31T09:25:22.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.8622526105675148, "accuracy_std_dev": 0.011754560721269014, "run_count": 8, "total_questions": 1533, "avg_cost_per_task_usd": 0.0046349673842139595, "avg_duration_per_task_ms": 249077.7371167645, "avg_output_tokens_per_task": 28899.352250489235, "last_run_timestamp": "2026-11-10T18:11:20.521Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.8622376508158508, "accuracy_std_dev": 0.05418494311168491, "run_count": 11, "total_questions": 2145, "avg_cost_per_task_usd": 0.07540476596736596, "avg_duration_per_task_ms": 485054.5156177156, "avg_output_tokens_per_task": 20286.058275058276, "last_run_timestamp": "2026-10-31T08:40:20.956Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8620293636363636, "accuracy_std_dev": 0.007276086157877692, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.8840942208448116, "avg_duration_per_task_ms": 412737.8966942149, "avg_output_tokens_per_task": 33491.24127640037, "last_run_timestamp": "2026-10-31T08:24:08.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.8620252278481012, "accuracy_std_dev": 0.030307500000000043, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.02300040253164557, "avg_duration_per_task_ms": 74866.74430379747, "avg_output_tokens_per_task": 18412.721518987342, "last_run_timestamp": "2026-10-17T16:05:00.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.86199412283237, "accuracy_std_dev": 0.01125442664474918, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.03778334826589595, "avg_duration_per_task_ms": 556168.3388728324, "avg_output_tokens_per_task": 35081.37789017341, "last_run_timestamp": "2026-10-31T16:11:17.542Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.8618590854662809, "accuracy_std_dev": 0.09124950603648056, "run_count": 23, "total_questions": 4493, "avg_cost_per_task_usd": 0.008348970843534385, "avg_duration_per_task_ms": 561394.6131760516, "avg_output_tokens_per_task": 28451.505230358336, "last_run_timestamp": "2026-11-06T22:28:22.713Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.8614718571428571, "accuracy_std_dev": 0.00972174233985086, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.014086771284271283, "avg_duration_per_task_ms": 839496.9516594517, "avg_output_tokens_per_task": 28569.064935064936, "last_run_timestamp": "2026-10-31T16:17:15.719Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.8614316774441879, "accuracy_std_dev": 0.016000304270056075, "run_count": 8, "total_questions": 1299, "avg_cost_per_task_usd": 0.013207368745188604, "avg_duration_per_task_ms": 402522.6505003849, "avg_output_tokens_per_task": 23550.163202463435, "last_run_timestamp": "2026-11-04T21:18:56.861Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.8607304246575341, "accuracy_std_dev": 0.01148253802030522, "run_count": 7, "total_questions": 1314, "avg_cost_per_task_usd": 0.02474931582952816, "avg_duration_per_task_ms": 139190.8691019787, "avg_output_tokens_per_task": 37895.824200913245, "last_run_timestamp": "2026-10-31T08:41:38.334Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.860546054237288, "accuracy_std_dev": 0.01522905267434664, "run_count": 9, "total_questions": 1770, "avg_cost_per_task_usd": 0.1431118, "avg_duration_per_task_ms": 301601.77118644066, "avg_output_tokens_per_task": 30086.04463276836, "last_run_timestamp": "2026-10-31T09:06:29.131Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Io Net", "accuracy": 0.860269, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00987439393939394, "avg_duration_per_task_ms": 74831.28787878787, "avg_output_tokens_per_task": 9997.058080808081, "last_run_timestamp": "2026-10-24T08:14:36.264Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.860028857142857, "accuracy_std_dev": 0.027595884472811062, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.08208907575757575, "avg_duration_per_task_ms": 365641.4213564214, "avg_output_tokens_per_task": 23745.34992784993, "last_run_timestamp": "2026-10-31T16:12:07.201Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.8597679673992673, "accuracy_std_dev": 0.009900199512162502, "run_count": 17, "total_questions": 2730, "avg_cost_per_task_usd": 0.17633374102564103, "avg_duration_per_task_ms": 246920.36886446885, "avg_output_tokens_per_task": 32290.727472527473, "last_run_timestamp": "2026-09-26T08:54:22.466Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Io Net", "accuracy": 0.8596746149425287, "accuracy_std_dev": 0.000831499999999985, "run_count": 2, "total_questions": 348, "avg_cost_per_task_usd": 0.0899417959770115, "avg_duration_per_task_ms": 1024745.4281609196, "avg_output_tokens_per_task": 35825.26724137931, "last_run_timestamp": "2026-10-27T13:57:40.548Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.8596166717241379, "accuracy_std_dev": 0.025707588739385667, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.08281105701149424, "avg_duration_per_task_ms": 496103.58022988506, "avg_output_tokens_per_task": 41921.770574712646, "last_run_timestamp": "2026-10-31T16:30:10.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.859429902017291, "accuracy_std_dev": 0.05098775585122179, "run_count": 11, "total_questions": 2082, "avg_cost_per_task_usd": 0.011825536023054757, "avg_duration_per_task_ms": 442610.81796349667, "avg_output_tokens_per_task": 25201.657060518733, "last_run_timestamp": "2026-10-31T08:27:46.434Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.8593071428571429, "accuracy_std_dev": 0.07000414295174368, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.07085715151515151, "avg_duration_per_task_ms": 1036918.955988456, "avg_output_tokens_per_task": 34951.957431457435, "last_run_timestamp": "2026-10-31T16:05:14.716Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.8592592, "accuracy_std_dev": 0.008288623417673138, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00619469898989899, "avg_duration_per_task_ms": 555649.1151515151, "avg_output_tokens_per_task": 32304.478787878787, "last_run_timestamp": "2026-10-31T16:16:55.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "StreamLake", "accuracy": 0.859217, "accuracy_std_dev": 0.016791584253726632, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.09269683143939395, "avg_duration_per_task_ms": 184470.00315656565, "avg_output_tokens_per_task": 25619.8952020202, "last_run_timestamp": "2026-10-31T16:04:29.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.8588918181818183, "accuracy_std_dev": 0.009103185045188405, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8537164830119376, "avg_duration_per_task_ms": 482176.7851239669, "avg_output_tokens_per_task": 33957.568411386594, "last_run_timestamp": "2026-10-31T08:48:00.920Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.8588278767123287, "accuracy_std_dev": 0.014585318196353787, "run_count": 3, "total_questions": 438, "avg_cost_per_task_usd": 0.005061143835616439, "avg_duration_per_task_ms": 414471.07305936073, "avg_output_tokens_per_task": 13714.753424657534, "last_run_timestamp": "2026-10-31T16:45:39.994Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.8588262857142857, "accuracy_std_dev": 0.023627943787656442, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.13331038239538243, "avg_duration_per_task_ms": 1309203.440836941, "avg_output_tokens_per_task": 52545.645021645025, "last_run_timestamp": "2026-10-31T16:08:31.067Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009637575757575758, "avg_duration_per_task_ms": 172308.71717171717, "avg_output_tokens_per_task": 16318.166666666666, "last_run_timestamp": "2026-08-22T09:56:05.550Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.08651329797979797, "avg_duration_per_task_ms": 499241.5707070707, "avg_output_tokens_per_task": 35858.23737373737, "last_run_timestamp": "2026-10-04T03:06:10.317Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flashx-20260918", "model_slug": "z-ai/glm-5.3-flashx", "model_name": "Z.ai: GLM 5.3 FlashX", "provider_name": "Novita", "accuracy": 0.858586, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.019546777777777778, "avg_duration_per_task_ms": 118523.70707070707, "avg_output_tokens_per_task": 15401.424242424242, "last_run_timestamp": "2026-11-12T21:02:44.111Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.8584327272727273, "accuracy_std_dev": 0.01770708058217958, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.8375554407713499, "avg_duration_per_task_ms": 457074.21533516987, "avg_output_tokens_per_task": 33325.96694214876, "last_run_timestamp": "2026-10-31T08:40:44.982Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.8584054285714288, "accuracy_std_dev": 0.00621956903323561, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.06424625072150071, "avg_duration_per_task_ms": 592730.4098124098, "avg_output_tokens_per_task": 35466.90873015873, "last_run_timestamp": "2026-10-31T16:08:56.835Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8581648333333333, "accuracy_std_dev": 0.025909089369798823, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.8336664709595959, "avg_duration_per_task_ms": 439138.7053872054, "avg_output_tokens_per_task": 33233.371632996634, "last_run_timestamp": "2026-10-31T08:35:35.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.8579124, "accuracy_std_dev": 0.017384854495796048, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.02057177575757576, "avg_duration_per_task_ms": 445221.4888888889, "avg_output_tokens_per_task": 40903.725252525255, "last_run_timestamp": "2026-10-31T08:25:16.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.8572763333333334, "accuracy_std_dev": 0.011384368786483787, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.030025383277216613, "avg_duration_per_task_ms": 811489.1139169473, "avg_output_tokens_per_task": 34239.19135802469, "last_run_timestamp": "2026-10-17T08:19:33.830Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.857222111631923, "accuracy_std_dev": 0.02112164754592157, "run_count": 41, "total_questions": 8107, "avg_cost_per_task_usd": 0.29757759652152466, "avg_duration_per_task_ms": 345556.9242629826, "avg_output_tokens_per_task": 53533.547551498705, "last_run_timestamp": "2026-10-31T16:26:37.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Wafer", "accuracy": 0.8569025, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018898722222222223, "avg_duration_per_task_ms": 164757.63383838383, "avg_output_tokens_per_task": 31337.035353535353, "last_run_timestamp": "2026-10-24T08:17:56.393Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.8569022222222222, "accuracy_std_dev": 0.02084630876346526, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.029755960157126823, "avg_duration_per_task_ms": 618931.7239057239, "avg_output_tokens_per_task": 34230.03367003367, "last_run_timestamp": "2026-11-10T07:06:13.165Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.8569019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20989276262626264, "avg_duration_per_task_ms": 263850.72222222225, "avg_output_tokens_per_task": 20893.070707070707, "last_run_timestamp": "2026-10-01T16:31:36.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.8568247201068804, "accuracy_std_dev": 0.015059304005690374, "run_count": 9, "total_questions": 1497, "avg_cost_per_task_usd": 0.004656902471609886, "avg_duration_per_task_ms": 413401.8376753507, "avg_output_tokens_per_task": 9906.163660654642, "last_run_timestamp": "2026-10-31T09:01:06.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.8565399329113924, "accuracy_std_dev": 0.05737653598692915, "run_count": 8, "total_questions": 1580, "avg_cost_per_task_usd": 0.07123199113924049, "avg_duration_per_task_ms": 1681972.4069620252, "avg_output_tokens_per_task": 20457.24683544304, "last_run_timestamp": "2026-10-31T16:13:09.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.8561370454545454, "accuracy_std_dev": 0.0107404034056508, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.15750592470156105, "avg_duration_per_task_ms": 443794.3243801653, "avg_output_tokens_per_task": 45571.12006427915, "last_run_timestamp": "2026-11-10T19:22:00.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.8560681935807422, "accuracy_std_dev": 0.011718959590718295, "run_count": 6, "total_questions": 997, "avg_cost_per_task_usd": 0.09006469408224674, "avg_duration_per_task_ms": 1284430.8184553662, "avg_output_tokens_per_task": 40831.585757271816, "last_run_timestamp": "2026-10-31T16:04:23.355Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.8556465066354411, "accuracy_std_dev": 0.020339777161859043, "run_count": 13, "total_questions": 2562, "avg_cost_per_task_usd": 0.08425942115534739, "avg_duration_per_task_ms": 311656.0573770492, "avg_output_tokens_per_task": 25603.65261514442, "last_run_timestamp": "2026-11-10T06:48:32.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.8552316718446601, "accuracy_std_dev": 0.0035618211195257714, "run_count": 8, "total_questions": 1545, "avg_cost_per_task_usd": 0.0868813637540453, "avg_duration_per_task_ms": 692640.7055016181, "avg_output_tokens_per_task": 43299.1715210356, "last_run_timestamp": "2026-10-31T16:11:17.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.8550984285714284, "accuracy_std_dev": 0.023860950363405426, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04428129942279942, "avg_duration_per_task_ms": 655320.3124098124, "avg_output_tokens_per_task": 36753.11616161616, "last_run_timestamp": "2026-10-31T16:02:14.360Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.8549381666666667, "accuracy_std_dev": 0.008098201002211657, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015701257575757577, "avg_duration_per_task_ms": 717198.1986531987, "avg_output_tokens_per_task": 19145.29797979798, "last_run_timestamp": "2026-10-31T16:05:36.294Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.854737857142857, "accuracy_std_dev": 0.020612976983503603, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04354637373737373, "avg_duration_per_task_ms": 599777.3571428572, "avg_output_tokens_per_task": 37914.28932178932, "last_run_timestamp": "2026-10-31T16:03:21.441Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.8545531474226805, "accuracy_std_dev": 0.018117679996070137, "run_count": 10, "total_questions": 1940, "avg_cost_per_task_usd": 0.08358154742268041, "avg_duration_per_task_ms": 325785.14793814434, "avg_output_tokens_per_task": 28959.811340206186, "last_run_timestamp": "2026-10-31T08:56:36.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Parasail", "accuracy": 0.85452, "accuracy_std_dev": null, "run_count": 1, "total_questions": 118, "avg_cost_per_task_usd": 0.11718616949152542, "avg_duration_per_task_ms": 289492.906779661, "avg_output_tokens_per_task": 26545.86440677966, "last_run_timestamp": "2026-08-22T17:29:49.714Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.8540377841483979, "accuracy_std_dev": 0.008466319992902585, "run_count": 9, "total_questions": 1779, "avg_cost_per_task_usd": 0.03240532996065205, "avg_duration_per_task_ms": 782505.4761101742, "avg_output_tokens_per_task": 33540.41765036537, "last_run_timestamp": "2026-10-17T08:19:39.242Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.8539581570952205, "accuracy_std_dev": 0.020366121093319547, "run_count": 14, "total_questions": 2699, "avg_cost_per_task_usd": 0.22091287662097076, "avg_duration_per_task_ms": 240855.18228973693, "avg_output_tokens_per_task": 44279.295294553536, "last_run_timestamp": "2026-10-24T16:29:51.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.8538717999999998, "accuracy_std_dev": 0.04946631113151657, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.005526387878787878, "avg_duration_per_task_ms": 190074.4404040404, "avg_output_tokens_per_task": 19384.131313131315, "last_run_timestamp": "2026-11-12T15:01:54.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.8537664327538884, "accuracy_std_dev": 0.01580166085394366, "run_count": 12, "total_questions": 2186, "avg_cost_per_task_usd": 0.04007263220494053, "avg_duration_per_task_ms": 147595.21180237876, "avg_output_tokens_per_task": 19176.248856358645, "last_run_timestamp": "2026-10-31T16:17:39.770Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.8537037, "accuracy_std_dev": 0.012630753865466617, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.10325877878787879, "avg_duration_per_task_ms": 455294.6515151515, "avg_output_tokens_per_task": 31795.09494949495, "last_run_timestamp": "2026-10-31T08:16:11.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.8536755833333335, "accuracy_std_dev": 0.03297161670148616, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.7094076557239056, "avg_duration_per_task_ms": 485859.81018518517, "avg_output_tokens_per_task": 28361.97053872054, "last_run_timestamp": "2026-10-31T08:09:42.551Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.853571253359394, "accuracy_std_dev": 0.031222921105250087, "run_count": 21, "total_questions": 4093, "avg_cost_per_task_usd": 0.003661041534326899, "avg_duration_per_task_ms": 73059.1441485463, "avg_output_tokens_per_task": 13652.101881260689, "last_run_timestamp": "2026-10-31T08:12:47.827Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.8533128573596358, "accuracy_std_dev": 0.012686455748159126, "run_count": 10, "total_questions": 1977, "avg_cost_per_task_usd": 0.08956905058168944, "avg_duration_per_task_ms": 291296.215477997, "avg_output_tokens_per_task": 29754.587253414265, "last_run_timestamp": "2026-11-10T18:31:05.774Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.8533057272727274, "accuracy_std_dev": 0.008106430938580997, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.09884094352617079, "avg_duration_per_task_ms": 553983.2718089991, "avg_output_tokens_per_task": 30406.843893480258, "last_run_timestamp": "2026-10-31T16:01:53.623Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.852923, "accuracy_std_dev": 0.04905195955496394, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.16991116437098255, "avg_duration_per_task_ms": 168270.75114784206, "avg_output_tokens_per_task": 16771.511019283746, "last_run_timestamp": "2026-10-31T08:12:45.468Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.8523164263322883, "accuracy_std_dev": 0.010034068893524688, "run_count": 5, "total_questions": 957, "avg_cost_per_task_usd": 0.08246928213166145, "avg_duration_per_task_ms": 272317.5339602926, "avg_output_tokens_per_task": 27908.60710553814, "last_run_timestamp": "2026-10-31T08:44:39.028Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.8521062280219781, "accuracy_std_dev": 0.006055500000000047, "run_count": 2, "total_questions": 364, "avg_cost_per_task_usd": 0.005501717032967033, "avg_duration_per_task_ms": 539158.467032967, "avg_output_tokens_per_task": 15491.851648351649, "last_run_timestamp": "2026-10-17T08:17:58.236Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Tenstorrent", "accuracy": 0.851852, "accuracy_std_dev": 0.005323694440893428, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.011784135101010102, "avg_duration_per_task_ms": 222866.32323232322, "avg_output_tokens_per_task": 29178.76893939394, "last_run_timestamp": "2026-11-04T18:07:25.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.8518518181818181, "accuracy_std_dev": 0.019763599225105198, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1265775752984389, "avg_duration_per_task_ms": 369781.93434343435, "avg_output_tokens_per_task": 40176.337006427915, "last_run_timestamp": "2026-10-31T16:08:08.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.8518517499999999, "accuracy_std_dev": 0.0020620621201845155, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.21063902146464644, "avg_duration_per_task_ms": 251655.29545454544, "avg_output_tokens_per_task": 19971.159090909092, "last_run_timestamp": "2026-10-01T16:27:58.636Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.8517752727272727, "accuracy_std_dev": 0.014241886054177125, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1367359797979798, "avg_duration_per_task_ms": 439341.9412304867, "avg_output_tokens_per_task": 49949.342975206615, "last_run_timestamp": "2026-10-31T16:09:19.244Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.8517346009049774, "accuracy_std_dev": 0.012647581720936017, "run_count": 7, "total_questions": 1105, "avg_cost_per_task_usd": 0.00618648778280543, "avg_duration_per_task_ms": 347941.31402714935, "avg_output_tokens_per_task": 15241.635294117646, "last_run_timestamp": "2026-10-31T16:08:33.701Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.8517072126779972, "accuracy_std_dev": 0.008685929974427657, "run_count": 11, "total_questions": 2177, "avg_cost_per_task_usd": 0.03815233899862195, "avg_duration_per_task_ms": 381837.27468994027, "avg_output_tokens_per_task": 38692.63114377584, "last_run_timestamp": "2026-10-31T16:26:59.494Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Parasail", "accuracy": 0.8512906666666668, "accuracy_std_dev": 0.00555530025191155, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.017986696969696972, "avg_duration_per_task_ms": 129036.91750841751, "avg_output_tokens_per_task": 29778.104377104377, "last_run_timestamp": "2026-10-31T08:56:30.449Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.8507849505644222, "accuracy_std_dev": 0.03243006205932389, "run_count": 13, "total_questions": 2569, "avg_cost_per_task_usd": 0.008747444920202414, "avg_duration_per_task_ms": 480929.24172829895, "avg_output_tokens_per_task": 31305.17399766446, "last_run_timestamp": "2026-11-10T08:10:11.354Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.8502042599087539, "accuracy_std_dev": 0.01590746948888509, "run_count": 37, "total_questions": 7014, "avg_cost_per_task_usd": 0.16371538508696884, "avg_duration_per_task_ms": 188635.26946107784, "avg_output_tokens_per_task": 29341.152266894784, "last_run_timestamp": "2026-10-31T09:24:22.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.8501685000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.20853091414141414, "avg_duration_per_task_ms": 280266.345959596, "avg_output_tokens_per_task": 20755.358585858587, "last_run_timestamp": "2026-10-01T16:20:44.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.850060552, "accuracy_std_dev": 0.01012123337849893, "run_count": 7, "total_questions": 1375, "avg_cost_per_task_usd": 0.0072516559999999996, "avg_duration_per_task_ms": 627511.1978181818, "avg_output_tokens_per_task": 17945.874909090908, "last_run_timestamp": "2026-10-31T16:12:28.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "River", "accuracy": 0.8498725725190839, "accuracy_std_dev": 0.009880999999999973, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.10012308905852417, "avg_duration_per_task_ms": 342870.14503816795, "avg_output_tokens_per_task": 28230.23409669211, "last_run_timestamp": "2026-11-12T21:00:32.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.8497887502109704, "accuracy_std_dev": 0.08999992173407093, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.08140508607594936, "avg_duration_per_task_ms": 452408.74599156115, "avg_output_tokens_per_task": 20689.04641350211, "last_run_timestamp": "2026-10-31T16:16:30.410Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.8497475000000001, "accuracy_std_dev": 0.008615114711946663, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.16274348863636365, "avg_duration_per_task_ms": 1045148.2323232323, "avg_output_tokens_per_task": 53462.20012626263, "last_run_timestamp": "2026-10-31T16:39:29.347Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "NextBit", "accuracy": 0.8495076506329113, "accuracy_std_dev": 0.005079115528864803, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.007176422784810127, "avg_duration_per_task_ms": 1312452.7088607594, "avg_output_tokens_per_task": 17778.172151898732, "last_run_timestamp": "2026-11-02T17:22:59.212Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Fireworks", "accuracy": 0.8494951090909092, "accuracy_std_dev": 0.012579000000000007, "run_count": 2, "total_questions": 165, "avg_cost_per_task_usd": 0.005448739393939394, "avg_duration_per_task_ms": 236133.1696969697, "avg_output_tokens_per_task": 25659.842424242423, "last_run_timestamp": "2026-10-31T16:10:36.259Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.8493265000000001, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2185174267676768, "avg_duration_per_task_ms": 797991.7474747475, "avg_output_tokens_per_task": 49726.7398989899, "last_run_timestamp": "2026-08-29T18:53:23.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.8492499090909091, "accuracy_std_dev": 0.017323134780636528, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.05284465794306703, "avg_duration_per_task_ms": 274246.5909090909, "avg_output_tokens_per_task": 45771.48347107438, "last_run_timestamp": "2026-10-31T16:53:23.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SambaNova", "accuracy": 0.8491455076923077, "accuracy_std_dev": 0.009587999999999985, "run_count": 2, "total_questions": 390, "avg_cost_per_task_usd": 0.14742170256410256, "avg_duration_per_task_ms": 263938.93333333335, "avg_output_tokens_per_task": 33257.10769230769, "last_run_timestamp": "2026-11-11T19:11:45.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "StreamLake", "accuracy": 0.8491083950617284, "accuracy_std_dev": 0.03011880589939707, "run_count": 6, "total_questions": 486, "avg_cost_per_task_usd": 0.026545792181069956, "avg_duration_per_task_ms": 90656.26131687243, "avg_output_tokens_per_task": 8757.810699588477, "last_run_timestamp": "2026-11-02T17:24:29.725Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Fireworks", "accuracy": 0.8490562452830188, "accuracy_std_dev": 0.004459999999999964, "run_count": 2, "total_questions": 265, "avg_cost_per_task_usd": 0.017200528301886793, "avg_duration_per_task_ms": 165535.98490566039, "avg_output_tokens_per_task": 28515.603773584906, "last_run_timestamp": "2026-10-31T08:20:07.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.8489438181818183, "accuracy_std_dev": 0.010179603259086118, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.054512841597796145, "avg_duration_per_task_ms": 250309.3691460055, "avg_output_tokens_per_task": 23083.604683195594, "last_run_timestamp": "2026-10-31T16:08:08.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.84890575, "accuracy_std_dev": 0.05035112340045156, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.1629483771043771, "avg_duration_per_task_ms": 162891.44065656565, "avg_output_tokens_per_task": 16117.493265993266, "last_run_timestamp": "2026-10-31T08:45:37.246Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.8488202684365781, "accuracy_std_dev": 0.035164132211494975, "run_count": 4, "total_questions": 678, "avg_cost_per_task_usd": 0.027723622418879056, "avg_duration_per_task_ms": 605422.2300884955, "avg_output_tokens_per_task": 13768.485250737464, "last_run_timestamp": "2026-10-17T08:18:07.707Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.8487092666666665, "accuracy_std_dev": 0.01899568044746548, "run_count": 30, "total_questions": 5940, "avg_cost_per_task_usd": 0.24548661565656565, "avg_duration_per_task_ms": 502671.36026936024, "avg_output_tokens_per_task": 48693.465656565655, "last_run_timestamp": "2026-10-31T17:01:17.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.8485243124999999, "accuracy_std_dev": 0.026025999999999994, "run_count": 2, "total_questions": 384, "avg_cost_per_task_usd": 0.007128966145833333, "avg_duration_per_task_ms": 721849.125, "avg_output_tokens_per_task": 18412.1875, "last_run_timestamp": "2026-08-29T18:08:44.008Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.848485, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0012582575757575758, "avg_duration_per_task_ms": 88400.0101010101, "avg_output_tokens_per_task": 17501.79797979798, "last_run_timestamp": "2026-10-01T08:22:54.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.8484845, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.09501090909090909, "avg_duration_per_task_ms": 411875.595959596, "avg_output_tokens_per_task": 30402.136363636364, "last_run_timestamp": "2026-08-29T11:19:17.684Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.848064, "accuracy_std_dev": 0.003834285135458768, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.05505487626262626, "avg_duration_per_task_ms": 405186.76767676766, "avg_output_tokens_per_task": 28115.5202020202, "last_run_timestamp": "2026-08-29T19:17:08.288Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.8479167041666666, "accuracy_std_dev": 0.00828816902819917, "run_count": 5, "total_questions": 960, "avg_cost_per_task_usd": 0.16138468541666665, "avg_duration_per_task_ms": 277988.090625, "avg_output_tokens_per_task": 24274.81875, "last_run_timestamp": "2026-10-28T05:08:57.249Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.8477888928571429, "accuracy_std_dev": 0.029215151613674717, "run_count": 10, "total_questions": 1960, "avg_cost_per_task_usd": 0.1592395025510204, "avg_duration_per_task_ms": 438119.2229591837, "avg_output_tokens_per_task": 36131.57244897959, "last_run_timestamp": "2026-10-31T08:13:34.182Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.8475227142857141, "accuracy_std_dev": 0.03525429692033209, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01139066955266955, "avg_duration_per_task_ms": 480834.867965368, "avg_output_tokens_per_task": 40336.09018759019, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.8475027499999999, "accuracy_std_dev": 0.011176715424824044, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.0940345744949495, "avg_duration_per_task_ms": 518383.0500841751, "avg_output_tokens_per_task": 34690.1148989899, "last_run_timestamp": "2026-10-31T16:05:36.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.8471073636363637, "accuracy_std_dev": 0.010373179378243821, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.019450327364554636, "avg_duration_per_task_ms": 145903.90633608814, "avg_output_tokens_per_task": 18981.601469237834, "last_run_timestamp": "2026-10-31T16:05:14.635Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.8469436303724929, "accuracy_std_dev": 0.009207938453420692, "run_count": 11, "total_questions": 2094, "avg_cost_per_task_usd": 0.06545764565425023, "avg_duration_per_task_ms": 187651.79751671443, "avg_output_tokens_per_task": 20359.965616045843, "last_run_timestamp": "2026-10-31T16:08:31.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.8469361045003815, "accuracy_std_dev": 0.04231320024956013, "run_count": 8, "total_questions": 1311, "avg_cost_per_task_usd": 0.02445259496567506, "avg_duration_per_task_ms": 437957.52021357743, "avg_output_tokens_per_task": 38826.28070175439, "last_run_timestamp": "2026-10-31T08:55:46.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.8468582586644126, "accuracy_std_dev": 0.013598361278927057, "run_count": 6, "total_questions": 1183, "avg_cost_per_task_usd": 0.26757311665257816, "avg_duration_per_task_ms": 498008.1420118343, "avg_output_tokens_per_task": 60614.17836010144, "last_run_timestamp": "2026-10-31T16:45:45.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.8466142222222222, "accuracy_std_dev": 0.010836379374616662, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.14184849326599328, "avg_duration_per_task_ms": 144926.00224466893, "avg_output_tokens_per_task": 32057.991582491584, "last_run_timestamp": "2026-10-31T08:40:49.272Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.8465387902641119, "accuracy_std_dev": 0.025032600265254102, "run_count": 10, "total_questions": 1931, "avg_cost_per_task_usd": 0.10684122164681513, "avg_duration_per_task_ms": 489966.58933195233, "avg_output_tokens_per_task": 34902.65044018643, "last_run_timestamp": "2026-10-31T09:15:06.724Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "KT", "accuracy": 0.8459595, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006682772727272727, "avg_duration_per_task_ms": 172002.25757575757, "avg_output_tokens_per_task": 19423.946969696968, "last_run_timestamp": "2026-11-11T01:47:13.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.8459293152073732, "accuracy_std_dev": 0.017246738162321286, "run_count": 11, "total_questions": 2170, "avg_cost_per_task_usd": 0.010452895391705068, "avg_duration_per_task_ms": 454110.96359447006, "avg_output_tokens_per_task": 34047.99170506912, "last_run_timestamp": "2026-10-31T08:27:51.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.8456636089524402, "accuracy_std_dev": 0.018418318454142355, "run_count": 17, "total_questions": 3217, "avg_cost_per_task_usd": 0.14255490332608023, "avg_duration_per_task_ms": 266408.3546782717, "avg_output_tokens_per_task": 28803.044140503574, "last_run_timestamp": "2026-10-31T08:33:10.900Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.8451602030567684, "accuracy_std_dev": 0.02474124360172708, "run_count": 5, "total_questions": 916, "avg_cost_per_task_usd": 0.028169002183406112, "avg_duration_per_task_ms": 430146.16484716156, "avg_output_tokens_per_task": 38733.88864628821, "last_run_timestamp": "2026-11-11T15:01:28.360Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.8451178333333333, "accuracy_std_dev": 0.04928392495671267, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.09621632239057239, "avg_duration_per_task_ms": 1254499.6717171718, "avg_output_tokens_per_task": 37856.02356902357, "last_run_timestamp": "2026-10-31T16:01:53.650Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.8446468343465044, "accuracy_std_dev": 0.01783018725336332, "run_count": 10, "total_questions": 1974, "avg_cost_per_task_usd": 0.0660811013171226, "avg_duration_per_task_ms": 245353.75430597772, "avg_output_tokens_per_task": 20336.286220871327, "last_run_timestamp": "2026-10-31T08:32:53.731Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.8444862499999999, "accuracy_std_dev": 0.011931925502093117, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.13937785227272728, "avg_duration_per_task_ms": 872540.9444444445, "avg_output_tokens_per_task": 55302.43308080808, "last_run_timestamp": "2026-10-31T16:10:36.259Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Modal", "accuracy": 0.844322, "accuracy_std_dev": null, "run_count": 1, "total_questions": 182, "avg_cost_per_task_usd": 0.0016337307692307693, "avg_duration_per_task_ms": 52884.66483516483, "avg_output_tokens_per_task": 15097.615384615385, "last_run_timestamp": "2026-11-04T09:00:10.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.844303929113924, "accuracy_std_dev": 0.0093105, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.008944994936708861, "avg_duration_per_task_ms": 480112.0911392405, "avg_output_tokens_per_task": 19303.701265822783, "last_run_timestamp": "2026-08-29T19:08:46.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.844276, "accuracy_std_dev": 0.009259000000000017, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.036391522727272724, "avg_duration_per_task_ms": 53072.40404040404, "avg_output_tokens_per_task": 14468.015151515152, "last_run_timestamp": "2026-10-01T19:12:26.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.844032090487239, "accuracy_std_dev": 0.01125640617422735, "run_count": 7, "total_questions": 1293, "avg_cost_per_task_usd": 0.007000730858468677, "avg_duration_per_task_ms": 651373.6117556071, "avg_output_tokens_per_task": 17260.92807424594, "last_run_timestamp": "2026-10-31T16:34:27.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.8439077604166666, "accuracy_std_dev": 0.009070260174817954, "run_count": 12, "total_questions": 2112, "avg_cost_per_task_usd": 0.12450989630681818, "avg_duration_per_task_ms": 153233.97348484848, "avg_output_tokens_per_task": 31747.522727272728, "last_run_timestamp": "2026-10-31T08:45:40.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cohere", "accuracy": 0.843434, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.11786523232323233, "avg_duration_per_task_ms": 258487.5, "avg_output_tokens_per_task": 33493.90909090909, "last_run_timestamp": "2026-11-13T00:01:54.511Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.8434174840336134, "accuracy_std_dev": 0.0058964576706039785, "run_count": 7, "total_questions": 1190, "avg_cost_per_task_usd": 0.07875789159663865, "avg_duration_per_task_ms": 236163.5705882353, "avg_output_tokens_per_task": 19324.579831932773, "last_run_timestamp": "2026-10-08T22:17:54.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.8433784397283531, "accuracy_std_dev": 0.003767029717016947, "run_count": 6, "total_questions": 1178, "avg_cost_per_task_usd": 0.1209133650254669, "avg_duration_per_task_ms": 239041.5050933786, "avg_output_tokens_per_task": 32072.49235993209, "last_run_timestamp": "2026-10-31T09:12:37.593Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.8432478707692308, "accuracy_std_dev": 0.040161275624730285, "run_count": 6, "total_questions": 975, "avg_cost_per_task_usd": 0.045744206153846154, "avg_duration_per_task_ms": 765827.1415384616, "avg_output_tokens_per_task": 33817.81538461539, "last_run_timestamp": "2026-09-26T09:56:00.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.8429531428571428, "accuracy_std_dev": 0.006641024607448381, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005697821789321789, "avg_duration_per_task_ms": 491338.5937950938, "avg_output_tokens_per_task": 26035.01370851371, "last_run_timestamp": "2026-10-31T16:45:40.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.8428716426914153, "accuracy_std_dev": 0.019938122355458276, "run_count": 7, "total_questions": 1293, "avg_cost_per_task_usd": 0.006783645784996132, "avg_duration_per_task_ms": 813203.3232791957, "avg_output_tokens_per_task": 18142.77648878577, "last_run_timestamp": "2026-10-31T16:04:29.121Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Darkbloom", "accuracy": 0.842743368078176, "accuracy_std_dev": 0.03194672742801677, "run_count": 5, "total_questions": 921, "avg_cost_per_task_usd": 0.011946617806731813, "avg_duration_per_task_ms": 1590074.9272529858, "avg_output_tokens_per_task": 34007.04125950054, "last_run_timestamp": "2026-11-13T03:05:40.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.8426947486187845, "accuracy_std_dev": 0.027181441039645704, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.01345692311233886, "avg_duration_per_task_ms": 682984.7329650092, "avg_output_tokens_per_task": 42267.71869244936, "last_run_timestamp": "2026-10-31T08:57:00.536Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.8425925, "accuracy_std_dev": 0.005892500000000023, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.08320908333333334, "avg_duration_per_task_ms": 696703.1691919192, "avg_output_tokens_per_task": 44847.378787878784, "last_run_timestamp": "2026-10-04T00:07:08.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.8425437868421053, "accuracy_std_dev": 0.00573439306339873, "run_count": 6, "total_questions": 1140, "avg_cost_per_task_usd": 0.08756267807017544, "avg_duration_per_task_ms": 282086.36140350875, "avg_output_tokens_per_task": 21480.616666666665, "last_run_timestamp": "2026-10-01T16:06:12.801Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.842516, "accuracy_std_dev": 0.010871595667777411, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.2414962369146006, "avg_duration_per_task_ms": 281047.5523415978, "avg_output_tokens_per_task": 54665.6129476584, "last_run_timestamp": "2026-10-31T16:57:18.670Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.8422337612565445, "accuracy_std_dev": 0.023212455418589382, "run_count": 10, "total_questions": 1910, "avg_cost_per_task_usd": 0.10300291204188483, "avg_duration_per_task_ms": 203070.60157068062, "avg_output_tokens_per_task": 30819.346073298428, "last_run_timestamp": "2026-10-31T08:52:06.333Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.8420896313446128, "accuracy_std_dev": 0.007016167749483269, "run_count": 6, "total_questions": 1123, "avg_cost_per_task_usd": 0.05385980587711488, "avg_duration_per_task_ms": 265038.4915405165, "avg_output_tokens_per_task": 22350.218165627783, "last_run_timestamp": "2026-09-26T09:50:39.842Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.842080958563536, "accuracy_std_dev": 0.01924201603413352, "run_count": 11, "total_questions": 2172, "avg_cost_per_task_usd": 0.02133896546961326, "avg_duration_per_task_ms": 337269.2900552486, "avg_output_tokens_per_task": 42459.25598526704, "last_run_timestamp": "2026-10-31T08:44:51.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.8420313611111111, "accuracy_std_dev": 0.011106999512701637, "run_count": 36, "total_questions": 7128, "avg_cost_per_task_usd": 0.15133402413019081, "avg_duration_per_task_ms": 168852.7923681257, "avg_output_tokens_per_task": 33127.37401795735, "last_run_timestamp": "2026-11-11T03:10:59.273Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.841751, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.15673991919191918, "avg_duration_per_task_ms": 1499085.4595959596, "avg_output_tokens_per_task": 52100.893939393936, "last_run_timestamp": "2026-08-29T18:25:52.753Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.841751, "accuracy_std_dev": 0.004761657064510223, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.06159810101010101, "avg_duration_per_task_ms": 79946.62794612795, "avg_output_tokens_per_task": 13559.319865319865, "last_run_timestamp": "2026-10-02T23:24:30.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.8416364037363778, "accuracy_std_dev": 0.015589563439410502, "run_count": 11, "total_questions": 1927, "avg_cost_per_task_usd": 0.1375403404255319, "avg_duration_per_task_ms": 272709.34509600414, "avg_output_tokens_per_task": 32850.36170212766, "last_run_timestamp": "2026-10-31T09:42:32.959Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.8414446363636365, "accuracy_std_dev": 0.019030622147536685, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.13825922910927457, "avg_duration_per_task_ms": 553721.6345270891, "avg_output_tokens_per_task": 44693.3769513315, "last_run_timestamp": "2026-10-31T16:18:28.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "PrimeIntellect", "accuracy": 0.841228, "accuracy_std_dev": null, "run_count": 1, "total_questions": 190, "avg_cost_per_task_usd": 0.0038195157894736842, "avg_duration_per_task_ms": 400775.6, "avg_output_tokens_per_task": 17024.936842105264, "last_run_timestamp": "2026-11-01T18:00:07.350Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.8412182282003711, "accuracy_std_dev": 0.02921932152413157, "run_count": 11, "total_questions": 2156, "avg_cost_per_task_usd": 0.06684410575139148, "avg_duration_per_task_ms": 1133131.4666048237, "avg_output_tokens_per_task": 33323.142393320966, "last_run_timestamp": "2026-10-31T16:05:36.103Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.8409620247933886, "accuracy_std_dev": 0.0474612402328205, "run_count": 8, "total_questions": 1573, "avg_cost_per_task_usd": 0.19201579465988555, "avg_duration_per_task_ms": 598063.229497775, "avg_output_tokens_per_task": 45700.98919262556, "last_run_timestamp": "2026-10-31T16:25:21.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.8409089999999999, "accuracy_std_dev": 0.045212357351171156, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0038926938131313127, "avg_duration_per_task_ms": 564281.6047979798, "avg_output_tokens_per_task": 43083.319444444445, "last_run_timestamp": "2026-10-31T16:18:02.140Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.8409089999999999, "accuracy_std_dev": 0.010943000000000036, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.03708839646464646, "avg_duration_per_task_ms": 47515.843434343435, "avg_output_tokens_per_task": 14746.045454545454, "last_run_timestamp": "2026-10-01T19:13:29.958Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.8409089999999999, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.025631659090909094, "avg_duration_per_task_ms": 236135.89141414143, "avg_output_tokens_per_task": 21303.184343434343, "last_run_timestamp": "2026-08-29T18:12:43.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.840891889212828, "accuracy_std_dev": 0.01539818649719325, "run_count": 16, "total_questions": 3087, "avg_cost_per_task_usd": 0.022577127632005183, "avg_duration_per_task_ms": 250189.5425979916, "avg_output_tokens_per_task": 38449.29964366699, "last_run_timestamp": "2026-10-31T08:54:15.638Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.8408359820854386, "accuracy_std_dev": 0.011558364030339736, "run_count": 11, "total_questions": 2177, "avg_cost_per_task_usd": 0.14478948920532844, "avg_duration_per_task_ms": 203069.29581993568, "avg_output_tokens_per_task": 32673.29122645843, "last_run_timestamp": "2026-10-31T09:21:48.528Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.8407407, "accuracy_std_dev": 0.024165205569371837, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.013536882323232325, "avg_duration_per_task_ms": 528967.1176767676, "avg_output_tokens_per_task": 40146.823737373736, "last_run_timestamp": "2026-10-31T08:45:25.206Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.8406767200863932, "accuracy_std_dev": 0.03362626844282443, "run_count": 12, "total_questions": 2315, "avg_cost_per_task_usd": 0.030011868250539957, "avg_duration_per_task_ms": 286135.6241900648, "avg_output_tokens_per_task": 9270.458315334774, "last_run_timestamp": "2026-10-31T08:16:12.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.8406685714285715, "accuracy_std_dev": 0.01850043854968342, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02123720418470418, "avg_duration_per_task_ms": 436896.4862914863, "avg_output_tokens_per_task": 42244.7886002886, "last_run_timestamp": "2026-10-31T08:09:30.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.8406374175298806, "accuracy_std_dev": 0.019535919283063564, "run_count": 8, "total_questions": 1255, "avg_cost_per_task_usd": 0.019024535458167333, "avg_duration_per_task_ms": 105363.14422310756, "avg_output_tokens_per_task": 16342.26374501992, "last_run_timestamp": "2026-11-11T18:58:39.610Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.8404879999999999, "accuracy_std_dev": 0.020003728864889173, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.037026556818181816, "avg_duration_per_task_ms": 113840.68308080808, "avg_output_tokens_per_task": 34919.68308080808, "last_run_timestamp": "2026-10-31T09:18:03.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.8403734545454546, "accuracy_std_dev": 0.034018740226698145, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.052804829201101924, "avg_duration_per_task_ms": 204681.84710743802, "avg_output_tokens_per_task": 21900.736455463728, "last_run_timestamp": "2026-10-31T16:28:22.255Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.8403390039479797, "accuracy_std_dev": 0.012076015641029829, "run_count": 22, "total_questions": 4306, "avg_cost_per_task_usd": 0.10892927728750582, "avg_duration_per_task_ms": 343834.3037621923, "avg_output_tokens_per_task": 34310.84417092429, "last_run_timestamp": "2026-11-06T23:05:09.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.8401781603650587, "accuracy_std_dev": 0.014388231606073072, "run_count": 16, "total_questions": 3068, "avg_cost_per_task_usd": 0.07506749576271186, "avg_duration_per_task_ms": 388300.5521512386, "avg_output_tokens_per_task": 22304.916558018253, "last_run_timestamp": "2026-10-31T08:43:48.577Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.8400672857142857, "accuracy_std_dev": 0.010723216824312801, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09338691269841272, "avg_duration_per_task_ms": 1164264.9783549784, "avg_output_tokens_per_task": 47107.84848484849, "last_run_timestamp": "2026-10-31T16:33:40.856Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.8398570000000002, "accuracy_std_dev": 0.010883944758220736, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.21990282449494947, "avg_duration_per_task_ms": 317333.1452020202, "avg_output_tokens_per_task": 49811.11111111111, "last_run_timestamp": "2026-09-12T08:53:53.147Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.8398265714285714, "accuracy_std_dev": 0.008557108588054464, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03983322727272727, "avg_duration_per_task_ms": 52955.70346320346, "avg_output_tokens_per_task": 14785.20634920635, "last_run_timestamp": "2026-10-01T19:13:57.987Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.8397870000000001, "accuracy_std_dev": 0.06305207943174174, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013501392255892255, "avg_duration_per_task_ms": 269245.1767676768, "avg_output_tokens_per_task": 24293.242424242424, "last_run_timestamp": "2026-10-01T17:44:47.331Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "SiliconFlow", "accuracy": 0.8394550909090909, "accuracy_std_dev": 0.011310932454565176, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0878581368227732, "avg_duration_per_task_ms": 712685.4439853076, "avg_output_tokens_per_task": 50105.391184573, "last_run_timestamp": "2026-10-31T16:46:10.481Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.8392255000000001, "accuracy_std_dev": 0.002525500000000014, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.028252058080808082, "avg_duration_per_task_ms": 430220.3787878788, "avg_output_tokens_per_task": 23440.575757575756, "last_run_timestamp": "2026-08-29T18:16:26.161Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.8391051428571429, "accuracy_std_dev": 0.00943165809432062, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.150706683982684, "avg_duration_per_task_ms": 316554.5598845599, "avg_output_tokens_per_task": 62067.65873015873, "last_run_timestamp": "2026-10-31T16:45:48.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.8389995344074279, "accuracy_std_dev": 0.011455015921374732, "run_count": 37, "total_questions": 7324, "avg_cost_per_task_usd": 0.26465632960131075, "avg_duration_per_task_ms": 285335.27007099945, "avg_output_tokens_per_task": 57439.03536318951, "last_run_timestamp": "2026-11-11T03:18:20.413Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.8387440275454993, "accuracy_std_dev": 0.019690853286246784, "run_count": 11, "total_questions": 2033, "avg_cost_per_task_usd": 0.07400631874077719, "avg_duration_per_task_ms": 403244.14658140676, "avg_output_tokens_per_task": 24425.756025577964, "last_run_timestamp": "2026-10-31T16:18:04.837Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.8386898181818183, "accuracy_std_dev": 0.022320469186247312, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1030642401285583, "avg_duration_per_task_ms": 384608.90220385673, "avg_output_tokens_per_task": 35835.99357208448, "last_run_timestamp": "2026-10-31T08:36:48.545Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.8385089339622642, "accuracy_std_dev": 0.013106640774000717, "run_count": 13, "total_questions": 2544, "avg_cost_per_task_usd": 0.3152681855345912, "avg_duration_per_task_ms": 413001.90290880506, "avg_output_tokens_per_task": 53140.55856918239, "last_run_timestamp": "2026-10-31T16:56:08.871Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.8383837500000001, "accuracy_std_dev": 0.0064105770167357185, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.06462981691919192, "avg_duration_per_task_ms": 108153.94696969698, "avg_output_tokens_per_task": 13551.515151515152, "last_run_timestamp": "2026-10-01T19:14:05.661Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.8383837142857142, "accuracy_std_dev": 0.012139811903865117, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1527254054834055, "avg_duration_per_task_ms": 220532.05050505052, "avg_output_tokens_per_task": 34470.08658008658, "last_run_timestamp": "2026-10-31T08:45:52.318Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.8383140372413792, "accuracy_std_dev": 0.03002615781240096, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.08971148505747126, "avg_duration_per_task_ms": 771909.1783908046, "avg_output_tokens_per_task": 31778.30390804598, "last_run_timestamp": "2026-10-31T16:01:53.624Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Together", "accuracy": 0.8380034220665499, "accuracy_std_dev": 0.010074705190062243, "run_count": 3, "total_questions": 571, "avg_cost_per_task_usd": 0.01617205078809107, "avg_duration_per_task_ms": 843918.8598949212, "avg_output_tokens_per_task": 17113.875656742555, "last_run_timestamp": "2026-08-29T18:29:51.235Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.8378226666666666, "accuracy_std_dev": 0.00975199857579064, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.15828378956228956, "avg_duration_per_task_ms": 446608.7861952862, "avg_output_tokens_per_task": 56298.93097643097, "last_run_timestamp": "2026-10-10T08:30:28.864Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.8375420000000001, "accuracy_std_dev": 0.007576000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.028651648989898987, "avg_duration_per_task_ms": 499930.6313131313, "avg_output_tokens_per_task": 23687.20202020202, "last_run_timestamp": "2026-08-29T18:16:39.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.837542, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.06186689898989899, "avg_duration_per_task_ms": 75089.61363636363, "avg_output_tokens_per_task": 13620.507575757576, "last_run_timestamp": "2026-10-01T19:12:44.401Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.8374978740239459, "accuracy_std_dev": 0.020695537575457914, "run_count": 11, "total_questions": 1921, "avg_cost_per_task_usd": 0.09704594429984383, "avg_duration_per_task_ms": 615903.0510150963, "avg_output_tokens_per_task": 30251.17542946382, "last_run_timestamp": "2026-10-31T16:13:36.316Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.8371593636363637, "accuracy_std_dev": 0.013027195241792026, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.141865238292011, "avg_duration_per_task_ms": 181084.47015610652, "avg_output_tokens_per_task": 32015.05968778696, "last_run_timestamp": "2026-10-31T08:16:11.213Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.8369407142857143, "accuracy_std_dev": 0.020626950634783515, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10078755411255412, "avg_duration_per_task_ms": 1217533.0151515151, "avg_output_tokens_per_task": 46038.86075036075, "last_run_timestamp": "2026-10-31T16:08:56.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.836139, "accuracy_std_dev": 0.0007933738084913017, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.05002690909090909, "avg_duration_per_task_ms": 537584.335016835, "avg_output_tokens_per_task": 49932.06734006734, "last_run_timestamp": "2026-09-30T11:44:08.104Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.8361277244542866, "accuracy_std_dev": 0.026614847390527968, "run_count": 16, "total_questions": 3161, "avg_cost_per_task_usd": 0.02298093546346093, "avg_duration_per_task_ms": 307170.6646630813, "avg_output_tokens_per_task": 18491.486871243276, "last_run_timestamp": "2026-10-31T08:31:11.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.8358585, "accuracy_std_dev": 0.01430950000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0011451691919191918, "avg_duration_per_task_ms": 116299.33333333333, "avg_output_tokens_per_task": 18355.219696969696, "last_run_timestamp": "2026-10-01T08:23:48.135Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.8356902, "accuracy_std_dev": 0.013324017868496005, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0493619898989899, "avg_duration_per_task_ms": 1019058.498989899, "avg_output_tokens_per_task": 29486.257575757576, "last_run_timestamp": "2026-10-17T08:20:47.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.835577888888889, "accuracy_std_dev": 0.03515262553634887, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.10956455723905723, "avg_duration_per_task_ms": 1079749.1066217732, "avg_output_tokens_per_task": 34161.65993265993, "last_run_timestamp": "2026-10-24T16:17:57.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.8351849999999998, "accuracy_std_dev": null, "run_count": 1, "total_questions": 180, "avg_cost_per_task_usd": 0.0030483999999999997, "avg_duration_per_task_ms": 288802.22222222225, "avg_output_tokens_per_task": 16703.5, "last_run_timestamp": "2026-10-01T08:43:11.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.8350828124309392, "accuracy_std_dev": 0.015312607695726205, "run_count": 19, "total_questions": 3620, "avg_cost_per_task_usd": 0.04118064751381217, "avg_duration_per_task_ms": 454296.9718232044, "avg_output_tokens_per_task": 45405.74917127072, "last_run_timestamp": "2026-10-31T16:27:38.037Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.8348581735005084, "accuracy_std_dev": 0.011926219294758369, "run_count": 18, "total_questions": 2951, "avg_cost_per_task_usd": 0.1190305462555066, "avg_duration_per_task_ms": 291312.74483226024, "avg_output_tokens_per_task": 28021.837004405286, "last_run_timestamp": "2026-10-31T08:09:30.690Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.8347106363636364, "accuracy_std_dev": 0.07009507473200917, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.21278814876033061, "avg_duration_per_task_ms": 570294.2180899908, "avg_output_tokens_per_task": 48358.50367309458, "last_run_timestamp": "2026-10-31T16:18:48.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.8346120933503837, "accuracy_std_dev": 0.029860364128538167, "run_count": 6, "total_questions": 782, "avg_cost_per_task_usd": 0.09530405242966751, "avg_duration_per_task_ms": 330065.4283887468, "avg_output_tokens_per_task": 39472.66240409207, "last_run_timestamp": "2026-10-31T16:08:31.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.8345096889411766, "accuracy_std_dev": 0.010890152035457172, "run_count": 11, "total_questions": 2125, "avg_cost_per_task_usd": 0.2468675487058824, "avg_duration_per_task_ms": 311483.9990588235, "avg_output_tokens_per_task": 55935.44047058823, "last_run_timestamp": "2026-10-31T16:28:00.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Modal", "accuracy": 0.8344576188870152, "accuracy_std_dev": 0.007843600547934322, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.033416617200674535, "avg_duration_per_task_ms": 208747.37942664418, "avg_output_tokens_per_task": 14517.00168634064, "last_run_timestamp": "2026-10-31T08:47:23.899Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.8341543388067869, "accuracy_std_dev": 0.008301927538228698, "run_count": 10, "total_questions": 1827, "avg_cost_per_task_usd": 0.0431522140120416, "avg_duration_per_task_ms": 94743.94636015326, "avg_output_tokens_per_task": 15061.821018062397, "last_run_timestamp": "2026-10-31T08:22:53.415Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.8341254705882353, "accuracy_std_dev": 0.010649185432078013, "run_count": 17, "total_questions": 3366, "avg_cost_per_task_usd": 0.25025737195484254, "avg_duration_per_task_ms": 384709.65567439096, "avg_output_tokens_per_task": 40253.002970885325, "last_run_timestamp": "2026-10-31T08:41:02.403Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.8340364430379746, "accuracy_std_dev": 0.01069546220626094, "run_count": 6, "total_questions": 1185, "avg_cost_per_task_usd": 0.0954248565400844, "avg_duration_per_task_ms": 293770.05400843883, "avg_output_tokens_per_task": 23403.932489451476, "last_run_timestamp": "2026-10-01T16:11:05.832Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.8338943333333333, "accuracy_std_dev": 0.011690806112307073, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.03773407239057239, "avg_duration_per_task_ms": 276396.1228956229, "avg_output_tokens_per_task": 28290.345117845118, "last_run_timestamp": "2026-09-12T09:56:44.384Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.833814142857143, "accuracy_std_dev": 0.020613070322551417, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.10298939610389608, "avg_duration_per_task_ms": 1389091.401875902, "avg_output_tokens_per_task": 47060.21284271284, "last_run_timestamp": "2026-10-31T16:08:33.697Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.8334736666666667, "accuracy_std_dev": 0.007416946298099319, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.08273329461279462, "avg_duration_per_task_ms": 719974.9494949495, "avg_output_tokens_per_task": 60693.068181818184, "last_run_timestamp": "2026-10-31T16:05:36.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.8333335000000001, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02674798484848485, "avg_duration_per_task_ms": 520849.86363636365, "avg_output_tokens_per_task": 22257.11111111111, "last_run_timestamp": "2026-08-29T19:19:06.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.8333332, "accuracy_std_dev": 0.00946378413532349, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.10132244444444444, "avg_duration_per_task_ms": 359966.904040404, "avg_output_tokens_per_task": 46075.3101010101, "last_run_timestamp": "2026-10-17T16:06:16.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Makora", "accuracy": 0.8333329999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.003982064102564103, "avg_duration_per_task_ms": 34178.44871794872, "avg_output_tokens_per_task": 8341.692307692309, "last_run_timestamp": "2026-09-09T15:03:02.824Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.8330927142857143, "accuracy_std_dev": 0.013865607871011393, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.23029656782106786, "avg_duration_per_task_ms": 1228281.2157287158, "avg_output_tokens_per_task": 52325.608225108226, "last_run_timestamp": "2026-10-31T16:30:07.021Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.8330725023474179, "accuracy_std_dev": 0.010168177287347969, "run_count": 7, "total_questions": 1278, "avg_cost_per_task_usd": 0.029710525039123633, "avg_duration_per_task_ms": 655555.5469483568, "avg_output_tokens_per_task": 42323.18544600939, "last_run_timestamp": "2026-10-03T09:40:53.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.8329060884237507, "accuracy_std_dev": 0.015886988525511157, "run_count": 12, "total_questions": 2341, "avg_cost_per_task_usd": 0.2567825809483127, "avg_duration_per_task_ms": 427385.35027765913, "avg_output_tokens_per_task": 58144.50192225545, "last_run_timestamp": "2026-10-31T16:06:46.433Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.832772, "accuracy_std_dev": 0.0698509274145066, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.024015700336700335, "avg_duration_per_task_ms": 613738.0185185185, "avg_output_tokens_per_task": 59848.74074074074, "last_run_timestamp": "2026-10-31T08:32:40.133Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.8327310906791908, "accuracy_std_dev": 0.020914023298509846, "run_count": 14, "total_questions": 2768, "avg_cost_per_task_usd": 0.02508389053468208, "avg_duration_per_task_ms": 393221.2268786127, "avg_output_tokens_per_task": 46919.84176300578, "last_run_timestamp": "2026-10-31T08:25:10.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.8326857692307692, "accuracy_std_dev": 0.011020358329744859, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.03860720590520591, "avg_duration_per_task_ms": 230762.9052059052, "avg_output_tokens_per_task": 14476.024864024865, "last_run_timestamp": "2026-10-31T08:25:00.370Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.8325688873598369, "accuracy_std_dev": 0.019237670701256954, "run_count": 10, "total_questions": 1962, "avg_cost_per_task_usd": 0.013576182466870541, "avg_duration_per_task_ms": 438297.6029561672, "avg_output_tokens_per_task": 39848.997961264016, "last_run_timestamp": "2026-10-31T08:26:26.747Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.8325147922396857, "accuracy_std_dev": 0.029981271466154875, "run_count": 11, "total_questions": 2036, "avg_cost_per_task_usd": 0.011545964145383104, "avg_duration_per_task_ms": 399504.8664047151, "avg_output_tokens_per_task": 22873.502946954814, "last_run_timestamp": "2026-10-31T08:40:04.563Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.8323711428571429, "accuracy_std_dev": 0.01158903863174871, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008056092352092353, "avg_duration_per_task_ms": 525780.0266955267, "avg_output_tokens_per_task": 20733.657287157286, "last_run_timestamp": "2026-10-31T16:01:53.649Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.8323670282608697, "accuracy_std_dev": 0.013729350448377906, "run_count": 8, "total_questions": 1380, "avg_cost_per_task_usd": 0.20936515, "avg_duration_per_task_ms": 475047.03623188403, "avg_output_tokens_per_task": 47461.076811594205, "last_run_timestamp": "2026-10-31T16:35:08.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.8320348961038961, "accuracy_std_dev": 0.011673500000000003, "run_count": 2, "total_questions": 385, "avg_cost_per_task_usd": 0.02834737922077922, "avg_duration_per_task_ms": 116343.6909090909, "avg_output_tokens_per_task": 22533.716883116882, "last_run_timestamp": "2026-08-29T18:29:13.946Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.8318478292335116, "accuracy_std_dev": 0.010811745465322452, "run_count": 17, "total_questions": 2805, "avg_cost_per_task_usd": 0.020570678431372547, "avg_duration_per_task_ms": 255956.05668449198, "avg_output_tokens_per_task": 40944.02673796791, "last_run_timestamp": "2026-10-31T08:20:07.217Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.8317821610051186, "accuracy_std_dev": 0.013001907940092663, "run_count": 11, "total_questions": 2149, "avg_cost_per_task_usd": 0.15365638017682642, "avg_duration_per_task_ms": 272025.46998604, "avg_output_tokens_per_task": 34707.94276407632, "last_run_timestamp": "2026-11-05T00:00:09.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.831769061433447, "accuracy_std_dev": 0.006801018573632123, "run_count": 12, "total_questions": 2344, "avg_cost_per_task_usd": 0.03448967320819113, "avg_duration_per_task_ms": 208520.74530716724, "avg_output_tokens_per_task": 14194.120733788395, "last_run_timestamp": "2026-10-31T08:37:28.112Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-20261104", "model_slug": "tencent/hy4", "model_name": "Tencent: Hy4", "provider_name": "Tencent", "accuracy": 0.83165, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06418498484848485, "avg_duration_per_task_ms": 493220.3232323232, "avg_output_tokens_per_task": 51169.5101010101, "last_run_timestamp": "2026-11-10T19:46:03.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.8316496, "accuracy_std_dev": 0.007062535765573171, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.022192562626262628, "avg_duration_per_task_ms": 11557.427272727273, "avg_output_tokens_per_task": 14346.162626262627, "last_run_timestamp": "2026-10-31T08:15:46.337Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.8316495, "accuracy_std_dev": 0.055555500000000035, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04943669696969697, "avg_duration_per_task_ms": 1128190.707070707, "avg_output_tokens_per_task": 25885.7702020202, "last_run_timestamp": "2026-08-29T19:26:41.546Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "PrimeIntellect", "accuracy": 0.831532, "accuracy_std_dev": null, "run_count": 1, "total_questions": 185, "avg_cost_per_task_usd": 0.01692656216216216, "avg_duration_per_task_ms": 560352.7027027027, "avg_output_tokens_per_task": 13578.437837837839, "last_run_timestamp": "2026-10-31T18:41:15.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.8314150997442455, "accuracy_std_dev": 0.012107134010057875, "run_count": 8, "total_questions": 1564, "avg_cost_per_task_usd": 0.02770090409207161, "avg_duration_per_task_ms": 583377.9558823529, "avg_output_tokens_per_task": 14708.68989769821, "last_run_timestamp": "2026-10-31T08:48:50.088Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.8313539815914488, "accuracy_std_dev": 0.013525888763386546, "run_count": 9, "total_questions": 1684, "avg_cost_per_task_usd": 0.02395000890736342, "avg_duration_per_task_ms": 287524.2090261283, "avg_output_tokens_per_task": 47679.917458432305, "last_run_timestamp": "2026-10-31T08:55:40.953Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Amazon Bedrock", "accuracy": 0.8310883333333333, "accuracy_std_dev": 0.009654676391377534, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.20155932323232326, "avg_duration_per_task_ms": 266665.86195286195, "avg_output_tokens_per_task": 37910.23063973064, "last_run_timestamp": "2026-10-31T08:09:31.329Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.8309762, "accuracy_std_dev": 0.004338052761320463, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.04722357777777777, "avg_duration_per_task_ms": 258256.24646464645, "avg_output_tokens_per_task": 15298.06111111111, "last_run_timestamp": "2026-10-31T08:20:09.369Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "CoreWeave", "accuracy": 0.830421644785601, "accuracy_std_dev": 0.015773715151716793, "run_count": 12, "total_questions": 1889, "avg_cost_per_task_usd": 0.04062480518793012, "avg_duration_per_task_ms": 302854.55849655904, "avg_output_tokens_per_task": 33135.881948120696, "last_run_timestamp": "2026-10-31T16:19:08.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.8302074978687126, "accuracy_std_dev": 0.018798024809619108, "run_count": 12, "total_questions": 2346, "avg_cost_per_task_usd": 0.14303728772378516, "avg_duration_per_task_ms": 291115.02642796247, "avg_output_tokens_per_task": 32292.15643648764, "last_run_timestamp": "2026-10-31T08:14:43.196Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.82996625, "accuracy_std_dev": 0.034767651864736286, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.20791771717171717, "avg_duration_per_task_ms": 1562678.1717171718, "avg_output_tokens_per_task": 49349.89141414141, "last_run_timestamp": "2026-10-31T16:31:54.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.8294050000000001, "accuracy_std_dev": 0.004827531943619518, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02358655555555556, "avg_duration_per_task_ms": 376720.68518518517, "avg_output_tokens_per_task": 21014.917508417508, "last_run_timestamp": "2026-10-04T00:17:30.035Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.8288813468715698, "accuracy_std_dev": 0.048221988340541345, "run_count": 14, "total_questions": 2733, "avg_cost_per_task_usd": 0.09670949213318698, "avg_duration_per_task_ms": 542922.2052689352, "avg_output_tokens_per_task": 29030.10171972192, "last_run_timestamp": "2026-10-31T08:36:47.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "SiliconFlow", "accuracy": 0.8285234285714286, "accuracy_std_dev": 0.008462147918097126, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.048541420634920646, "avg_duration_per_task_ms": 445627.215007215, "avg_output_tokens_per_task": 26857.45165945166, "last_run_timestamp": "2026-10-31T16:10:31.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.8284931249999998, "accuracy_std_dev": 0.006861031326220224, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.029469156565656567, "avg_duration_per_task_ms": 133597.33396464647, "avg_output_tokens_per_task": 22565.453914141413, "last_run_timestamp": "2026-10-31T16:08:08.538Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.82828275, "accuracy_std_dev": 0.004610363346581274, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0223316047979798, "avg_duration_per_task_ms": 615131.3409090909, "avg_output_tokens_per_task": 56159.15404040404, "last_run_timestamp": "2026-10-17T08:18:07.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.8281987, "accuracy_std_dev": 0.027781391164050814, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.038244685353535354, "avg_duration_per_task_ms": 378160.9792929293, "avg_output_tokens_per_task": 38488.014646464646, "last_run_timestamp": "2026-10-24T16:01:48.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.8281297272727273, "accuracy_std_dev": 0.007612402500834456, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.06470958723599633, "avg_duration_per_task_ms": 560112.3668503214, "avg_output_tokens_per_task": 50830.36317722681, "last_run_timestamp": "2026-10-31T16:08:56.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.8280023333333335, "accuracy_std_dev": 0.027257023221344042, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.1507835429292929, "avg_duration_per_task_ms": 1669561.2037037036, "avg_output_tokens_per_task": 49318.73569023569, "last_run_timestamp": "2026-10-31T16:08:56.833Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.8275762754844144, "accuracy_std_dev": 0.06091378546606307, "run_count": 6, "total_questions": 1187, "avg_cost_per_task_usd": 0.20493896293176075, "avg_duration_per_task_ms": 1744742.5619208089, "avg_output_tokens_per_task": 46988.62679022746, "last_run_timestamp": "2026-10-31T16:22:14.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.8275058461538461, "accuracy_std_dev": 0.01065993423883712, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.035353606837606834, "avg_duration_per_task_ms": 305384.14801864803, "avg_output_tokens_per_task": 15605.497668997668, "last_run_timestamp": "2026-11-10T17:02:13.232Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.8270801428571428, "accuracy_std_dev": 0.019019147618188608, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.09306387085137086, "avg_duration_per_task_ms": 1107119.2698412698, "avg_output_tokens_per_task": 45248.218614718615, "last_run_timestamp": "2026-10-31T16:08:08.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.8270583636363636, "accuracy_std_dev": 0.00941702237510474, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.04288225068870524, "avg_duration_per_task_ms": 229942.66437098256, "avg_output_tokens_per_task": 14178.259871441689, "last_run_timestamp": "2026-10-31T08:36:47.145Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.8269358, "accuracy_std_dev": 0.006767606146932625, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.04578402121212121, "avg_duration_per_task_ms": 60291.652525252524, "avg_output_tokens_per_task": 11285.754545454545, "last_run_timestamp": "2026-09-26T09:19:03.901Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.8265989999999999, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01892201515151515, "avg_duration_per_task_ms": 495658.10101010103, "avg_output_tokens_per_task": 18768.85606060606, "last_run_timestamp": "2026-08-29T18:35:57.732Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Nebius", "accuracy": 0.8265989999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.051792560606060606, "avg_duration_per_task_ms": 150332.1313131313, "avg_output_tokens_per_task": 17173.429292929293, "last_run_timestamp": "2026-11-04T18:02:01.782Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.826459, "accuracy_std_dev": 0.009410002267091476, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.14383780639730645, "avg_duration_per_task_ms": 198190.3564814815, "avg_output_tokens_per_task": 35703.72769360269, "last_run_timestamp": "2026-10-31T08:32:01.077Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.8264159143968872, "accuracy_std_dev": 0.0040531378816294926, "run_count": 4, "total_questions": 771, "avg_cost_per_task_usd": 0.13822659403372245, "avg_duration_per_task_ms": 272716.0363164721, "avg_output_tokens_per_task": 34382.090791180286, "last_run_timestamp": "2026-10-31T08:38:15.143Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.8264111325878595, "accuracy_std_dev": 0.01425677691801343, "run_count": 10, "total_questions": 1878, "avg_cost_per_task_usd": 0.02454355484558041, "avg_duration_per_task_ms": 264967.17305644305, "avg_output_tokens_per_task": 50263.019169329076, "last_run_timestamp": "2026-10-31T08:17:29.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.8263836795903439, "accuracy_std_dev": 0.029166866420929173, "run_count": 7, "total_questions": 1367, "avg_cost_per_task_usd": 0.07172207900512069, "avg_duration_per_task_ms": 248028.12801755668, "avg_output_tokens_per_task": 21803.532553035846, "last_run_timestamp": "2026-10-31T09:17:12.929Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "GMICloud", "accuracy": 0.8262311079545455, "accuracy_std_dev": 0.0007570000000000077, "run_count": 2, "total_questions": 352, "avg_cost_per_task_usd": 0.03419934943181818, "avg_duration_per_task_ms": 187950.64204545456, "avg_output_tokens_per_task": 10629.338068181818, "last_run_timestamp": "2026-11-10T06:16:01.718Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.8261184761715648, "accuracy_std_dev": 0.021573637693110777, "run_count": 13, "total_questions": 2518, "avg_cost_per_task_usd": 0.022734466243050037, "avg_duration_per_task_ms": 240016.77958697377, "avg_output_tokens_per_task": 45243.0552025417, "last_run_timestamp": "2026-11-12T09:43:53.858Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "SiliconFlow", "accuracy": 0.8260636363636363, "accuracy_std_dev": 0.058329593420450306, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.051863643709825526, "avg_duration_per_task_ms": 534077.3172635445, "avg_output_tokens_per_task": 24831.462809917357, "last_run_timestamp": "2026-10-31T16:01:53.625Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.8260381111111113, "accuracy_std_dev": 0.019244088918963866, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.012581548821548822, "avg_duration_per_task_ms": 794792.8148148148, "avg_output_tokens_per_task": 37224.931537598204, "last_run_timestamp": "2026-10-17T08:04:21.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.82575775, "accuracy_std_dev": 0.010065865980505628, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.14380135795454543, "avg_duration_per_task_ms": 418612.6521464646, "avg_output_tokens_per_task": 28784.420454545456, "last_run_timestamp": "2026-10-31T08:40:11.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.8257574999999999, "accuracy_std_dev": 0.015993499999999994, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2609003181818182, "avg_duration_per_task_ms": 231557.84848484848, "avg_output_tokens_per_task": 25997.717171717173, "last_run_timestamp": "2026-10-01T19:12:42.150Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.8254768888888889, "accuracy_std_dev": 0.05394208057505227, "run_count": 18, "total_questions": 3564, "avg_cost_per_task_usd": 0.007830105218855219, "avg_duration_per_task_ms": 541491.9023569024, "avg_output_tokens_per_task": 17689.749438832772, "last_run_timestamp": "2026-10-31T16:45:45.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.825389385046729, "accuracy_std_dev": 0.014184335849948446, "run_count": 11, "total_questions": 2140, "avg_cost_per_task_usd": 0.03383954813084113, "avg_duration_per_task_ms": 407090.0654205608, "avg_output_tokens_per_task": 47991.5476635514, "last_run_timestamp": "2026-10-31T08:48:01.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.824495, "accuracy_std_dev": 0.023148000000000002, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.11508162373737373, "avg_duration_per_task_ms": 562224.2297979798, "avg_output_tokens_per_task": 32139.14393939394, "last_run_timestamp": "2026-08-29T10:53:09.444Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Baidu", "accuracy": 0.8244826858605246, "accuracy_std_dev": 0.009259257906785777, "run_count": 8, "total_questions": 1563, "avg_cost_per_task_usd": 0.008255877799104288, "avg_duration_per_task_ms": 352541.7984644914, "avg_output_tokens_per_task": 19223.356365962893, "last_run_timestamp": "2026-11-10T18:18:26.718Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.8244347142857142, "accuracy_std_dev": 0.01399044581863215, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.1977735064935065, "avg_duration_per_task_ms": 958897.5014430014, "avg_output_tokens_per_task": 52647.930735930735, "last_run_timestamp": "2026-10-31T16:50:02.353Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.8243546666666667, "accuracy_std_dev": 0.03111308863734931, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.25956171212121215, "avg_duration_per_task_ms": 233681.0909090909, "avg_output_tokens_per_task": 25863.887205387204, "last_run_timestamp": "2026-10-01T19:17:25.275Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.8243546666666666, "accuracy_std_dev": 0.03801456116045827, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.029119777356902358, "avg_duration_per_task_ms": 574036.2205387205, "avg_output_tokens_per_task": 53199.25168350169, "last_run_timestamp": "2026-11-01T11:18:30.093Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.8242303292875989, "accuracy_std_dev": 0.016434035763013298, "run_count": 20, "total_questions": 3790, "avg_cost_per_task_usd": 0.030382969920844324, "avg_duration_per_task_ms": 266171.0530343008, "avg_output_tokens_per_task": 14053.306068601583, "last_run_timestamp": "2026-10-31T08:10:37.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.8241675555555555, "accuracy_std_dev": 0.024242370268744616, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.011208364197530863, "avg_duration_per_task_ms": 802349.7974186308, "avg_output_tokens_per_task": 39790.19079685747, "last_run_timestamp": "2026-10-17T08:02:36.531Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "DeepInfra", "accuracy": 0.8240802188112345, "accuracy_std_dev": 0.011135789059800642, "run_count": 8, "total_questions": 1531, "avg_cost_per_task_usd": 0.02478115610711953, "avg_duration_per_task_ms": 443159.53821032005, "avg_output_tokens_per_task": 24668.269105160027, "last_run_timestamp": "2026-10-31T16:42:08.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Featherless", "accuracy": 0.822917, "accuracy_std_dev": null, "run_count": 1, "total_questions": 176, "avg_cost_per_task_usd": 0.036827073863636366, "avg_duration_per_task_ms": 130682.76704545454, "avg_output_tokens_per_task": 8202.113636363636, "last_run_timestamp": "2026-11-14T00:00:42.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DigitalOcean", "accuracy": 0.8225597382029736, "accuracy_std_dev": 0.013482485406421735, "run_count": 8, "total_questions": 1547, "avg_cost_per_task_usd": 0.018713002585649646, "avg_duration_per_task_ms": 601976.5061409179, "avg_output_tokens_per_task": 20229.648351648353, "last_run_timestamp": "2026-10-31T16:13:07.320Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.8225107142857142, "accuracy_std_dev": 0.013731671521541366, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007629560606060605, "avg_duration_per_task_ms": 439138.09451659454, "avg_output_tokens_per_task": 18615.266233766233, "last_run_timestamp": "2026-10-31T16:13:09.863Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.0244105, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.2415952904040404, "avg_duration_per_task_ms": 181953.0101010101, "avg_output_tokens_per_task": 24066.49494949495, "last_run_timestamp": "2026-10-01T19:12:31.045Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.8223905000000001, "accuracy_std_dev": 0.019360500000000003, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.023091813131313133, "avg_duration_per_task_ms": 454744.9166666667, "avg_output_tokens_per_task": 21871.75, "last_run_timestamp": "2026-08-29T18:09:03.379Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.8223905, "accuracy_std_dev": 0.019262412208322532, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.03192426473063974, "avg_duration_per_task_ms": 260830.34259259258, "avg_output_tokens_per_task": 14583.287878787878, "last_run_timestamp": "2026-10-31T08:17:29.187Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.8220747500000002, "accuracy_std_dev": 0.06699828629105002, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.029949901515151516, "avg_duration_per_task_ms": 747877.6811868687, "avg_output_tokens_per_task": 30535.987373737375, "last_run_timestamp": "2026-10-17T08:18:04.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.8220297142857144, "accuracy_std_dev": 0.020075599465977787, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006425138528138528, "avg_duration_per_task_ms": 406726.4509379509, "avg_output_tokens_per_task": 18776.16810966811, "last_run_timestamp": "2026-10-31T16:12:28.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.8216246926454446, "accuracy_std_dev": 0.030556920617104074, "run_count": 5, "total_questions": 911, "avg_cost_per_task_usd": 0.02863867069154775, "avg_duration_per_task_ms": 543580.5345773875, "avg_output_tokens_per_task": 16890.06256860593, "last_run_timestamp": "2026-10-17T16:03:44.233Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cohere", "accuracy": 0.8215486666666668, "accuracy_std_dev": 0.015611992961253275, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.02833054377104377, "avg_duration_per_task_ms": 228888.5606060606, "avg_output_tokens_per_task": 56421.28451178451, "last_run_timestamp": "2026-10-31T08:03:09.999Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.821448952348303, "accuracy_std_dev": 0.023090870602325352, "run_count": 15, "total_questions": 2917, "avg_cost_per_task_usd": 0.02527841960918752, "avg_duration_per_task_ms": 834456.179293795, "avg_output_tokens_per_task": 51002.6424408639, "last_run_timestamp": "2026-11-12T02:16:49.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.8213506405228759, "accuracy_std_dev": 0.02248216924587128, "run_count": 3, "total_questions": 459, "avg_cost_per_task_usd": 0.08758356862745097, "avg_duration_per_task_ms": 304618.9760348584, "avg_output_tokens_per_task": 34720.03050108933, "last_run_timestamp": "2026-08-29T10:57:24.797Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.8213131235765501, "accuracy_std_dev": 0.034016985407619854, "run_count": 12, "total_questions": 2371, "avg_cost_per_task_usd": 0.08167362969211302, "avg_duration_per_task_ms": 336290.14972585405, "avg_output_tokens_per_task": 20283.436524673132, "last_run_timestamp": "2026-10-31T16:02:14.359Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.8212070710659898, "accuracy_std_dev": 0.011946281848340899, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.01807649915397631, "avg_duration_per_task_ms": 302940.0592216582, "avg_output_tokens_per_task": 16413.382402707277, "last_run_timestamp": "2026-08-29T09:59:35.603Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.8210525603238865, "accuracy_std_dev": 0.07599126782917703, "run_count": 7, "total_questions": 1235, "avg_cost_per_task_usd": 0.007044106882591094, "avg_duration_per_task_ms": 525854.9595141701, "avg_output_tokens_per_task": 14201.662348178137, "last_run_timestamp": "2026-10-03T10:22:02.091Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.820736340116279, "accuracy_std_dev": 0.02413812698210204, "run_count": 7, "total_questions": 1376, "avg_cost_per_task_usd": 0.01966482122093023, "avg_duration_per_task_ms": 283517.4375, "avg_output_tokens_per_task": 17874.627180232557, "last_run_timestamp": "2026-10-03T09:25:02.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.8207070000000001, "accuracy_std_dev": 0.024411000000000016, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04063828030303031, "avg_duration_per_task_ms": 83296.42424242424, "avg_output_tokens_per_task": 26969.131313131315, "last_run_timestamp": "2026-10-01T19:23:01.327Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.8207070000000001, "accuracy_std_dev": 0.007575999999999972, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.07039754797979798, "avg_duration_per_task_ms": 879593.0631313132, "avg_output_tokens_per_task": 40130.17171717172, "last_run_timestamp": "2026-08-30T00:37:59.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.8203833076923077, "accuracy_std_dev": 0.018076297388844354, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.042996285159285164, "avg_duration_per_task_ms": 241748.8038073038, "avg_output_tokens_per_task": 14318.661227661227, "last_run_timestamp": "2026-10-31T08:09:30.241Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.8202743410372041, "accuracy_std_dev": 0.011484132116496031, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.09063781341600902, "avg_duration_per_task_ms": 168767.6234498309, "avg_output_tokens_per_task": 22236.227170236754, "last_run_timestamp": "2026-09-19T10:14:07.966Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.8200275393258427, "accuracy_std_dev": 0.011794769394173345, "run_count": 9, "total_questions": 1691, "avg_cost_per_task_usd": 0.03818077646363098, "avg_duration_per_task_ms": 215799.96629213484, "avg_output_tokens_per_task": 14692.68894145476, "last_run_timestamp": "2026-10-31T08:55:11.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.819865, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.08986796717171718, "avg_duration_per_task_ms": 195255.59343434343, "avg_output_tokens_per_task": 25486.853535353537, "last_run_timestamp": "2026-08-29T18:58:14.477Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.819865, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.13854771717171718, "avg_duration_per_task_ms": 540824.1212121212, "avg_output_tokens_per_task": 31240.651515151516, "last_run_timestamp": "2026-09-07T06:00:57.184Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.8194060909090909, "accuracy_std_dev": 0.013884218840464785, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.033594433884297524, "avg_duration_per_task_ms": 211994.46005509642, "avg_output_tokens_per_task": 13897.44214876033, "last_run_timestamp": "2026-10-31T08:09:30.951Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.8193976666666667, "accuracy_std_dev": 0.013990614068010025, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.017960378226711565, "avg_duration_per_task_ms": 372150.59315375984, "avg_output_tokens_per_task": 13044.02581369248, "last_run_timestamp": "2026-11-08T03:44:37.321Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.8193842857142857, "accuracy_std_dev": 0.05010024752352396, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.20815405194805192, "avg_duration_per_task_ms": 879947.1565656565, "avg_output_tokens_per_task": 51956.655122655124, "last_run_timestamp": "2026-10-31T16:08:08.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.8192149207920791, "accuracy_std_dev": 0.018823182524040232, "run_count": 6, "total_questions": 909, "avg_cost_per_task_usd": 0.11224348184818483, "avg_duration_per_task_ms": 401079.6094609461, "avg_output_tokens_per_task": 46532.80198019802, "last_run_timestamp": "2026-10-17T16:06:46.637Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.8189473014736841, "accuracy_std_dev": 0.010171864904671552, "run_count": 12, "total_questions": 2375, "avg_cost_per_task_usd": 0.03239228378947368, "avg_duration_per_task_ms": 542473.8867368421, "avg_output_tokens_per_task": 15887.374736842105, "last_run_timestamp": "2026-11-13T01:53:10.915Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.8186628571428571, "accuracy_std_dev": 0.011047136143797251, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.024918836219336223, "avg_duration_per_task_ms": 190477.32106782106, "avg_output_tokens_per_task": 16519.824675324675, "last_run_timestamp": "2026-10-03T10:10:35.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.818182, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.10403051010101011, "avg_duration_per_task_ms": 527188.4621212122, "avg_output_tokens_per_task": 38544.431818181816, "last_run_timestamp": "2026-08-29T18:26:14.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "KT", "accuracy": 0.818182, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.023744409090909087, "avg_duration_per_task_ms": 279872.4292929293, "avg_output_tokens_per_task": 47254.60101010101, "last_run_timestamp": "2026-11-11T02:01:08.052Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.8179302505415164, "accuracy_std_dev": 0.004936247761122178, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.02662889386281589, "avg_duration_per_task_ms": 314143.65198555955, "avg_output_tokens_per_task": 26506.814440433212, "last_run_timestamp": "2026-10-31T16:04:48.178Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Tenstorrent", "accuracy": 0.817884202247191, "accuracy_std_dev": 0.008045999999999998, "run_count": 2, "total_questions": 356, "avg_cost_per_task_usd": 0.006599825842696629, "avg_duration_per_task_ms": 342879.2528089888, "avg_output_tokens_per_task": 26299.71629213483, "last_run_timestamp": "2026-11-08T04:17:28.377Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.8178727706422019, "accuracy_std_dev": 0.03174962566960436, "run_count": 5, "total_questions": 981, "avg_cost_per_task_usd": 0.023857419979612642, "avg_duration_per_task_ms": 166236.06523955148, "avg_output_tokens_per_task": 14523.781855249745, "last_run_timestamp": "2026-10-31T08:07:51.399Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "StreamLake", "accuracy": 0.8177376144306652, "accuracy_std_dev": 0.03151916867911054, "run_count": 9, "total_questions": 1774, "avg_cost_per_task_usd": 0.013423886696730551, "avg_duration_per_task_ms": 855848.4718151072, "avg_output_tokens_per_task": 39679.556933483655, "last_run_timestamp": "2026-10-17T08:19:03.202Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.8172424786585366, "accuracy_std_dev": 0.032381379178472315, "run_count": 8, "total_questions": 984, "avg_cost_per_task_usd": 0.0567359319105691, "avg_duration_per_task_ms": 328244.918699187, "avg_output_tokens_per_task": 21710.94918699187, "last_run_timestamp": "2026-10-31T16:26:17.224Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.8172197142857143, "accuracy_std_dev": 0.01956805149591904, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.009476494949494948, "avg_duration_per_task_ms": 577700.4018759019, "avg_output_tokens_per_task": 19548.448051948053, "last_run_timestamp": "2026-10-31T16:08:08.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.8171817336142323, "accuracy_std_dev": 0.04982830145510509, "run_count": 11, "total_questions": 2136, "avg_cost_per_task_usd": 0.018343408239700373, "avg_duration_per_task_ms": 793559.7439138577, "avg_output_tokens_per_task": 48776.84925093633, "last_run_timestamp": "2026-10-31T08:31:53.887Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.8164981428571428, "accuracy_std_dev": 0.013316748910714663, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.026554869408369406, "avg_duration_per_task_ms": 277649.69264069264, "avg_output_tokens_per_task": 25393.416305916307, "last_run_timestamp": "2026-10-31T16:05:36.303Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.8161921818181818, "accuracy_std_dev": 0.005314989759986385, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.34637480073461896, "avg_duration_per_task_ms": 358931.4338842975, "avg_output_tokens_per_task": 22900.560146923784, "last_run_timestamp": "2026-10-31T08:17:29.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.8158860909090909, "accuracy_std_dev": 0.011310770084341126, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.31360446280991733, "avg_duration_per_task_ms": 289460.9269972452, "avg_output_tokens_per_task": 20716.85261707989, "last_run_timestamp": "2026-10-31T08:15:24.404Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.8157767142857143, "accuracy_std_dev": 0.008190962872306891, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02609219480519481, "avg_duration_per_task_ms": 163166.5974025974, "avg_output_tokens_per_task": 23394.820346320346, "last_run_timestamp": "2026-10-31T16:09:44.821Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.8156564999999999, "accuracy_std_dev": 0.015993499999999994, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018819113636363637, "avg_duration_per_task_ms": 509395.56313131313, "avg_output_tokens_per_task": 18621.05808080808, "last_run_timestamp": "2026-08-30T00:16:38.394Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.8154460000000001, "accuracy_std_dev": 0.020060118637485684, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.12119656250000001, "avg_duration_per_task_ms": 800383.8623737374, "avg_output_tokens_per_task": 63007.16098484849, "last_run_timestamp": "2026-10-31T16:10:09.226Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.8153198, "accuracy_std_dev": 0.009214558967199644, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.030977021212121213, "avg_duration_per_task_ms": 329320.2121212121, "avg_output_tokens_per_task": 20456.146464646463, "last_run_timestamp": "2026-10-17T16:09:25.569Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.8152402988505746, "accuracy_std_dev": 0.029604949040447602, "run_count": 4, "total_questions": 783, "avg_cost_per_task_usd": 0.28207722477650066, "avg_duration_per_task_ms": 913648.006385696, "avg_output_tokens_per_task": 58474.00127713921, "last_run_timestamp": "2026-10-31T16:12:07.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.8149678181818182, "accuracy_std_dev": 0.009372109986726867, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.3310624931129476, "avg_duration_per_task_ms": 290282.1565656566, "avg_output_tokens_per_task": 20829.40679522498, "last_run_timestamp": "2026-10-31T08:27:57.306Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.8148578328496808, "accuracy_std_dev": 0.00794054106748679, "run_count": 9, "total_questions": 1723, "avg_cost_per_task_usd": 0.04277770748694139, "avg_duration_per_task_ms": 184421.18630295995, "avg_output_tokens_per_task": 16933.951247823563, "last_run_timestamp": "2026-11-08T04:02:09.197Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.8143555909090909, "accuracy_std_dev": 0.010013659936938356, "run_count": 22, "total_questions": 4356, "avg_cost_per_task_usd": 0.36341748691460063, "avg_duration_per_task_ms": 357684.4742883379, "avg_output_tokens_per_task": 22888.384756657484, "last_run_timestamp": "2026-10-31T08:33:07.626Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.8140932857142855, "accuracy_std_dev": 0.003468669103119897, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.012047585137085136, "avg_duration_per_task_ms": 883223.4725829726, "avg_output_tokens_per_task": 18603.608946608947, "last_run_timestamp": "2026-10-31T16:13:09.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.813973, "accuracy_std_dev": 0.0008419999999999539, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006049762626262625, "avg_duration_per_task_ms": 483167.9494949495, "avg_output_tokens_per_task": 18365.43181818182, "last_run_timestamp": "2026-08-29T19:09:30.030Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.813973, "accuracy_std_dev": 0.01262599999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04186610353535354, "avg_duration_per_task_ms": 76135.87878787878, "avg_output_tokens_per_task": 27786.689393939392, "last_run_timestamp": "2026-10-01T19:16:58.595Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.8133718571428571, "accuracy_std_dev": 0.016208212586997554, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.04426409668109668, "avg_duration_per_task_ms": 84524.13997113997, "avg_output_tokens_per_task": 27404.910533910534, "last_run_timestamp": "2026-10-01T19:25:58.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.813051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 189, "avg_cost_per_task_usd": 0.00766168253968254, "avg_duration_per_task_ms": 356938.3703703704, "avg_output_tokens_per_task": 33904.37566137566, "last_run_timestamp": "2026-10-14T18:00:03.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.8128251818181819, "accuracy_std_dev": 0.015105822103463543, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.3406421253443526, "avg_duration_per_task_ms": 353395.67309458216, "avg_output_tokens_per_task": 22530.53305785124, "last_run_timestamp": "2026-10-31T08:46:10.648Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.8120124868421054, "accuracy_std_dev": 0.08527436462791073, "run_count": 7, "total_questions": 1368, "avg_cost_per_task_usd": 0.0753633048245614, "avg_duration_per_task_ms": 666389.4437134503, "avg_output_tokens_per_task": 27234.816520467837, "last_run_timestamp": "2026-10-10T16:29:40.552Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.8117845999999999, "accuracy_std_dev": 0.01594972366657178, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.026062873737373735, "avg_duration_per_task_ms": 323333.02424242423, "avg_output_tokens_per_task": 17452.424242424244, "last_run_timestamp": "2026-09-19T09:05:52.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "River", "accuracy": 0.811448, "accuracy_std_dev": 0.01683499999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.020069919191919194, "avg_duration_per_task_ms": 287040.1212121212, "avg_output_tokens_per_task": 49971.14141414141, "last_run_timestamp": "2026-11-12T20:40:55.525Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.8108866666666666, "accuracy_std_dev": 0.07631444718764303, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.09796533164983165, "avg_duration_per_task_ms": 791756.9158249159, "avg_output_tokens_per_task": 39080.49158249158, "last_run_timestamp": "2026-10-03T17:58:22.995Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Mistral", "accuracy": 0.8107880872313526, "accuracy_std_dev": 0.020350576011120062, "run_count": 8, "total_questions": 1582, "avg_cost_per_task_usd": 0.025940944374209858, "avg_duration_per_task_ms": 221408.600505689, "avg_output_tokens_per_task": 51645.53539823009, "last_run_timestamp": "2026-11-11T03:11:48.726Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.8107262857142856, "accuracy_std_dev": 0.02121111496843299, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.29609035353535357, "avg_duration_per_task_ms": 1026681.7662337662, "avg_output_tokens_per_task": 67260.45959595959, "last_run_timestamp": "2026-10-31T16:12:28.234Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.810606, "accuracy_std_dev": 0.010546997511140292, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.044063582702020196, "avg_duration_per_task_ms": 62743.707702020205, "avg_output_tokens_per_task": 15119.022095959595, "last_run_timestamp": "2026-10-31T08:43:59.854Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.8099128055555557, "accuracy_std_dev": 0.02251018815558855, "run_count": 5, "total_questions": 612, "avg_cost_per_task_usd": 0.06893759803921569, "avg_duration_per_task_ms": 1733704.9052287582, "avg_output_tokens_per_task": 39280.651960784315, "last_run_timestamp": "2026-10-31T16:05:14.633Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.8098844285714284, "accuracy_std_dev": 0.01753250978575957, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.23733018614718615, "avg_duration_per_task_ms": 1080648.3585858585, "avg_output_tokens_per_task": 63860.35858585859, "last_run_timestamp": "2026-10-31T16:12:28.230Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.8097645, "accuracy_std_dev": 0.021885500000000002, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.044968992424242425, "avg_duration_per_task_ms": 208495.61616161617, "avg_output_tokens_per_task": 22388.290404040403, "last_run_timestamp": "2026-10-01T19:19:28.473Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Chutes", "accuracy": 0.8097639999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.06257432828282827, "avg_duration_per_task_ms": 366659.297979798, "avg_output_tokens_per_task": 47164.93434343435, "last_run_timestamp": "2026-11-01T18:00:07.322Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "PrimeIntellect", "accuracy": 0.8096608426103648, "accuracy_std_dev": 0.012817931849127972, "run_count": 3, "total_questions": 521, "avg_cost_per_task_usd": 0.0079749443378119, "avg_duration_per_task_ms": 289711.4932821497, "avg_output_tokens_per_task": 31380.54894433781, "last_run_timestamp": "2026-10-31T16:04:23.357Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.8081742222222223, "accuracy_std_dev": 0.08442583700078994, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.11942196632996631, "avg_duration_per_task_ms": 649773.9371492704, "avg_output_tokens_per_task": 32692.530303030304, "last_run_timestamp": "2026-10-31T17:00:44.290Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "upstage/solar-pro4-20260810", "model_slug": "upstage/solar-pro4", "model_name": "Upstage: Solar Pro 4", "provider_name": "KT", "accuracy": 0.808081, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.05294089393939394, "avg_duration_per_task_ms": 409688.16161616164, "avg_output_tokens_per_task": 43927.25252525252, "last_run_timestamp": "2026-11-11T02:02:05.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.8078402857142857, "accuracy_std_dev": 0.013630118652814278, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.02220728354978355, "avg_duration_per_task_ms": 246897.44444444444, "avg_output_tokens_per_task": 17520.37012987013, "last_run_timestamp": "2026-10-03T09:13:36.277Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.807532574733096, "accuracy_std_dev": 0.023704551349195916, "run_count": 6, "total_questions": 1124, "avg_cost_per_task_usd": 0.039211227758007124, "avg_duration_per_task_ms": 252786.0587188612, "avg_output_tokens_per_task": 39158.109430604985, "last_run_timestamp": "2026-10-31T16:09:44.822Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Particle AI", "accuracy": 0.80723875, "accuracy_std_dev": 0.007668707465244688, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.02855746717171717, "avg_duration_per_task_ms": 256654.82196969696, "avg_output_tokens_per_task": 56910.10353535353, "last_run_timestamp": "2026-11-03T01:57:50.570Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.8068179999999999, "accuracy_std_dev": 0.027363478050131, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.046258910353535354, "avg_duration_per_task_ms": 196125.59974747474, "avg_output_tokens_per_task": 21946.10606060606, "last_run_timestamp": "2026-10-01T19:34:33.810Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.8063975, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.17857601262626266, "avg_duration_per_task_ms": 685061.2348484849, "avg_output_tokens_per_task": 56591.982323232325, "last_run_timestamp": "2026-08-29T19:16:33.097Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.8063972857142858, "accuracy_std_dev": 0.03755804913811719, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.18206293145743146, "avg_duration_per_task_ms": 561423.8109668109, "avg_output_tokens_per_task": 56651.43217893218, "last_run_timestamp": "2026-10-31T16:15:54.668Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.806397, "accuracy_std_dev": 0.003367000000000009, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.04417995707070707, "avg_duration_per_task_ms": 210130.46464646465, "avg_output_tokens_per_task": 21993.772727272728, "last_run_timestamp": "2026-10-01T19:26:25.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Parasail", "accuracy": 0.8047882407261246, "accuracy_std_dev": 0.03134006389099038, "run_count": 9, "total_questions": 1267, "avg_cost_per_task_usd": 0.007205067876874505, "avg_duration_per_task_ms": 327813.7805840568, "avg_output_tokens_per_task": 28563.743488555643, "last_run_timestamp": "2026-10-31T16:04:29.125Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8038719166666666, "accuracy_std_dev": 0.01983021612530708, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.3387011611952862, "avg_duration_per_task_ms": 355211.85016835015, "avg_output_tokens_per_task": 22440.643097643097, "last_run_timestamp": "2026-10-31T08:53:23.022Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.80303, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.05439790909090909, "avg_duration_per_task_ms": 437250.6313131313, "avg_output_tokens_per_task": 23570.0202020202, "last_run_timestamp": "2026-08-22T17:18:01.115Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.8028182609489051, "accuracy_std_dev": 0.07654548992882891, "run_count": 9, "total_questions": 1644, "avg_cost_per_task_usd": 0.019102108272506083, "avg_duration_per_task_ms": 309069.2311435523, "avg_output_tokens_per_task": 40420.16605839416, "last_run_timestamp": "2026-10-31T08:33:53.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.8016327222511724, "accuracy_std_dev": 0.1200400557122913, "run_count": 10, "total_questions": 1919, "avg_cost_per_task_usd": 0.10716173944762898, "avg_duration_per_task_ms": 645521.1886399166, "avg_output_tokens_per_task": 30270.389265242313, "last_run_timestamp": "2026-10-31T16:40:50.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.8007854444444445, "accuracy_std_dev": 0.13709711112858092, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.009282588103254769, "avg_duration_per_task_ms": 461356.3843995511, "avg_output_tokens_per_task": 29453.966891133558, "last_run_timestamp": "2026-11-10T07:09:23.085Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.8001147113402061, "accuracy_std_dev": 0.1242203500272354, "run_count": 3, "total_questions": 582, "avg_cost_per_task_usd": 0.11411094845360827, "avg_duration_per_task_ms": 1146503.6958762887, "avg_output_tokens_per_task": 38030.19243986254, "last_run_timestamp": "2026-10-24T16:13:05.627Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.7996630000000001, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01377076515151515, "avg_duration_per_task_ms": 290656.38383838383, "avg_output_tokens_per_task": 14276.108585858587, "last_run_timestamp": "2026-08-29T19:12:57.906Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7980561818181818, "accuracy_std_dev": 0.04308950805605, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.0430184435261708, "avg_duration_per_task_ms": 516278.7502295684, "avg_output_tokens_per_task": 42641.152892561986, "last_run_timestamp": "2026-10-31T16:13:09.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "AtlasCloud", "accuracy": 0.7979796666666666, "accuracy_std_dev": 0.004956316261992276, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.014001683501683501, "avg_duration_per_task_ms": 340373.8703703704, "avg_output_tokens_per_task": 14266.20707070707, "last_run_timestamp": "2026-10-04T00:00:36.942Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.79586, "accuracy_std_dev": 0.006604009430641372, "run_count": 5, "total_questions": 934, "avg_cost_per_task_usd": 0.05025589079229122, "avg_duration_per_task_ms": 1087344.4850107066, "avg_output_tokens_per_task": 22883.693790149893, "last_run_timestamp": "2026-10-17T16:06:10.469Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7955151838886546, "accuracy_std_dev": 0.021167952217358864, "run_count": 12, "total_questions": 2371, "avg_cost_per_task_usd": 0.0645802066638549, "avg_duration_per_task_ms": 194462.45592576973, "avg_output_tokens_per_task": 20212.99072121468, "last_run_timestamp": "2026-10-31T08:22:42.983Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "StreamLake", "accuracy": 0.7954545, "accuracy_std_dev": 0.010942500000000022, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.014605156565656566, "avg_duration_per_task_ms": 301530.7196969697, "avg_output_tokens_per_task": 13980.005050505051, "last_run_timestamp": "2026-08-29T16:10:33.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.7950563915254236, "accuracy_std_dev": 0.05895769598336308, "run_count": 6, "total_questions": 1180, "avg_cost_per_task_usd": 0.007454267796610169, "avg_duration_per_task_ms": 818243.9161016949, "avg_output_tokens_per_task": 18528.625423728812, "last_run_timestamp": "2026-10-24T16:21:54.634Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7950336666666666, "accuracy_std_dev": 0.03307482054205114, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.03994933501683502, "avg_duration_per_task_ms": 251252.56818181818, "avg_output_tokens_per_task": 12806.098484848484, "last_run_timestamp": "2026-09-19T09:05:41.015Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7943721428571427, "accuracy_std_dev": 0.008070114451809616, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.017872657287157288, "avg_duration_per_task_ms": 60674.55339105339, "avg_output_tokens_per_task": 5803.906204906205, "last_run_timestamp": "2026-10-31T16:10:36.258Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.7942385555555556, "accuracy_std_dev": 0.012314380393228746, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.004576078563411897, "avg_duration_per_task_ms": 256060.55387205386, "avg_output_tokens_per_task": 29957.45061728395, "last_run_timestamp": "2026-10-17T16:03:34.733Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7934233709949409, "accuracy_std_dev": 0.05668542998401107, "run_count": 6, "total_questions": 1186, "avg_cost_per_task_usd": 0.01885696121416526, "avg_duration_per_task_ms": 1103043.2150084318, "avg_output_tokens_per_task": 18169.33305227656, "last_run_timestamp": "2026-10-31T16:45:39.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "StreamLake", "accuracy": 0.7932098333333334, "accuracy_std_dev": 0.006693092768336313, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.05432222053872054, "avg_duration_per_task_ms": 460720.4478114478, "avg_output_tokens_per_task": 23581.273569023568, "last_run_timestamp": "2026-10-17T16:03:34.700Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7923028091603053, "accuracy_std_dev": 0.012838036519947284, "run_count": 8, "total_questions": 1572, "avg_cost_per_task_usd": 0.01830340776081425, "avg_duration_per_task_ms": 189913.7779898219, "avg_output_tokens_per_task": 15102.418575063613, "last_run_timestamp": "2026-10-08T22:29:11.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.7920875, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02811962121212121, "avg_duration_per_task_ms": 464497.5707070707, "avg_output_tokens_per_task": 23480.530303030304, "last_run_timestamp": "2026-08-29T18:09:57.600Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.7913752307692307, "accuracy_std_dev": 0.03764112974915735, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.10873279254079253, "avg_duration_per_task_ms": 370663.50815850816, "avg_output_tokens_per_task": 27165.075757575756, "last_run_timestamp": "2026-11-10T06:47:58.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.791077650901999, "accuracy_std_dev": 0.011802764782558223, "run_count": 11, "total_questions": 2051, "avg_cost_per_task_usd": 0.00883669673330083, "avg_duration_per_task_ms": 380017.00048756704, "avg_output_tokens_per_task": 34825.446123842026, "last_run_timestamp": "2026-10-31T16:15:26.662Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7909730112634672, "accuracy_std_dev": 0.015262253874258334, "run_count": 11, "total_questions": 2042, "avg_cost_per_task_usd": 0.004672809010773752, "avg_duration_per_task_ms": 1542673.8849167484, "avg_output_tokens_per_task": 30652.18511263467, "last_run_timestamp": "2026-10-31T16:08:08.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7905242857142856, "accuracy_std_dev": 0.03382434521682478, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.16089077777777777, "avg_duration_per_task_ms": 885338.5844155845, "avg_output_tokens_per_task": 66850.18398268399, "last_run_timestamp": "2026-10-31T16:04:51.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.7888407142857142, "accuracy_std_dev": 0.009727651246601654, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0210191240981241, "avg_duration_per_task_ms": 240196.7041847042, "avg_output_tokens_per_task": 12493.777056277057, "last_run_timestamp": "2026-10-31T16:10:36.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.7887204999999999, "accuracy_std_dev": 0.022727499999999956, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.02925468181818182, "avg_duration_per_task_ms": 425922.92171717173, "avg_output_tokens_per_task": 25932.09595959596, "last_run_timestamp": "2026-08-29T18:05:36.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.7886696422660725, "accuracy_std_dev": 0.030494803180214167, "run_count": 8, "total_questions": 1571, "avg_cost_per_task_usd": 0.011359753660089117, "avg_duration_per_task_ms": 640851.9363462763, "avg_output_tokens_per_task": 37620.8707829408, "last_run_timestamp": "2026-10-31T16:03:21.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.7884532538560409, "accuracy_std_dev": 0.014149184070446418, "run_count": 8, "total_questions": 1556, "avg_cost_per_task_usd": 0.011693410025706941, "avg_duration_per_task_ms": 858091.7879177377, "avg_output_tokens_per_task": 38791.87210796915, "last_run_timestamp": "2026-11-11T08:00:18.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.7883595714285714, "accuracy_std_dev": 0.03112971003424745, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.03924310822510823, "avg_duration_per_task_ms": 198693.47186147186, "avg_output_tokens_per_task": 21702.200577200576, "last_run_timestamp": "2026-10-31T16:08:31.063Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Novita", "accuracy": 0.7878784999999998, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006171542929292929, "avg_duration_per_task_ms": 348997.62626262626, "avg_output_tokens_per_task": 14953.704545454546, "last_run_timestamp": "2026-08-29T19:20:50.265Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "GMICloud", "accuracy": 0.7878303017902814, "accuracy_std_dev": 0.15859528241072463, "run_count": 8, "total_questions": 1564, "avg_cost_per_task_usd": 0.01594299360613811, "avg_duration_per_task_ms": 555279.5345268542, "avg_output_tokens_per_task": 26007.55051150895, "last_run_timestamp": "2026-10-17T08:10:43.825Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "PrimeIntellect", "accuracy": 0.7872267663934426, "accuracy_std_dev": 0.013963506030602306, "run_count": 3, "total_questions": 488, "avg_cost_per_task_usd": 0.008632290983606556, "avg_duration_per_task_ms": 586832.4200819673, "avg_output_tokens_per_task": 37091.375, "last_run_timestamp": "2026-10-31T16:20:33.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7871492666666666, "accuracy_std_dev": 0.11453375331110427, "run_count": 15, "total_questions": 2970, "avg_cost_per_task_usd": 0.01689483905723906, "avg_duration_per_task_ms": 609264.4754208755, "avg_output_tokens_per_task": 30856.628282828282, "last_run_timestamp": "2026-11-06T22:10:48.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.786195, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.2643429696969697, "avg_duration_per_task_ms": 180778.88888888888, "avg_output_tokens_per_task": 17428.555555555555, "last_run_timestamp": "2026-09-01T13:19:59.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.786195, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.057962348484848485, "avg_duration_per_task_ms": 638736.3484848485, "avg_output_tokens_per_task": 23055.60101010101, "last_run_timestamp": "2026-08-22T17:09:41.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "DeepInfra", "accuracy": 0.7852381866666666, "accuracy_std_dev": 0.022517162489606328, "run_count": 6, "total_questions": 1050, "avg_cost_per_task_usd": 0.039451490476190475, "avg_duration_per_task_ms": 759087.4819047619, "avg_output_tokens_per_task": 18229.481904761906, "last_run_timestamp": "2026-10-17T16:06:15.029Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Nebius", "accuracy": 0.78493275, "accuracy_std_dev": 0.010934561842501992, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.03407076388888889, "avg_duration_per_task_ms": 483005.77398989897, "avg_output_tokens_per_task": 67906.58964646465, "last_run_timestamp": "2026-10-31T08:31:13.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.782641111111111, "accuracy_std_dev": 0.020988606207731148, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.020732816498316497, "avg_duration_per_task_ms": 463791.91694725025, "avg_output_tokens_per_task": 27564.918630751963, "last_run_timestamp": "2026-11-13T02:10:04.930Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.7824160174311926, "accuracy_std_dev": 0.017840425618521553, "run_count": 8, "total_questions": 1090, "avg_cost_per_task_usd": 0.04672766880733945, "avg_duration_per_task_ms": 971010.7899082568, "avg_output_tokens_per_task": 23260.208256880735, "last_run_timestamp": "2026-10-31T16:04:29.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.7823447220338983, "accuracy_std_dev": 0.020068261404654758, "run_count": 6, "total_questions": 1180, "avg_cost_per_task_usd": 0.022374091525423728, "avg_duration_per_task_ms": 880666.9830508474, "avg_output_tokens_per_task": 37830.4686440678, "last_run_timestamp": "2026-10-24T16:37:21.981Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.7818664285714286, "accuracy_std_dev": 0.016143829133821997, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.3387973888888889, "avg_duration_per_task_ms": 1030179.549062049, "avg_output_tokens_per_task": 85385.09307359307, "last_run_timestamp": "2026-10-31T16:04:02.366Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "DeepInfra", "accuracy": 0.781081, "accuracy_std_dev": null, "run_count": 1, "total_questions": 185, "avg_cost_per_task_usd": 0.04895784324324324, "avg_duration_per_task_ms": 686648.6648648649, "avg_output_tokens_per_task": 21222.45945945946, "last_run_timestamp": "2026-08-22T16:06:14.894Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "PrimeIntellect", "accuracy": 0.77933, "accuracy_std_dev": null, "run_count": 1, "total_questions": 179, "avg_cost_per_task_usd": 0.00797877653631285, "avg_duration_per_task_ms": 499606.7541899441, "avg_output_tokens_per_task": 38804.5530726257, "last_run_timestamp": "2026-11-01T18:00:07.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.7792742222222223, "accuracy_std_dev": 0.04102605862613494, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.2320418187429854, "avg_duration_per_task_ms": 897509.8860830527, "avg_output_tokens_per_task": 77846.84624017957, "last_run_timestamp": "2026-10-17T16:06:46.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7787130000000001, "accuracy_std_dev": 0.07769734491388151, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.1402097396184063, "avg_duration_per_task_ms": 939428.9349046015, "avg_output_tokens_per_task": 73044.72109988777, "last_run_timestamp": "2026-11-10T07:06:06.465Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.776936, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013052893939393939, "avg_duration_per_task_ms": 64108.87373737374, "avg_output_tokens_per_task": 10319.823232323231, "last_run_timestamp": "2026-10-01T19:12:45.924Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.7760940000000002, "accuracy_std_dev": 0.012598160421267844, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.013424281144781143, "avg_duration_per_task_ms": 64995.4595959596, "avg_output_tokens_per_task": 10615.195286195287, "last_run_timestamp": "2026-10-02T23:20:38.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7752525, "accuracy_std_dev": 0.02576620138055538, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.011368148148148148, "avg_duration_per_task_ms": 910071.1422558923, "avg_output_tokens_per_task": 43361.55303030303, "last_run_timestamp": "2026-10-31T09:00:40.586Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7749944977228367, "accuracy_std_dev": 0.02795561944851158, "run_count": 8, "total_questions": 1537, "avg_cost_per_task_usd": 0.007373610930383865, "avg_duration_per_task_ms": 852404.3311646064, "avg_output_tokens_per_task": 18993.68770331815, "last_run_timestamp": "2026-10-31T16:04:29.109Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.7748314999999999, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004441212121212121, "avg_duration_per_task_ms": 4186.169191919192, "avg_output_tokens_per_task": 2455.989898989899, "last_run_timestamp": "2026-08-29T18:58:30.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.7746466307789741, "accuracy_std_dev": 0.006675073159711417, "run_count": 8, "total_questions": 1579, "avg_cost_per_task_usd": 0.011284899936668778, "avg_duration_per_task_ms": 924382.4306523115, "avg_output_tokens_per_task": 11730.89170360988, "last_run_timestamp": "2026-10-31T16:02:14.358Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.7744107142857142, "accuracy_std_dev": 0.007947545348170354, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005179176046176046, "avg_duration_per_task_ms": 801836.0988455989, "avg_output_tokens_per_task": 28610.891053391053, "last_run_timestamp": "2026-10-31T16:17:18.501Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7741524627118646, "accuracy_std_dev": 0.024920321732780762, "run_count": 7, "total_questions": 1180, "avg_cost_per_task_usd": 0.010626239830508475, "avg_duration_per_task_ms": 1250036.8076271187, "avg_output_tokens_per_task": 40411.0593220339, "last_run_timestamp": "2026-10-31T08:17:43.227Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7739515454545453, "accuracy_std_dev": 0.019647060545183753, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.005980178604224059, "avg_duration_per_task_ms": 705416.41184573, "avg_output_tokens_per_task": 37567.65151515151, "last_run_timestamp": "2026-10-31T16:03:21.447Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.7735690000000001, "accuracy_std_dev": 0.005892142861472351, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.014036280303030302, "avg_duration_per_task_ms": 123381.52777777778, "avg_output_tokens_per_task": 10576.208333333334, "last_run_timestamp": "2026-10-01T19:20:04.986Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 175, "avg_cost_per_task_usd": 0.13656869142857142, "avg_duration_per_task_ms": 1521793.1542857143, "avg_output_tokens_per_task": 38871.68571428571, "last_run_timestamp": "2026-08-22T17:30:52.646Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.7732982293193716, "accuracy_std_dev": 0.013698592432801259, "run_count": 5, "total_questions": 955, "avg_cost_per_task_usd": 0.00232973612565445, "avg_duration_per_task_ms": 792784.167539267, "avg_output_tokens_per_task": 15348.367539267016, "last_run_timestamp": "2026-10-31T16:10:36.260Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.7729887726692208, "accuracy_std_dev": 0.01457930786894563, "run_count": 8, "total_questions": 1566, "avg_cost_per_task_usd": 0.02178225223499361, "avg_duration_per_task_ms": 240176.16922094507, "avg_output_tokens_per_task": 29760.224137931036, "last_run_timestamp": "2026-11-05T00:00:07.969Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.7722016904681711, "accuracy_std_dev": 0.014666151853692065, "run_count": 15, "total_questions": 2969, "avg_cost_per_task_usd": 0.006039911754799596, "avg_duration_per_task_ms": 437079.74840013473, "avg_output_tokens_per_task": 29140.058268777368, "last_run_timestamp": "2026-10-31T16:02:14.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.772166, "accuracy_std_dev": 0.04805772781423054, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01390386111111111, "avg_duration_per_task_ms": 894648.4250841751, "avg_output_tokens_per_task": 40593.27777777778, "last_run_timestamp": "2026-10-17T08:18:06.210Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.7718855, "accuracy_std_dev": 0.0025254999999999583, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017657098484848484, "avg_duration_per_task_ms": 452144.0101010101, "avg_output_tokens_per_task": 17493.924242424244, "last_run_timestamp": "2026-08-29T18:21:38.666Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Io Net", "accuracy": 0.771605, "accuracy_std_dev": null, "run_count": 1, "total_questions": 162, "avg_cost_per_task_usd": 0.0022807283950617282, "avg_duration_per_task_ms": 290757.45061728393, "avg_output_tokens_per_task": 15097.716049382716, "last_run_timestamp": "2026-11-11T15:03:02.397Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "CoreWeave", "accuracy": 0.7714860192771085, "accuracy_std_dev": 0.016470310033005864, "run_count": 6, "total_questions": 830, "avg_cost_per_task_usd": 0.009284253012048193, "avg_duration_per_task_ms": 720483.7698795181, "avg_output_tokens_per_task": 30820.519277108433, "last_run_timestamp": "2026-10-31T16:24:40.574Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.7713804000000001, "accuracy_std_dev": 0.01747606822600553, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.014499158585858585, "avg_duration_per_task_ms": 1080917.518181818, "avg_output_tokens_per_task": 51478.64141414141, "last_run_timestamp": "2026-10-31T08:09:31.070Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.7712842857142856, "accuracy_std_dev": 0.011713027432421244, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.00498178354978355, "avg_duration_per_task_ms": 549029.411976912, "avg_output_tokens_per_task": 29264.81168831169, "last_run_timestamp": "2026-10-31T16:20:12.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "SiliconFlow", "accuracy": 0.7710438, "accuracy_std_dev": 0.0072995281874926425, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.020946333333333338, "avg_duration_per_task_ms": 767702.9585858586, "avg_output_tokens_per_task": 52149.88787878788, "last_run_timestamp": "2026-10-31T16:23:19.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.7706229166666667, "accuracy_std_dev": 0.010605727544259059, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.014841917929292927, "avg_duration_per_task_ms": 138155.14436026936, "avg_output_tokens_per_task": 29486.722643097644, "last_run_timestamp": "2026-10-31T16:51:23.396Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7705627142857142, "accuracy_std_dev": 0.03293717088949932, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.06274982683982684, "avg_duration_per_task_ms": 1017904.803030303, "avg_output_tokens_per_task": 30923.48124098124, "last_run_timestamp": "2026-10-31T16:08:31.065Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Mara", "accuracy": 0.7678882984293194, "accuracy_std_dev": 0.015081000000000011, "run_count": 2, "total_questions": 191, "avg_cost_per_task_usd": 0.038533612565445026, "avg_duration_per_task_ms": 148852.86910994764, "avg_output_tokens_per_task": 40034.97905759162, "last_run_timestamp": "2026-08-29T19:44:46.543Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.7673434936548225, "accuracy_std_dev": 0.011201352155404033, "run_count": 8, "total_questions": 1576, "avg_cost_per_task_usd": 0.025760851522842635, "avg_duration_per_task_ms": 78397.31725888325, "avg_output_tokens_per_task": 26980.187817258884, "last_run_timestamp": "2026-11-11T18:52:31.116Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.7671418075356415, "accuracy_std_dev": 0.022561888094930423, "run_count": 7, "total_questions": 982, "avg_cost_per_task_usd": 0.013307315682281062, "avg_duration_per_task_ms": 53274.17718940937, "avg_output_tokens_per_task": 17643.751527494907, "last_run_timestamp": "2026-10-31T16:39:49.876Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.7670034, "accuracy_std_dev": 0.04110764542320566, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.052571260606060606, "avg_duration_per_task_ms": 891054.0545454546, "avg_output_tokens_per_task": 23291.960606060606, "last_run_timestamp": "2026-10-17T16:03:24.588Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.7669798106936416, "accuracy_std_dev": 0.011686708686296891, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.019905706647398845, "avg_duration_per_task_ms": 1154492.3439306358, "avg_output_tokens_per_task": 60181.387283236996, "last_run_timestamp": "2026-10-31T16:03:21.445Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.7669488305084746, "accuracy_std_dev": 0.016365499999999977, "run_count": 2, "total_questions": 354, "avg_cost_per_task_usd": 0.21194037853107345, "avg_duration_per_task_ms": 159764.62146892655, "avg_output_tokens_per_task": 13982.706214689266, "last_run_timestamp": "2026-09-01T13:33:46.786Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.7668349999999999, "accuracy_std_dev": 0.004208999999999963, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.057140755050505046, "avg_duration_per_task_ms": 101024.92676767676, "avg_output_tokens_per_task": 22766.989898989897, "last_run_timestamp": "2026-10-01T19:11:42.323Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.76599325, "accuracy_std_dev": 0.01423513635998966, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.018594656565656564, "avg_duration_per_task_ms": 110236.10416666667, "avg_output_tokens_per_task": 30754.41792929293, "last_run_timestamp": "2026-10-31T16:52:10.938Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7653769053571428, "accuracy_std_dev": 0.011846747415441054, "run_count": 17, "total_questions": 3360, "avg_cost_per_task_usd": 0.016902593452380953, "avg_duration_per_task_ms": 110624.59017857142, "avg_output_tokens_per_task": 5988.1654761904765, "last_run_timestamp": "2026-10-31T09:06:38.813Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.76536225, "accuracy_std_dev": 0.00536920838555365, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004458252525252525, "avg_duration_per_task_ms": 487032.047979798, "avg_output_tokens_per_task": 29595.234848484848, "last_run_timestamp": "2026-10-31T16:16:55.698Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Tenstorrent", "accuracy": 0.7653508070175438, "accuracy_std_dev": 0.03268549999999998, "run_count": 2, "total_questions": 228, "avg_cost_per_task_usd": 0.014143986842105261, "avg_duration_per_task_ms": 648386.7105263158, "avg_output_tokens_per_task": 35206.833333333336, "last_run_timestamp": "2026-11-08T04:33:18.539Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7653198, "accuracy_std_dev": 0.07983844813722271, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.004923061616161616, "avg_duration_per_task_ms": 448741.63535353536, "avg_output_tokens_per_task": 16983.609090909093, "last_run_timestamp": "2026-10-17T08:26:03.926Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Parasail", "accuracy": 0.7650311428571429, "accuracy_std_dev": 0.012201722845900242, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.021599636363636363, "avg_duration_per_task_ms": 1135053.7258297259, "avg_output_tokens_per_task": 53902.65151515151, "last_run_timestamp": "2026-10-31T16:51:21.148Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.7649719016949154, "accuracy_std_dev": 0.008301844132480422, "run_count": 3, "total_questions": 590, "avg_cost_per_task_usd": 0.0026733169491525424, "avg_duration_per_task_ms": 102640.67457627118, "avg_output_tokens_per_task": 31256.29491525424, "last_run_timestamp": "2026-10-01T16:04:09.046Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "PrimeIntellect", "accuracy": 0.76431, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0026081363636363633, "avg_duration_per_task_ms": 302574.7474747475, "avg_output_tokens_per_task": 27415.863636363636, "last_run_timestamp": "2026-11-01T06:00:07.957Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.7643099999999999, "accuracy_std_dev": 0.01683499999999999, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.057833444444444444, "avg_duration_per_task_ms": 573166.5858585859, "avg_output_tokens_per_task": 26406.381313131315, "last_run_timestamp": "2026-08-29T18:26:43.144Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Sail Research", "accuracy": 0.7640448089887641, "accuracy_std_dev": 0.007632810273199963, "run_count": 3, "total_questions": 534, "avg_cost_per_task_usd": 0.012456207865168538, "avg_duration_per_task_ms": 247061.07865168538, "avg_output_tokens_per_task": 31050.842696629214, "last_run_timestamp": "2026-11-06T21:57:37.505Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Amazon Bedrock", "accuracy": 0.7635882857142857, "accuracy_std_dev": 0.013965487794808272, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.01827745598845599, "avg_duration_per_task_ms": 144604.34848484848, "avg_output_tokens_per_task": 30221.237734487735, "last_run_timestamp": "2026-10-31T16:08:33.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.7633477142857142, "accuracy_std_dev": 0.011623763131918104, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01745055844155844, "avg_duration_per_task_ms": 63367.67532467532, "avg_output_tokens_per_task": 28845.276334776336, "last_run_timestamp": "2026-10-31T16:23:59.761Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.7632998, "accuracy_std_dev": 0.007030464092789305, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.007469111111111111, "avg_duration_per_task_ms": 123178.52121212121, "avg_output_tokens_per_task": 29681.70909090909, "last_run_timestamp": "2026-10-31T16:38:46.120Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.7629069999999999, "accuracy_std_dev": 0.00819966182229484, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.2718667558922559, "avg_duration_per_task_ms": 264601.48148148146, "avg_output_tokens_per_task": 17934.003367003366, "last_run_timestamp": "2026-09-01T13:23:02.059Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.7626260000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.022027883838383836, "avg_duration_per_task_ms": 90904.42929292929, "avg_output_tokens_per_task": 14522.767676767677, "last_run_timestamp": "2026-10-04T03:02:47.748Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7626259999999999, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01717197474747475, "avg_duration_per_task_ms": 622307.4974747475, "avg_output_tokens_per_task": 61038.34090909091, "last_run_timestamp": "2026-10-03T10:05:09.589Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7614677541284404, "accuracy_std_dev": 0.10796943486314385, "run_count": 3, "total_questions": 545, "avg_cost_per_task_usd": 0.0038067559633027524, "avg_duration_per_task_ms": 346287.8605504587, "avg_output_tokens_per_task": 20923.759633027523, "last_run_timestamp": "2026-10-08T22:27:01.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.7602212857142857, "accuracy_std_dev": 0.01112536459388809, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.017620217893217896, "avg_duration_per_task_ms": 231549.2683982684, "avg_output_tokens_per_task": 29213.04834054834, "last_run_timestamp": "2026-10-31T16:22:37.563Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.7575759999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014556641414141414, "avg_duration_per_task_ms": 173179.91919191918, "avg_output_tokens_per_task": 23980.964646464647, "last_run_timestamp": "2026-09-01T13:30:26.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.7574711434482759, "accuracy_std_dev": 0.05835625158448523, "run_count": 11, "total_questions": 2175, "avg_cost_per_task_usd": 0.014400611954022986, "avg_duration_per_task_ms": 105760.68091954023, "avg_output_tokens_per_task": 6123.435402298851, "last_run_timestamp": "2026-10-31T08:19:36.628Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7563452081218275, "accuracy_std_dev": 0.0358022938821275, "run_count": 6, "total_questions": 1182, "avg_cost_per_task_usd": 0.014391308798646361, "avg_duration_per_task_ms": 1040556.6945854484, "avg_output_tokens_per_task": 51071.41201353638, "last_run_timestamp": "2026-10-31T08:41:02.325Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Venice", "accuracy": 0.755892, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.018171858585858586, "avg_duration_per_task_ms": 285760.0353535354, "avg_output_tokens_per_task": 24027.954545454544, "last_run_timestamp": "2026-09-01T12:59:55.618Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.75477, "accuracy_std_dev": 0.008054024625407281, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.022580063973063974, "avg_duration_per_task_ms": 93134.21212121213, "avg_output_tokens_per_task": 14890.888888888889, "last_run_timestamp": "2026-10-03T16:56:57.458Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.7539042317380352, "accuracy_std_dev": 0.03122595132646972, "run_count": 12, "total_questions": 1985, "avg_cost_per_task_usd": 0.045778590428211585, "avg_duration_per_task_ms": 263292.8277078086, "avg_output_tokens_per_task": 17254.131486146096, "last_run_timestamp": "2026-10-31T08:39:08.315Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.753787875, "accuracy_std_dev": 0.02753108760309653, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.2093788106060606, "avg_duration_per_task_ms": 1100289.5448232323, "avg_output_tokens_per_task": 94261.211489899, "last_run_timestamp": "2026-11-10T19:04:50.024Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "GMICloud", "accuracy": 0.7534605555555556, "accuracy_std_dev": 0.03816866389275402, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.350709655443322, "avg_duration_per_task_ms": 957253.5684624017, "avg_output_tokens_per_task": 84600.8125701459, "last_run_timestamp": "2026-11-10T07:04:16.019Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.7528056666666667, "accuracy_std_dev": 0.004601995207395269, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015185508417508417, "avg_duration_per_task_ms": 152622.4537037037, "avg_output_tokens_per_task": 24359.508417508416, "last_run_timestamp": "2026-09-17T02:17:49.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.7526884764267989, "accuracy_std_dev": 0.036890165142246543, "run_count": 3, "total_questions": 403, "avg_cost_per_task_usd": 0.056140801488833755, "avg_duration_per_task_ms": 723763.2431761787, "avg_output_tokens_per_task": 50423.94789081886, "last_run_timestamp": "2026-08-29T19:07:57.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.75252525, "accuracy_std_dev": 0.009888238426914091, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.01365957702020202, "avg_duration_per_task_ms": 93762.47853535354, "avg_output_tokens_per_task": 15014.820707070707, "last_run_timestamp": "2026-10-03T16:54:39.591Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.7525252, "accuracy_std_dev": 0.014481875091299456, "run_count": 10, "total_questions": 1980, "avg_cost_per_task_usd": 0.015641921717171717, "avg_duration_per_task_ms": 1362988.2676767677, "avg_output_tokens_per_task": 56157.93787878788, "last_run_timestamp": "2026-11-01T13:06:18.223Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.7525250000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008515843434343435, "avg_duration_per_task_ms": 110884.25757575757, "avg_output_tokens_per_task": 8312.732323232323, "last_run_timestamp": "2026-10-04T03:06:11.710Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7514741137971698, "accuracy_std_dev": 0.03193872589510217, "run_count": 9, "total_questions": 1696, "avg_cost_per_task_usd": 0.014983457547169812, "avg_duration_per_task_ms": 382125.23054245283, "avg_output_tokens_per_task": 49724.931603773584, "last_run_timestamp": "2026-11-11T06:34:22.186Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7514029999999999, "accuracy_std_dev": 0.013279676389129373, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.015289352693602695, "avg_duration_per_task_ms": 350945.7154882155, "avg_output_tokens_per_task": 62534.38636363636, "last_run_timestamp": "2026-10-31T16:38:47.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.7510417714285714, "accuracy_std_dev": 0.04292803699474532, "run_count": 6, "total_questions": 1120, "avg_cost_per_task_usd": 0.018983601785714288, "avg_duration_per_task_ms": 779779.53125, "avg_output_tokens_per_task": 47360.87946428572, "last_run_timestamp": "2026-10-31T16:47:37.740Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.750842, "accuracy_std_dev": 0.010996575951934644, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.014766501683501684, "avg_duration_per_task_ms": 106033.30808080808, "avg_output_tokens_per_task": 14469.075757575758, "last_run_timestamp": "2026-10-03T16:43:33.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "DeepInfra", "accuracy": 0.7495937585365854, "accuracy_std_dev": 0.031913877008090064, "run_count": 4, "total_questions": 410, "avg_cost_per_task_usd": 0.047968058536585374, "avg_duration_per_task_ms": 440215.96341463417, "avg_output_tokens_per_task": 14034.10731707317, "last_run_timestamp": "2026-09-19T08:53:38.031Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.7492805738953785, "accuracy_std_dev": 0.12024394573944976, "run_count": 11, "total_questions": 1969, "avg_cost_per_task_usd": 0.0343216658202133, "avg_duration_per_task_ms": 1417972.940578974, "avg_output_tokens_per_task": 36042.608938547484, "last_run_timestamp": "2026-10-31T16:04:48.182Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Google", "accuracy": 0.749158, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.021431318181818183, "avg_duration_per_task_ms": 113907.66161616161, "avg_output_tokens_per_task": 12443.959595959595, "last_run_timestamp": "2026-08-22T17:12:08.919Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.749158, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.07988080808080808, "avg_duration_per_task_ms": 853640.3434343435, "avg_output_tokens_per_task": 22758.171717171717, "last_run_timestamp": "2026-08-22T17:38:59.682Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.7487311624365482, "accuracy_std_dev": 0.019042500000000018, "run_count": 2, "total_questions": 394, "avg_cost_per_task_usd": 0.00260698730964467, "avg_duration_per_task_ms": 166191.14467005077, "avg_output_tokens_per_task": 30980.81725888325, "last_run_timestamp": "2026-10-01T16:04:24.420Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.7481013645569621, "accuracy_std_dev": 0.004003499999999993, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.010130237974683544, "avg_duration_per_task_ms": 298779.6734177215, "avg_output_tokens_per_task": 60721.07341772152, "last_run_timestamp": "2026-10-10T08:39:12.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.7453702499999999, "accuracy_std_dev": 0.0095511571910162, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.006491083333333334, "avg_duration_per_task_ms": 157741.5366161616, "avg_output_tokens_per_task": 7540.814393939394, "last_run_timestamp": "2026-10-03T16:58:32.590Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.7438658599640935, "accuracy_std_dev": 0.0027091388955820363, "run_count": 3, "total_questions": 557, "avg_cost_per_task_usd": 0.008930811490125673, "avg_duration_per_task_ms": 129877.56014362657, "avg_output_tokens_per_task": 11161.86355475763, "last_run_timestamp": "2026-10-03T17:03:40.665Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7431237709174842, "accuracy_std_dev": 0.10658009891156979, "run_count": 9, "total_questions": 1733, "avg_cost_per_task_usd": 0.005974010963646855, "avg_duration_per_task_ms": 574816.0046162724, "avg_output_tokens_per_task": 26107.00807847663, "last_run_timestamp": "2026-10-27T02:37:18.076Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.7418633333333333, "accuracy_std_dev": 0.005722799567422317, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.023266708754208755, "avg_duration_per_task_ms": 95383.41582491582, "avg_output_tokens_per_task": 15402.81313131313, "last_run_timestamp": "2026-10-03T16:58:08.462Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.741453, "accuracy_std_dev": null, "run_count": 1, "total_questions": 156, "avg_cost_per_task_usd": 0.09924664102564103, "avg_duration_per_task_ms": 1908511.7756410257, "avg_output_tokens_per_task": 29914.98717948718, "last_run_timestamp": "2026-08-22T17:09:42.218Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.7414141999999999, "accuracy_std_dev": 0.0027351667883330276, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.00532230808080808, "avg_duration_per_task_ms": 145939.18484848485, "avg_output_tokens_per_task": 8599.7, "last_run_timestamp": "2026-10-17T16:06:15.944Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.7407406666666667, "accuracy_std_dev": 0.008584264726165454, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004939484848484848, "avg_duration_per_task_ms": 108598.20370370371, "avg_output_tokens_per_task": 8056.897306397306, "last_run_timestamp": "2026-10-03T16:59:36.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.7401770526570047, "accuracy_std_dev": 0.0865423410071625, "run_count": 12, "total_questions": 2070, "avg_cost_per_task_usd": 0.0039563893719806766, "avg_duration_per_task_ms": 552542.9028985507, "avg_output_tokens_per_task": 24944.29468599034, "last_run_timestamp": "2026-11-04T18:42:44.798Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.7392041135371179, "accuracy_std_dev": 0.011064834624399307, "run_count": 7, "total_questions": 1374, "avg_cost_per_task_usd": 0.006584693595342066, "avg_duration_per_task_ms": 301209.6688500728, "avg_output_tokens_per_task": 26146.835516739447, "last_run_timestamp": "2026-10-31T16:16:13.798Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.7388167142857144, "accuracy_std_dev": 0.023177894935318268, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006051409090909092, "avg_duration_per_task_ms": 406464.2871572872, "avg_output_tokens_per_task": 23256.089466089466, "last_run_timestamp": "2026-10-31T16:08:31.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.7384959999999999, "accuracy_std_dev": 0.008170584087648423, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.06071549915824916, "avg_duration_per_task_ms": 125902.70959595959, "avg_output_tokens_per_task": 24199.07239057239, "last_run_timestamp": "2026-10-01T19:15:31.968Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.7380049999999998, "accuracy_std_dev": 0.023295947045569962, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.3327117329545455, "avg_duration_per_task_ms": 1075619.288510101, "avg_output_tokens_per_task": 99143.67108585859, "last_run_timestamp": "2026-11-10T19:38:16.555Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.7369396718907987, "accuracy_std_dev": 0.01294608625956121, "run_count": 10, "total_questions": 1978, "avg_cost_per_task_usd": 0.006695506572295248, "avg_duration_per_task_ms": 136027.5783619818, "avg_output_tokens_per_task": 6995.519716885743, "last_run_timestamp": "2026-10-17T16:03:44.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7365419711229947, "accuracy_std_dev": 0.1044922112691581, "run_count": 7, "total_questions": 935, "avg_cost_per_task_usd": 0.09201655614973261, "avg_duration_per_task_ms": 1267131.9229946523, "avg_output_tokens_per_task": 26199.03101604278, "last_run_timestamp": "2026-10-31T16:05:14.699Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.7361712857142858, "accuracy_std_dev": 0.00933947044559173, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0070149227994227995, "avg_duration_per_task_ms": 250938.70995670996, "avg_output_tokens_per_task": 8674.670995670996, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.7353578611713666, "accuracy_std_dev": 0.08764063669791727, "run_count": 7, "total_questions": 1383, "avg_cost_per_task_usd": 0.004913945770065076, "avg_duration_per_task_ms": 616099.819956616, "avg_output_tokens_per_task": 32251.255242227042, "last_run_timestamp": "2026-10-17T16:03:21.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7352091428571429, "accuracy_std_dev": 0.007011626557339723, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.11408521356421357, "avg_duration_per_task_ms": 767202.2121212122, "avg_output_tokens_per_task": 44535.16594516594, "last_run_timestamp": "2026-10-31T16:04:48.154Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.733364333488806, "accuracy_std_dev": 0.03217972751863435, "run_count": 11, "total_questions": 2144, "avg_cost_per_task_usd": 0.010386112406716419, "avg_duration_per_task_ms": 251820.99860074627, "avg_output_tokens_per_task": 48631.178638059704, "last_run_timestamp": "2026-10-31T08:46:43.680Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.7328843333333334, "accuracy_std_dev": 0.0007938452130121063, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0046818838383838384, "avg_duration_per_task_ms": 164385.80303030304, "avg_output_tokens_per_task": 7946.1447811447815, "last_run_timestamp": "2026-10-03T16:57:10.991Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.732440575, "accuracy_std_dev": 0.057246596407316816, "run_count": 3, "total_questions": 560, "avg_cost_per_task_usd": 0.014524144642857144, "avg_duration_per_task_ms": 428029.5660714286, "avg_output_tokens_per_task": 58878.205357142855, "last_run_timestamp": "2026-10-31T08:03:10.090Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.7323230000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0787700707070707, "avg_duration_per_task_ms": 1116199.0808080807, "avg_output_tokens_per_task": 26175.146464646463, "last_run_timestamp": "2026-08-22T17:45:06.645Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.732155, "accuracy_std_dev": 0.02424727451075689, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.016461675757575758, "avg_duration_per_task_ms": 673272.2666666667, "avg_output_tokens_per_task": 60883.92525252525, "last_run_timestamp": "2026-10-31T08:31:11.846Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.7320426666666667, "accuracy_std_dev": 0.006532295121590524, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.006095060606060606, "avg_duration_per_task_ms": 411941.8905723906, "avg_output_tokens_per_task": 7966.672558922559, "last_run_timestamp": "2026-10-31T16:08:56.903Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.7315579999999999, "accuracy_std_dev": 0.007776888066572624, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.19043861111111113, "avg_duration_per_task_ms": 238176.99081726355, "avg_output_tokens_per_task": 37896.631313131315, "last_run_timestamp": "2026-10-31T08:20:09.279Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Mara", "accuracy": 0.7294072124711316, "accuracy_std_dev": 0.019970689066868837, "run_count": 6, "total_questions": 866, "avg_cost_per_task_usd": 0.016059699769053117, "avg_duration_per_task_ms": 62647.899538106234, "avg_output_tokens_per_task": 9232.347575057736, "last_run_timestamp": "2026-10-31T16:30:53.500Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.7291284439132034, "accuracy_std_dev": 0.048248946497751195, "run_count": 14, "total_questions": 2719, "avg_cost_per_task_usd": 0.009029743655755791, "avg_duration_per_task_ms": 527982.3622655388, "avg_output_tokens_per_task": 26831.921662375873, "last_run_timestamp": "2026-10-31T16:52:55.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7289052424242423, "accuracy_std_dev": 0.011972354458274426, "run_count": 33, "total_questions": 6534, "avg_cost_per_task_usd": 0.2043336472298745, "avg_duration_per_task_ms": 238207.2739516376, "avg_output_tokens_per_task": 38129.07560453015, "last_run_timestamp": "2026-10-31T08:33:11.343Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7281145, "accuracy_std_dev": 0.004208499999999948, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.060785368686868686, "avg_duration_per_task_ms": 121296.46212121213, "avg_output_tokens_per_task": 24226.65404040404, "last_run_timestamp": "2026-10-01T19:12:41.089Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.7278849090909092, "accuracy_std_dev": 0.012272399739063737, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.18833512167125802, "avg_duration_per_task_ms": 344775.9109274564, "avg_output_tokens_per_task": 37475.9334251607, "last_run_timestamp": "2026-10-31T08:16:11.695Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.7272728571428572, "accuracy_std_dev": 0.016272465871865285, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005508002886002886, "avg_duration_per_task_ms": 323368.19624819624, "avg_output_tokens_per_task": 8110.7409812409815, "last_run_timestamp": "2026-10-31T16:22:55.771Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.7271808509532062, "accuracy_std_dev": 0.03540222199908109, "run_count": 6, "total_questions": 1154, "avg_cost_per_task_usd": 0.011152869150779895, "avg_duration_per_task_ms": 504501.5831889082, "avg_output_tokens_per_task": 55506.62998266898, "last_run_timestamp": "2026-10-10T08:24:27.616Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Cohere", "accuracy": 0.7267116666666668, "accuracy_std_dev": 0.01049864519301841, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.021460314814814814, "avg_duration_per_task_ms": 244130.04882154881, "avg_output_tokens_per_task": 76264.67003367003, "last_run_timestamp": "2026-10-31T08:19:36.711Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.7264309999999999, "accuracy_std_dev": 0.021044000000000007, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.1303771111111111, "avg_duration_per_task_ms": 318087.7601010101, "avg_output_tokens_per_task": 43267.416666666664, "last_run_timestamp": "2026-10-04T05:57:29.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.7252831818181819, "accuracy_std_dev": 0.010667148505559918, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.1864189655647383, "avg_duration_per_task_ms": 310620.87190082646, "avg_output_tokens_per_task": 37109.52800734619, "last_run_timestamp": "2026-10-31T08:09:30.774Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.7251266559865094, "accuracy_std_dev": 0.005701732738582379, "run_count": 3, "total_questions": 593, "avg_cost_per_task_usd": 0.01145135581787521, "avg_duration_per_task_ms": 98413.20741989883, "avg_output_tokens_per_task": 10340.510961214166, "last_run_timestamp": "2026-10-03T16:43:17.105Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.7234044255319149, "accuracy_std_dev": 0.1917994784299478, "run_count": 3, "total_questions": 517, "avg_cost_per_task_usd": 0.033565591876208896, "avg_duration_per_task_ms": 242772.43907156674, "avg_output_tokens_per_task": 37102.30560928433, "last_run_timestamp": "2026-09-29T20:57:20.673Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.7226813939393939, "accuracy_std_dev": 0.014086885758036204, "run_count": 33, "total_questions": 6534, "avg_cost_per_task_usd": 0.19769731894704623, "avg_duration_per_task_ms": 272979.47413529234, "avg_output_tokens_per_task": 36868.905111723296, "last_run_timestamp": "2026-10-31T08:29:35.324Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.72106475, "accuracy_std_dev": 0.011038884066675383, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.004405270833333332, "avg_duration_per_task_ms": 708002.5321969697, "avg_output_tokens_per_task": 7885.837121212121, "last_run_timestamp": "2026-10-31T16:02:18.363Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.720845, "accuracy_std_dev": 0.13313462084603755, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.6171295247933883, "avg_duration_per_task_ms": 640700.6698806244, "avg_output_tokens_per_task": 40986.32323232323, "last_run_timestamp": "2026-10-31T08:09:30.979Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7205387272727274, "accuracy_std_dev": 0.14017884451338383, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.6223622332415059, "avg_duration_per_task_ms": 640560.214876033, "avg_output_tokens_per_task": 41343.55968778696, "last_run_timestamp": "2026-10-31T08:28:17.671Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7205387272727272, "accuracy_std_dev": 0.13030117338555663, "run_count": 33, "total_questions": 6534, "avg_cost_per_task_usd": 0.6590140376492195, "avg_duration_per_task_ms": 429144.30731558, "avg_output_tokens_per_task": 41017.19651056015, "last_run_timestamp": "2026-10-31T08:31:32.893Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.719695823610473, "accuracy_std_dev": 0.13461887126264796, "run_count": 33, "total_questions": 6531, "avg_cost_per_task_usd": 0.6638264376052672, "avg_duration_per_task_ms": 604267.6686571735, "avg_output_tokens_per_task": 41332.92175777063, "last_run_timestamp": "2026-10-31T08:51:32.081Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7183742857142856, "accuracy_std_dev": 0.012394494661678109, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01898589466089466, "avg_duration_per_task_ms": 501749.6746031746, "avg_output_tokens_per_task": 67465.82683982684, "last_run_timestamp": "2026-10-31T08:06:04.074Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.7177330000000001, "accuracy_std_dev": 0.0028613344206273234, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.011208954545454548, "avg_duration_per_task_ms": 85332.55555555556, "avg_output_tokens_per_task": 10503.252525252525, "last_run_timestamp": "2026-10-03T16:59:33.949Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.7171206858771375, "accuracy_std_dev": 0.09050148781226955, "run_count": 8, "total_questions": 1579, "avg_cost_per_task_usd": 0.011264650411652945, "avg_duration_per_task_ms": 777628.9563014567, "avg_output_tokens_per_task": 32969.615579480684, "last_run_timestamp": "2026-10-31T16:08:08.535Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7166907142857143, "accuracy_std_dev": 0.03789838215813547, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.3448133593073593, "avg_duration_per_task_ms": 1259875.4646464647, "avg_output_tokens_per_task": 107708.73232323233, "last_run_timestamp": "2026-10-31T16:18:48.427Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.716486229602888, "accuracy_std_dev": 0.037837545924002784, "run_count": 7, "total_questions": 1385, "avg_cost_per_task_usd": 0.33939834729241875, "avg_duration_per_task_ms": 1241884.499638989, "avg_output_tokens_per_task": 105962.96534296029, "last_run_timestamp": "2026-10-31T16:18:28.925Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.7158777222222221, "accuracy_std_dev": 0.12243858819244856, "run_count": 5, "total_questions": 972, "avg_cost_per_task_usd": 0.03800503395061729, "avg_duration_per_task_ms": 541259.3065843622, "avg_output_tokens_per_task": 34594.609053497945, "last_run_timestamp": "2026-09-19T17:18:00.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.7150290909090908, "accuracy_std_dev": 0.14315254393475418, "run_count": 11, "total_questions": 2178, "avg_cost_per_task_usd": 0.6259749265381084, "avg_duration_per_task_ms": 641693.616620753, "avg_output_tokens_per_task": 41568.741965105604, "last_run_timestamp": "2026-10-31T08:22:47.508Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.7149273333333334, "accuracy_std_dev": 0.0015872190215034077, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01138610101010101, "avg_duration_per_task_ms": 186927.4898989899, "avg_output_tokens_per_task": 5929.171717171717, "last_run_timestamp": "2026-10-03T16:59:57.847Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.7132433333333333, "accuracy_std_dev": 0.008837261503932612, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00784910101010101, "avg_duration_per_task_ms": 79021.32323232324, "avg_output_tokens_per_task": 9968.289562289563, "last_run_timestamp": "2026-10-03T16:40:12.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Darkbloom", "accuracy": 0.713213, "accuracy_std_dev": null, "run_count": 1, "total_questions": 111, "avg_cost_per_task_usd": 0.007665369369369369, "avg_duration_per_task_ms": 1401051.936936937, "avg_output_tokens_per_task": 27183.8018018018, "last_run_timestamp": "2026-10-03T09:34:00.571Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.7115600000000001, "accuracy_std_dev": 0.0034590473640392213, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.006004388888888889, "avg_duration_per_task_ms": 110119.68181818182, "avg_output_tokens_per_task": 5570.340067340067, "last_run_timestamp": "2026-10-03T17:02:24.058Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.7115600000000001, "accuracy_std_dev": 0.00975207530050231, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.008760998316498316, "avg_duration_per_task_ms": 118710.20370370371, "avg_output_tokens_per_task": 5678.178451178451, "last_run_timestamp": "2026-10-03T16:45:33.688Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.7110037019328587, "accuracy_std_dev": 0.031890632320636075, "run_count": 10, "total_questions": 1966, "avg_cost_per_task_usd": 0.0072013123092573746, "avg_duration_per_task_ms": 309646.26246185147, "avg_output_tokens_per_task": 40164.048830111904, "last_run_timestamp": "2026-10-31T08:15:24.314Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.7104376666666667, "accuracy_std_dev": 0.004123715988711595, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015477676767676767, "avg_duration_per_task_ms": 105803.74579124579, "avg_output_tokens_per_task": 10237.208754208754, "last_run_timestamp": "2026-10-03T16:46:22.205Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "AkashML", "accuracy": 0.71001675, "accuracy_std_dev": 0.018948683824675006, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015486457070707072, "avg_duration_per_task_ms": 1105788.4797979798, "avg_output_tokens_per_task": 64321.568181818184, "last_run_timestamp": "2026-10-31T16:02:14.361Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.707071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008867828282828283, "avg_duration_per_task_ms": 116045.65151515152, "avg_output_tokens_per_task": 5749.39898989899, "last_run_timestamp": "2026-10-04T03:05:59.018Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.7061267289719626, "accuracy_std_dev": 0.026110787988875415, "run_count": 10, "total_questions": 1926, "avg_cost_per_task_usd": 0.00886934890965732, "avg_duration_per_task_ms": 374407.95742471446, "avg_output_tokens_per_task": 45095.30893042575, "last_run_timestamp": "2026-10-31T08:03:09.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.7048043337547408, "accuracy_std_dev": 0.0036369767922685555, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.010453273072060683, "avg_duration_per_task_ms": 132200.4804045512, "avg_output_tokens_per_task": 9436.553729456384, "last_run_timestamp": "2026-10-08T22:40:37.311Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.7045455, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01336774494949495, "avg_duration_per_task_ms": 231791.09848484848, "avg_output_tokens_per_task": 33323.15909090909, "last_run_timestamp": "2026-10-01T19:27:22.032Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.703844, "accuracy_std_dev": 0.13064484273849722, "run_count": 12, "total_questions": 2376, "avg_cost_per_task_usd": 0.611304741161616, "avg_duration_per_task_ms": 637806.0361952862, "avg_output_tokens_per_task": 40663.66961279461, "last_run_timestamp": "2026-10-31T08:38:33.095Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.7028622499999999, "accuracy_std_dev": 0.08200476941122087, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0070032777777777775, "avg_duration_per_task_ms": 172461.12373737374, "avg_output_tokens_per_task": 7212.60101010101, "last_run_timestamp": "2026-10-03T16:56:36.098Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Parasail", "accuracy": 0.7001132585034013, "accuracy_std_dev": 0.0646565, "run_count": 2, "total_questions": 294, "avg_cost_per_task_usd": 0.006708948979591837, "avg_duration_per_task_ms": 418040.18707482994, "avg_output_tokens_per_task": 32070.63945578231, "last_run_timestamp": "2026-08-29T12:30:09.617Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6952379824175824, "accuracy_std_dev": 0.04288600202768226, "run_count": 7, "total_questions": 1365, "avg_cost_per_task_usd": 0.031070087912087915, "avg_duration_per_task_ms": 755530.3274725275, "avg_output_tokens_per_task": 18572.407326007327, "last_run_timestamp": "2026-10-31T16:23:19.099Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.6947253333333333, "accuracy_std_dev": 0.03660940543388032, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.010800447811447811, "avg_duration_per_task_ms": 53436.71548821549, "avg_output_tokens_per_task": 8968.276094276094, "last_run_timestamp": "2026-10-03T16:36:16.895Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.694725, "accuracy_std_dev": 0.007570749896806768, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.007473897306397308, "avg_duration_per_task_ms": 291565.11447811447, "avg_output_tokens_per_task": 5943.813131313132, "last_run_timestamp": "2026-10-03T17:02:48.284Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.6944443846153847, "accuracy_std_dev": 0.1332283983619215, "run_count": 13, "total_questions": 2574, "avg_cost_per_task_usd": 0.02886994483294483, "avg_duration_per_task_ms": 752020.932012432, "avg_output_tokens_per_task": 41085.64568764569, "last_run_timestamp": "2026-11-13T02:11:08.485Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6927611428571429, "accuracy_std_dev": 0.04357394862080053, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.014586953823953825, "avg_duration_per_task_ms": 929278.9754689755, "avg_output_tokens_per_task": 50665.61544011544, "last_run_timestamp": "2026-10-31T16:11:46.431Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.6917989999999999, "accuracy_std_dev": 0.053895225463061976, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.14352880086580086, "avg_duration_per_task_ms": 700927.1991341992, "avg_output_tokens_per_task": 43119.867965367965, "last_run_timestamp": "2026-10-31T16:56:31.817Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.69149825, "accuracy_std_dev": 0.011926428811152977, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.005578758838383838, "avg_duration_per_task_ms": 386937.1830808081, "avg_output_tokens_per_task": 6154.853535353535, "last_run_timestamp": "2026-10-08T23:09:15.796Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.690446125, "accuracy_std_dev": 0.011976425410337372, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.00649373611111111, "avg_duration_per_task_ms": 110586.78093434343, "avg_output_tokens_per_task": 15958.085227272728, "last_run_timestamp": "2026-10-31T16:08:56.904Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.689815, "accuracy_std_dev": 0.0034448118235979193, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.013852137626262627, "avg_duration_per_task_ms": 234308.42045454544, "avg_output_tokens_per_task": 32893.11994949495, "last_run_timestamp": "2026-10-01T19:27:10.142Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.6884587777777779, "accuracy_std_dev": 0.0671979156138009, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.007667775533108866, "avg_duration_per_task_ms": 1347693.569023569, "avg_output_tokens_per_task": 59908.588664422, "last_run_timestamp": "2026-11-13T01:44:30.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DekaLLM", "accuracy": 0.6866834096080909, "accuracy_std_dev": 0.0076575271261354015, "run_count": 4, "total_questions": 791, "avg_cost_per_task_usd": 0.007103366624525917, "avg_duration_per_task_ms": 1621893.5347661187, "avg_output_tokens_per_task": 50543.86852085967, "last_run_timestamp": "2026-10-31T16:36:31.734Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.6866618266276519, "accuracy_std_dev": 0.06204446895442602, "run_count": 7, "total_questions": 1367, "avg_cost_per_task_usd": 0.051509942209217265, "avg_duration_per_task_ms": 1178589.2787125092, "avg_output_tokens_per_task": 24138.589612289685, "last_run_timestamp": "2026-10-31T16:02:14.362Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.6843435000000001, "accuracy_std_dev": 0.005892499999999967, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01356670707070707, "avg_duration_per_task_ms": 229761.702020202, "avg_output_tokens_per_task": 33820.56313131313, "last_run_timestamp": "2026-10-01T19:28:34.261Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SambaNova", "accuracy": 0.6821764194977844, "accuracy_std_dev": 0.028397019030270698, "run_count": 6, "total_questions": 677, "avg_cost_per_task_usd": 0.008931531757754801, "avg_duration_per_task_ms": 55086.47267355982, "avg_output_tokens_per_task": 5789.069423929099, "last_run_timestamp": "2026-10-31T16:13:09.867Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SiliconFlow", "accuracy": 0.6821534336283186, "accuracy_std_dev": 0.02131712035248777, "run_count": 7, "total_questions": 1356, "avg_cost_per_task_usd": 0.0023845553097345137, "avg_duration_per_task_ms": 215586.99410029498, "avg_output_tokens_per_task": 4060.5553097345132, "last_run_timestamp": "2026-10-31T16:18:48.428Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.6809765, "accuracy_std_dev": 0.000841499999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00570934090909091, "avg_duration_per_task_ms": 525980.6186868687, "avg_output_tokens_per_task": 16193.580808080807, "last_run_timestamp": "2026-08-29T18:40:21.422Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6804673184713376, "accuracy_std_dev": 0.00007650000000003487, "run_count": 2, "total_questions": 314, "avg_cost_per_task_usd": 0.03821310191082802, "avg_duration_per_task_ms": 669073.7388535031, "avg_output_tokens_per_task": 40207.68152866242, "last_run_timestamp": "2026-08-29T18:51:07.939Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.6803991483087598, "accuracy_std_dev": 0.010339641763888907, "run_count": 6, "total_questions": 1153, "avg_cost_per_task_usd": 0.018036671292281006, "avg_duration_per_task_ms": 929552.2064180399, "avg_output_tokens_per_task": 89989.56808326105, "last_run_timestamp": "2026-09-26T16:34:21.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Google", "accuracy": 0.679672686416185, "accuracy_std_dev": 0.07384322715989433, "run_count": 7, "total_questions": 1384, "avg_cost_per_task_usd": 0.010168833092485549, "avg_duration_per_task_ms": 385418.2059248555, "avg_output_tokens_per_task": 28139.643063583815, "last_run_timestamp": "2026-10-31T16:12:07.129Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.6785352626262626, "accuracy_std_dev": 0.014156861524116758, "run_count": 13, "total_questions": 1980, "avg_cost_per_task_usd": 0.005069473232323233, "avg_duration_per_task_ms": 756595.5121212121, "avg_output_tokens_per_task": 50528.773232323234, "last_run_timestamp": "2026-10-31T16:04:29.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6767676666666665, "accuracy_std_dev": 0.1409255711923457, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.007406489898989899, "avg_duration_per_task_ms": 706859.9983164984, "avg_output_tokens_per_task": 26149.993265993267, "last_run_timestamp": "2026-10-31T16:45:39.993Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.6734009999999999, "accuracy_std_dev": 0.063973, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.00633028787878788, "avg_duration_per_task_ms": 117057.46717171717, "avg_output_tokens_per_task": 7022.232323232323, "last_run_timestamp": "2026-08-29T18:01:46.940Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.6709957142857143, "accuracy_std_dev": 0.0163185813959275, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007971292207792208, "avg_duration_per_task_ms": 515085.1370851371, "avg_output_tokens_per_task": 61096.35209235209, "last_run_timestamp": "2026-10-31T16:08:31.061Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.6708364177407126, "accuracy_std_dev": 0.019300239418446996, "run_count": 8, "total_questions": 1319, "avg_cost_per_task_usd": 0.006608144806671721, "avg_duration_per_task_ms": 469922.3040181956, "avg_output_tokens_per_task": 47033.74753601213, "last_run_timestamp": "2026-10-31T16:03:21.483Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.6691920000000001, "accuracy_std_dev": 0.007576000000000027, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.002490050505050505, "avg_duration_per_task_ms": 88307.96464646465, "avg_output_tokens_per_task": 2307.4141414141413, "last_run_timestamp": "2026-08-29T20:54:16.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.6689113333333334, "accuracy_std_dev": 0.0067804647005611455, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0026409175084175086, "avg_duration_per_task_ms": 47935.90572390572, "avg_output_tokens_per_task": 2383.6902356902356, "last_run_timestamp": "2026-10-02T15:53:31.869Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.6683501428571427, "accuracy_std_dev": 0.007142653957110113, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.005125032467532468, "avg_duration_per_task_ms": 102657.88744588745, "avg_output_tokens_per_task": 9652.720057720058, "last_run_timestamp": "2026-10-03T09:23:08.256Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.66835, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013946772727272728, "avg_duration_per_task_ms": 324889.0353535354, "avg_output_tokens_per_task": 15315.242424242424, "last_run_timestamp": "2026-10-01T08:41:30.639Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.667508642857143, "accuracy_std_dev": 0.01141368504163279, "run_count": 14, "total_questions": 2772, "avg_cost_per_task_usd": 0.009206428571428572, "avg_duration_per_task_ms": 323973.83044733043, "avg_output_tokens_per_task": 60939.117965367965, "last_run_timestamp": "2026-10-31T16:08:08.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 196, "avg_cost_per_task_usd": 0.05274626530612245, "avg_duration_per_task_ms": 2852978.0561224488, "avg_output_tokens_per_task": 28832.622448979593, "last_run_timestamp": "2026-08-22T18:19:05.094Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Phala", "accuracy": 0.6666668592964823, "accuracy_std_dev": 0.0020275000000000154, "run_count": 2, "total_questions": 199, "avg_cost_per_task_usd": 0.004585065326633166, "avg_duration_per_task_ms": 615862.0050251256, "avg_output_tokens_per_task": 30806.96984924623, "last_run_timestamp": "2026-08-29T17:24:11.557Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.6666665714285714, "accuracy_std_dev": 0.017285925272620274, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.00577388023088023, "avg_duration_per_task_ms": 421454.3968253968, "avg_output_tokens_per_task": 15830.38961038961, "last_run_timestamp": "2026-10-31T16:17:15.718Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.6652235714285715, "accuracy_std_dev": 0.018626001977704986, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008422984848484848, "avg_duration_per_task_ms": 801001.3318903319, "avg_output_tokens_per_task": 56678.3354978355, "last_run_timestamp": "2026-10-31T16:08:56.902Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.663861, "accuracy_std_dev": 0.007815979699734826, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002078132996632997, "avg_duration_per_task_ms": 241271.14814814815, "avg_output_tokens_per_task": 3140.030303030303, "last_run_timestamp": "2026-10-05T20:44:48.611Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.6633000000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.013461267676767677, "avg_duration_per_task_ms": 327894.71717171714, "avg_output_tokens_per_task": 14738.742424242424, "last_run_timestamp": "2026-10-01T08:27:45.371Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.6632995, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0029102499999999996, "avg_duration_per_task_ms": 81353.34090909091, "avg_output_tokens_per_task": 2988.2853535353534, "last_run_timestamp": "2026-08-29T19:49:56.943Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.659933, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.014984520202020202, "avg_duration_per_task_ms": 348987.1767676768, "avg_output_tokens_per_task": 16461.863636363636, "last_run_timestamp": "2026-10-01T08:28:22.621Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.6599326666666666, "accuracy_std_dev": 0.013255807590972711, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.015927558922558923, "avg_duration_per_task_ms": 14095.8367003367, "avg_output_tokens_per_task": 1794.7289562289561, "last_run_timestamp": "2026-10-02T23:17:48.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.658249, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0066776414141414145, "avg_duration_per_task_ms": 101359.9898989899, "avg_output_tokens_per_task": 9307.358585858587, "last_run_timestamp": "2026-10-04T03:07:35.168Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.65736, "accuracy_std_dev": null, "run_count": 1, "total_questions": 197, "avg_cost_per_task_usd": 0.014864365482233502, "avg_duration_per_task_ms": 1717906.416243655, "avg_output_tokens_per_task": 64415.350253807104, "last_run_timestamp": "2026-10-01T08:48:38.304Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.657127, "accuracy_std_dev": 0.05754067397936871, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01756317676767677, "avg_duration_per_task_ms": 295248.13804713805, "avg_output_tokens_per_task": 9861.76430976431, "last_run_timestamp": "2026-10-03T17:36:48.437Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.6523570000000002, "accuracy_std_dev": 0.01094299999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.003023964646464647, "avg_duration_per_task_ms": 45756.330808080806, "avg_output_tokens_per_task": 2857.7727272727275, "last_run_timestamp": "2026-08-29T19:51:51.198Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.652357, "accuracy_std_dev": 0.01262599999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004161952020202021, "avg_duration_per_task_ms": 27160.11111111111, "avg_output_tokens_per_task": 2408.8232323232323, "last_run_timestamp": "2026-10-01T19:12:31.553Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.6523569999999999, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01652235606060606, "avg_duration_per_task_ms": 1917916.494949495, "avg_output_tokens_per_task": 65966.15909090909, "last_run_timestamp": "2026-10-09T00:15:37.020Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.6522367142857142, "accuracy_std_dev": 0.024854328583708306, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0057348217893217895, "avg_duration_per_task_ms": 642012.7799422799, "avg_output_tokens_per_task": 41761.8759018759, "last_run_timestamp": "2026-10-31T16:01:53.622Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.651515, "accuracy_std_dev": 0.1905456270048375, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.058297676767676776, "avg_duration_per_task_ms": 1098090.5791245792, "avg_output_tokens_per_task": 23174.20202020202, "last_run_timestamp": "2026-10-17T16:03:44.041Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.6492706666666667, "accuracy_std_dev": 0.010498645193018418, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0041127255892255896, "avg_duration_per_task_ms": 24949.132996632998, "avg_output_tokens_per_task": 2375.9343434343436, "last_run_timestamp": "2026-10-02T23:15:53.868Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.6492706666666667, "accuracy_std_dev": 0.0055553002519115975, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.002898063973063973, "avg_duration_per_task_ms": 166642.234006734, "avg_output_tokens_per_task": 10315.60606060606, "last_run_timestamp": "2026-10-03T17:16:11.892Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "CoreWeave", "accuracy": 0.647493, "accuracy_std_dev": null, "run_count": 1, "total_questions": 113, "avg_cost_per_task_usd": 0.005189893805309735, "avg_duration_per_task_ms": 619234.2654867256, "avg_output_tokens_per_task": 34365.4778761062, "last_run_timestamp": "2026-10-01T08:28:32.911Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DeepInfra", "accuracy": 0.6473065, "accuracy_std_dev": 0.012626499999999985, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.002362989898989899, "avg_duration_per_task_ms": 57564.656565656565, "avg_output_tokens_per_task": 2763.429292929293, "last_run_timestamp": "2026-08-29T16:11:23.388Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.646465, "accuracy_std_dev": 0.007273551310513057, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004810424242424242, "avg_duration_per_task_ms": 80194.98484848485, "avg_output_tokens_per_task": 9051.29797979798, "last_run_timestamp": "2026-10-03T16:51:25.541Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6450988763882182, "accuracy_std_dev": 0.03520097627520112, "run_count": 11, "total_questions": 2071, "avg_cost_per_task_usd": 0.005406431675519074, "avg_duration_per_task_ms": 792878.7054563013, "avg_output_tokens_per_task": 36344.80685659102, "last_run_timestamp": "2026-11-13T01:39:57.890Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Novita", "accuracy": 0.644781, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.002468472222222222, "avg_duration_per_task_ms": 55306.53787878788, "avg_output_tokens_per_task": 2662.2853535353534, "last_run_timestamp": "2026-08-29T17:03:33.297Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Crusoe", "accuracy": 0.6443605, "accuracy_std_dev": 0.007996499999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0038927146464646464, "avg_duration_per_task_ms": 63456.618686868685, "avg_output_tokens_per_task": 2359.770202020202, "last_run_timestamp": "2026-08-29T16:10:45.295Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.6418952857142858, "accuracy_std_dev": 0.013932428582414525, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.007236988455988457, "avg_duration_per_task_ms": 236970.17243867245, "avg_output_tokens_per_task": 11526.8658008658, "last_run_timestamp": "2026-10-03T10:20:18.247Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.641414, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00284089898989899, "avg_duration_per_task_ms": 209352.64646464647, "avg_output_tokens_per_task": 9266.040404040405, "last_run_timestamp": "2026-10-09T15:00:06.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.6409465555555555, "accuracy_std_dev": 0.012642544119063567, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.0015052222222222225, "avg_duration_per_task_ms": 370215.3058361392, "avg_output_tokens_per_task": 16243.532547699215, "last_run_timestamp": "2026-11-13T01:55:29.605Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.6401515, "accuracy_std_dev": 0.002759842885020819, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004124544191919192, "avg_duration_per_task_ms": 34476.35353535353, "avg_output_tokens_per_task": 2259.632575757576, "last_run_timestamp": "2026-10-01T19:13:29.560Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.6372055000000001, "accuracy_std_dev": 0.004208500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.015519383838383837, "avg_duration_per_task_ms": 14506.310606060606, "avg_output_tokens_per_task": 1746.2525252525252, "last_run_timestamp": "2026-10-01T19:12:45.033Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.6367847499999999, "accuracy_std_dev": 0.012277609526593496, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.015237281565656567, "avg_duration_per_task_ms": 18840.242424242424, "avg_output_tokens_per_task": 1618.909090909091, "last_run_timestamp": "2026-10-01T19:14:40.644Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.6366041428571428, "accuracy_std_dev": 0.13875937770258326, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.014381060606060606, "avg_duration_per_task_ms": 413704.7886002886, "avg_output_tokens_per_task": 12007.351370851371, "last_run_timestamp": "2026-10-31T16:01:53.629Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.6356422857142857, "accuracy_std_dev": 0.03719902777421659, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.006775251803751804, "avg_duration_per_task_ms": 102294.06782106782, "avg_output_tokens_per_task": 9446.849206349207, "last_run_timestamp": "2026-10-03T10:32:36.729Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Groq", "accuracy": 0.633429739884393, "accuracy_std_dev": 0.015053999999999956, "run_count": 2, "total_questions": 346, "avg_cost_per_task_usd": 0.007234326589595376, "avg_duration_per_task_ms": 26683.387283236993, "avg_output_tokens_per_task": 11969.693641618496, "last_run_timestamp": "2026-08-29T16:23:45.123Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.633117, "accuracy_std_dev": 0.010839761238804363, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.01982476406926407, "avg_duration_per_task_ms": 532402.84992785, "avg_output_tokens_per_task": 99141.9025974026, "last_run_timestamp": "2026-10-03T17:08:02.072Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.6322752857142858, "accuracy_std_dev": 0.02198430561439089, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.018390331168831167, "avg_duration_per_task_ms": 674719.9083694083, "avg_output_tokens_per_task": 89392.03174603175, "last_run_timestamp": "2026-10-03T17:40:51.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.632155, "accuracy_std_dev": 0.005892000000000008, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005966434343434343, "avg_duration_per_task_ms": 235473.01515151514, "avg_output_tokens_per_task": 48249.11363636364, "last_run_timestamp": "2026-10-01T08:24:52.614Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.6318743333333333, "accuracy_std_dev": 0.008054489114083432, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0027108804713804713, "avg_duration_per_task_ms": 284802.3080808081, "avg_output_tokens_per_task": 8805.387205387206, "last_run_timestamp": "2026-10-03T17:03:50.406Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6308699561200923, "accuracy_std_dev": 0.2332223523022349, "run_count": 7, "total_questions": 1299, "avg_cost_per_task_usd": 0.009968328714395689, "avg_duration_per_task_ms": 169608.67359507314, "avg_output_tokens_per_task": 16432.014626635875, "last_run_timestamp": "2026-10-31T16:54:17.849Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "DekaLLM", "accuracy": 0.6306148156028369, "accuracy_std_dev": 0.05281625698168657, "run_count": 3, "total_questions": 282, "avg_cost_per_task_usd": 0.0037584255319148938, "avg_duration_per_task_ms": 208452.59574468085, "avg_output_tokens_per_task": 12308.468085106382, "last_run_timestamp": "2026-10-23T09:29:14.171Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "SiliconFlow", "accuracy": 0.6294955496017379, "accuracy_std_dev": 0.007954455460524567, "run_count": 7, "total_questions": 1381, "avg_cost_per_task_usd": 0.0018547262853005066, "avg_duration_per_task_ms": 225698.48443157133, "avg_output_tokens_per_task": 10120.918175235336, "last_run_timestamp": "2026-10-31T16:10:09.221Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.62878825, "accuracy_std_dev": 0.0068900206957236525, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.001630263888888889, "avg_duration_per_task_ms": 77290.25, "avg_output_tokens_per_task": 8242.138888888889, "last_run_timestamp": "2026-10-03T16:47:32.559Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.627104375, "accuracy_std_dev": 0.008333089267155095, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.024497946969696968, "avg_duration_per_task_ms": 1060409.1414141415, "avg_output_tokens_per_task": 101874.34027777778, "last_run_timestamp": "2026-10-23T09:03:34.804Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.6253323876329787, "accuracy_std_dev": 0.21157174135765242, "run_count": 8, "total_questions": 1504, "avg_cost_per_task_usd": 0.010399158909574468, "avg_duration_per_task_ms": 276119.94281914894, "avg_output_tokens_per_task": 11912.803191489362, "last_run_timestamp": "2026-10-17T08:17:57.529Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.6245795, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.004032424242424243, "avg_duration_per_task_ms": 154426.69696969696, "avg_output_tokens_per_task": 13170.60101010101, "last_run_timestamp": "2026-08-29T18:20:19.448Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.621446169117647, "accuracy_std_dev": 0.008868533772011368, "run_count": 7, "total_questions": 1360, "avg_cost_per_task_usd": 0.017286922794117646, "avg_duration_per_task_ms": 67123.2, "avg_output_tokens_per_task": 57389.13382352941, "last_run_timestamp": "2026-10-31T16:52:47.514Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.6212125000000001, "accuracy_std_dev": 0.011784500000000031, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0023548611111111114, "avg_duration_per_task_ms": 64938.90151515151, "avg_output_tokens_per_task": 2165.318181818182, "last_run_timestamp": "2026-10-01T19:21:21.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.621212, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00552190909090909, "avg_duration_per_task_ms": 174055.64141414143, "avg_output_tokens_per_task": 10415.944444444445, "last_run_timestamp": "2026-10-04T03:09:40.101Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.6191077500000001, "accuracy_std_dev": 0.015215861565731303, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.004869287878787879, "avg_duration_per_task_ms": 286467.2689393939, "avg_output_tokens_per_task": 15960.142676767677, "last_run_timestamp": "2026-10-23T08:08:11.470Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.6173469387755102, "accuracy_std_dev": 0.01386353555194346, "run_count": 3, "total_questions": 588, "avg_cost_per_task_usd": 0.017528746598639455, "avg_duration_per_task_ms": 899129.887755102, "avg_output_tokens_per_task": 18516.460884353743, "last_run_timestamp": "2026-10-03T11:06:38.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Fireworks", "accuracy": 0.61687, "accuracy_std_dev": null, "run_count": 1, "total_questions": 164, "avg_cost_per_task_usd": 0.010753804878048782, "avg_duration_per_task_ms": 340169.43292682926, "avg_output_tokens_per_task": 37200.54878048781, "last_run_timestamp": "2026-08-22T17:46:40.285Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.6136364999999999, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.005658078282828283, "avg_duration_per_task_ms": 686384.2954545454, "avg_output_tokens_per_task": 18940.444444444445, "last_run_timestamp": "2026-08-29T19:06:01.882Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.6125139999999999, "accuracy_std_dev": 0.004677949194536715, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004050520202020202, "avg_duration_per_task_ms": 21444.56228956229, "avg_output_tokens_per_task": 2151.1430976430975, "last_run_timestamp": "2026-10-03T17:03:50.598Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.6123375670628184, "accuracy_std_dev": 0.008273645159312946, "run_count": 3, "total_questions": 589, "avg_cost_per_task_usd": 0.003321100169779287, "avg_duration_per_task_ms": 426798.44821731746, "avg_output_tokens_per_task": 13430.818336162989, "last_run_timestamp": "2026-10-08T22:16:42.056Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.6094280000000001, "accuracy_std_dev": 0.010100999999999971, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.008297136363636364, "avg_duration_per_task_ms": 338157.7095959596, "avg_output_tokens_per_task": 17281.333333333332, "last_run_timestamp": "2026-08-29T19:52:22.642Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.609428, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00778269696969697, "avg_duration_per_task_ms": 302947.1666666667, "avg_output_tokens_per_task": 15369.68686868687, "last_run_timestamp": "2026-10-08T22:57:53.933Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.607744, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.04297312626262627, "avg_duration_per_task_ms": 320571.8686868687, "avg_output_tokens_per_task": 21377.484848484848, "last_run_timestamp": "2026-08-22T10:29:25.897Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6074393716319267, "accuracy_std_dev": 0.07076264023225816, "run_count": 10, "total_questions": 1967, "avg_cost_per_task_usd": 0.012005192679206914, "avg_duration_per_task_ms": 1240549.4712760549, "avg_output_tokens_per_task": 56702.37315709202, "last_run_timestamp": "2026-11-13T01:55:21.496Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.6060606666666667, "accuracy_std_dev": 0.013538129913527765, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0036622003367003365, "avg_duration_per_task_ms": 32739.0202020202, "avg_output_tokens_per_task": 2163.530303030303, "last_run_timestamp": "2026-10-03T17:03:29.436Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Mara", "accuracy": 0.6059481003717473, "accuracy_std_dev": 0.024067533945591992, "run_count": 3, "total_questions": 269, "avg_cost_per_task_usd": 0.03276318587360595, "avg_duration_per_task_ms": 68520.27137546468, "avg_output_tokens_per_task": 7032.501858736059, "last_run_timestamp": "2026-10-31T16:08:33.699Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.6043773333333333, "accuracy_std_dev": 0.004955863351178606, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0035131498316498316, "avg_duration_per_task_ms": 35423.020202020205, "avg_output_tokens_per_task": 2122.909090909091, "last_run_timestamp": "2026-10-03T16:57:23.266Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.6030365565974547, "accuracy_std_dev": 0.012321947684289796, "run_count": 8, "total_questions": 1493, "avg_cost_per_task_usd": 0.008523111185532485, "avg_duration_per_task_ms": 567385.2880107167, "avg_output_tokens_per_task": 14003.128600133958, "last_run_timestamp": "2026-10-08T22:50:36.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.6021326666666665, "accuracy_std_dev": 0.003459479954495376, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.011048464646464645, "avg_duration_per_task_ms": 31195.183501683503, "avg_output_tokens_per_task": 2103.8636363636365, "last_run_timestamp": "2026-10-03T17:43:44.972Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.599327, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.012440297979797978, "avg_duration_per_task_ms": 230792.08080808082, "avg_output_tokens_per_task": 41242.71717171717, "last_run_timestamp": "2026-10-04T03:37:16.537Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.5985173543758967, "accuracy_std_dev": 0.021740745111426138, "run_count": 4, "total_questions": 697, "avg_cost_per_task_usd": 0.002190916786226686, "avg_duration_per_task_ms": 51577.241032998565, "avg_output_tokens_per_task": 2056.6857962697272, "last_run_timestamp": "2026-10-08T23:15:07.689Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "NextBit", "accuracy": 0.5978837142857143, "accuracy_std_dev": 0.01426892240114143, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0030797525252525253, "avg_duration_per_task_ms": 643402.1818181818, "avg_output_tokens_per_task": 13628.22077922078, "last_run_timestamp": "2026-11-01T11:08:51.026Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.5976433333333334, "accuracy_std_dev": 0.0036366213562713575, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.003224691919191919, "avg_duration_per_task_ms": 76581.48821548822, "avg_output_tokens_per_task": 2072.1902356902356, "last_run_timestamp": "2026-10-03T16:59:44.383Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.5965210000000001, "accuracy_std_dev": 0.03456524546419422, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.030341809764309766, "avg_duration_per_task_ms": 1499106.207070707, "avg_output_tokens_per_task": 75742.45791245792, "last_run_timestamp": "2026-10-13T21:28:41.390Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5963805, "accuracy_std_dev": 0.0012624999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01735105808080808, "avg_duration_per_task_ms": 112232.29292929293, "avg_output_tokens_per_task": 43198.833333333336, "last_run_timestamp": "2026-10-01T19:14:48.696Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Novita", "accuracy": 0.595620394160584, "accuracy_std_dev": 0.009567520605183589, "run_count": 7, "total_questions": 1370, "avg_cost_per_task_usd": 0.009091688321167881, "avg_duration_per_task_ms": 479820.2708029197, "avg_output_tokens_per_task": 45264.08613138686, "last_run_timestamp": "2026-10-03T17:04:03.802Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.591751, "accuracy_std_dev": 0.0025249999999999995, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0029208080808080807, "avg_duration_per_task_ms": 521963.51515151514, "avg_output_tokens_per_task": 11688.588383838383, "last_run_timestamp": "2026-10-01T16:32:22.405Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.5897068571428572, "accuracy_std_dev": 0.008664658157111118, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.010310333333333333, "avg_duration_per_task_ms": 10820.968253968254, "avg_output_tokens_per_task": 13292.640692640693, "last_run_timestamp": "2026-10-31T16:21:16.340Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DeepInfra", "accuracy": 0.5880285, "accuracy_std_dev": 0.0387325, "run_count": 2, "total_questions": 142, "avg_cost_per_task_usd": 0.03859155633802817, "avg_duration_per_task_ms": 448942.5140845071, "avg_output_tokens_per_task": 18452.429577464787, "last_run_timestamp": "2026-08-29T19:07:49.385Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SambaNova", "accuracy": 0.5873616193353474, "accuracy_std_dev": 0.048784153240032144, "run_count": 7, "total_questions": 662, "avg_cost_per_task_usd": 0.03112247280966767, "avg_duration_per_task_ms": 62861.41993957704, "avg_output_tokens_per_task": 6675.285498489426, "last_run_timestamp": "2026-10-31T16:41:30.138Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.5833335000000001, "accuracy_std_dev": 0.007575500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.025034136363636363, "avg_duration_per_task_ms": 1106648.0227272727, "avg_output_tokens_per_task": 68373.12626262626, "last_run_timestamp": "2026-08-30T00:32:50.853Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Novita", "accuracy": 0.5824916666666666, "accuracy_std_dev": 0.004956316261992276, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.006444644781144781, "avg_duration_per_task_ms": 663316.5858585859, "avg_output_tokens_per_task": 42705.53198653198, "last_run_timestamp": "2026-10-03T16:12:07.492Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Fireworks", "accuracy": 0.5810130708860759, "accuracy_std_dev": 0.012019500000000016, "run_count": 2, "total_questions": 395, "avg_cost_per_task_usd": 0.01962629113924051, "avg_duration_per_task_ms": 376708.9949367089, "avg_output_tokens_per_task": 97953.7417721519, "last_run_timestamp": "2026-10-31T08:09:25.783Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Google", "accuracy": 0.5774414999999999, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.010289136363636365, "avg_duration_per_task_ms": 331432.74242424243, "avg_output_tokens_per_task": 40883.856060606064, "last_run_timestamp": "2026-10-17T16:06:05.579Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "GMICloud", "accuracy": 0.5709940405679513, "accuracy_std_dev": 0.154429513944712, "run_count": 5, "total_questions": 986, "avg_cost_per_task_usd": 0.0030763367139959434, "avg_duration_per_task_ms": 63553.28093306288, "avg_output_tokens_per_task": 2517.3924949290063, "last_run_timestamp": "2026-11-10T06:14:22.398Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.5614105961383749, "accuracy_std_dev": 0.06778815603610641, "run_count": 7, "total_questions": 1243, "avg_cost_per_task_usd": 0.06303039259855191, "avg_duration_per_task_ms": 382701.67095736123, "avg_output_tokens_per_task": 23894.755430410296, "last_run_timestamp": "2026-10-31T16:21:16.340Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Mancer 2", "accuracy": 0.5608581453590192, "accuracy_std_dev": 0.2662797085149478, "run_count": 6, "total_questions": 1142, "avg_cost_per_task_usd": 0.09908757443082312, "avg_duration_per_task_ms": 1949562.3021015762, "avg_output_tokens_per_task": 42074.91681260946, "last_run_timestamp": "2026-10-31T16:02:18.348Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.56060625, "accuracy_std_dev": 0.009297342103391676, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.017428513888888886, "avg_duration_per_task_ms": 127446.60858585859, "avg_output_tokens_per_task": 43389.579545454544, "last_run_timestamp": "2026-10-01T19:27:01.395Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.5527496666666667, "accuracy_std_dev": 0.01040809115811132, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.004829589225589226, "avg_duration_per_task_ms": 647930.4865319865, "avg_output_tokens_per_task": 17015.548821548822, "last_run_timestamp": "2026-10-08T23:30:03.872Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.5521885000000001, "accuracy_std_dev": 0.005050500000000013, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.01778226515151515, "avg_duration_per_task_ms": 116667.00757575757, "avg_output_tokens_per_task": 44273.378787878784, "last_run_timestamp": "2026-10-01T19:15:36.386Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.5469078490566038, "accuracy_std_dev": 0.022963942121508662, "run_count": 10, "total_questions": 1908, "avg_cost_per_task_usd": 0.005774233752620544, "avg_duration_per_task_ms": 784958.8862683438, "avg_output_tokens_per_task": 14661.248427672956, "last_run_timestamp": "2026-11-08T04:29:32.011Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.5462965, "accuracy_std_dev": 0.08670050000000001, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.006357126262626262, "avg_duration_per_task_ms": 124531.47474747474, "avg_output_tokens_per_task": 5650.492424242424, "last_run_timestamp": "2026-08-29T20:57:58.443Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.538721, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.005473, "avg_duration_per_task_ms": 117831.5808080808, "avg_output_tokens_per_task": 11819.661616161617, "last_run_timestamp": "2026-10-04T03:06:12.419Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.5373176666666667, "accuracy_std_dev": 0.013403444242266851, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0019087592592592591, "avg_duration_per_task_ms": 97924.95622895623, "avg_output_tokens_per_task": 1886.641414141414, "last_run_timestamp": "2026-10-08T22:10:22.349Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.5332481099744245, "accuracy_std_dev": 0.020375000000000032, "run_count": 2, "total_questions": 391, "avg_cost_per_task_usd": 0.004206966751918158, "avg_duration_per_task_ms": 160590.6675191816, "avg_output_tokens_per_task": 4524.092071611253, "last_run_timestamp": "2026-08-29T20:34:10.117Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.5280586666666666, "accuracy_std_dev": 0.004827144451490511, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.01799902861952862, "avg_duration_per_task_ms": 14939.786195286195, "avg_output_tokens_per_task": 1607.080808080808, "last_run_timestamp": "2026-10-02T23:16:52.382Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.5266555, "accuracy_std_dev": 0.023895888382383546, "run_count": 6, "total_questions": 1188, "avg_cost_per_task_usd": 0.01547567760942761, "avg_duration_per_task_ms": 13716.89898989899, "avg_output_tokens_per_task": 1354.1801346801346, "last_run_timestamp": "2026-10-02T23:17:37.980Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.5202019999999999, "accuracy_std_dev": 0.0033669999999999534, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017511833333333334, "avg_duration_per_task_ms": 16177.118686868687, "avg_output_tokens_per_task": 1559.209595959596, "last_run_timestamp": "2026-10-01T16:04:12.703Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "Azure", "accuracy": 0.5168355, "accuracy_std_dev": 0.005050499999999958, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017194939393939394, "avg_duration_per_task_ms": 13199.247474747475, "avg_output_tokens_per_task": 1523.2777777777778, "last_run_timestamp": "2026-10-01T16:28:34.286Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.5165396259541984, "accuracy_std_dev": 0.0053874999999999895, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.015734966921119593, "avg_duration_per_task_ms": 637615.4732824428, "avg_output_tokens_per_task": 40479.653944020356, "last_run_timestamp": "2026-08-30T00:11:29.373Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "Azure", "accuracy": 0.5151519999999999, "accuracy_std_dev": 0.006733999999999962, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018041959595959597, "avg_duration_per_task_ms": 13498.479797979799, "avg_output_tokens_per_task": 1611.5353535353536, "last_run_timestamp": "2026-10-01T16:26:08.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.510943, "accuracy_std_dev": 0.0008420000000000094, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.013169762626262625, "avg_duration_per_task_ms": 12925.411616161617, "avg_output_tokens_per_task": 1121.608585858586, "last_run_timestamp": "2026-10-01T16:03:45.472Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.508418, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.030087782828282828, "avg_duration_per_task_ms": 3926274.5656565656, "avg_output_tokens_per_task": 21326.782828282827, "last_run_timestamp": "2026-08-22T17:37:26.215Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.5077319793814432, "accuracy_std_dev": 0.009572000000000025, "run_count": 2, "total_questions": 388, "avg_cost_per_task_usd": 0.011120894329896908, "avg_duration_per_task_ms": 426473.57216494845, "avg_output_tokens_per_task": 27736.909793814433, "last_run_timestamp": "2026-08-30T00:23:23.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.5050510000000001, "accuracy_std_dev": 0.002749143987983661, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0008273518518518518, "avg_duration_per_task_ms": 18715.93265993266, "avg_output_tokens_per_task": 1873.8602693602693, "last_run_timestamp": "2026-10-02T23:16:50.042Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.505051, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.009755954545454545, "avg_duration_per_task_ms": 700759.0555555555, "avg_output_tokens_per_task": 64762.606060606064, "last_run_timestamp": "2026-10-01T08:24:17.043Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.4995792500000001, "accuracy_std_dev": 0.02025051336849268, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0008581388888888889, "avg_duration_per_task_ms": 12992.723484848484, "avg_output_tokens_per_task": 1853.0088383838383, "last_run_timestamp": "2026-10-01T19:12:29.219Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Groq", "accuracy": 0.49951899999999994, "accuracy_std_dev": 0.012685149528934545, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0023819927849927846, "avg_duration_per_task_ms": 9173.090187590187, "avg_output_tokens_per_task": 2368.319624819625, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "Alibaba", "accuracy": 0.49242400000000003, "accuracy_std_dev": 0.0008419999999999817, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0017634166666666666, "avg_duration_per_task_ms": 30891.38888888889, "avg_output_tokens_per_task": 1731.7373737373737, "last_run_timestamp": "2026-10-01T16:05:56.758Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.49074049999999997, "accuracy_std_dev": 0.009259500000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0008383863636363636, "avg_duration_per_task_ms": 22708.954545454544, "avg_output_tokens_per_task": 1902.2954545454545, "last_run_timestamp": "2026-10-01T19:13:37.127Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DigitalOcean", "accuracy": 0.4902728888888888, "accuracy_std_dev": 0.1180581110088911, "run_count": 9, "total_questions": 1782, "avg_cost_per_task_usd": 0.003499423120089787, "avg_duration_per_task_ms": 205174.32940516275, "avg_output_tokens_per_task": 8349.423120089787, "last_run_timestamp": "2026-11-13T10:41:52.262Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.48436714285714283, "accuracy_std_dev": 0.012361779684964115, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0026505512265512263, "avg_duration_per_task_ms": 49856.58441558442, "avg_output_tokens_per_task": 2925.426406926407, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.4836306971153846, "accuracy_std_dev": 0.010925716722180446, "run_count": 15, "total_questions": 2912, "avg_cost_per_task_usd": 0.0013556023351648351, "avg_duration_per_task_ms": 59721.06524725275, "avg_output_tokens_per_task": 2363.5096153846152, "last_run_timestamp": "2026-10-31T16:01:53.654Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "SambaNova", "accuracy": 0.48172171428571425, "accuracy_std_dev": 0.01107352294844762, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0023519292929292932, "avg_duration_per_task_ms": 16333.764790764792, "avg_output_tokens_per_task": 2210.2922077922076, "last_run_timestamp": "2026-10-31T16:05:14.701Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.4787955114503817, "accuracy_std_dev": 0.007795999999999997, "run_count": 2, "total_questions": 393, "avg_cost_per_task_usd": 0.011332788804071247, "avg_duration_per_task_ms": 362429.45801526716, "avg_output_tokens_per_task": 28246.409669211196, "last_run_timestamp": "2026-08-30T01:15:14.669Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Parasail", "accuracy": 0.47715228571428564, "accuracy_std_dev": 0.014837295958055795, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0013270916305916304, "avg_duration_per_task_ms": 44547.831168831166, "avg_output_tokens_per_task": 2383.0901875901877, "last_run_timestamp": "2026-10-31T16:36:11.519Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.4765793529411765, "accuracy_std_dev": 0.006944000000000006, "run_count": 2, "total_questions": 306, "avg_cost_per_task_usd": 0.0011232875816993464, "avg_duration_per_task_ms": 126677.51307189543, "avg_output_tokens_per_task": 2069.3627450980393, "last_run_timestamp": "2026-10-01T16:04:45.916Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.4754687142857143, "accuracy_std_dev": 0.010214854935188212, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0013394170274170275, "avg_duration_per_task_ms": 85516.39105339105, "avg_output_tokens_per_task": 2360.8838383838383, "last_run_timestamp": "2026-10-31T16:23:41.139Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.475084, "accuracy_std_dev": 0.006683163113376778, "run_count": 5, "total_questions": 990, "avg_cost_per_task_usd": 0.0031415444444444444, "avg_duration_per_task_ms": 73694.87777777777, "avg_output_tokens_per_task": 2214.6858585858586, "last_run_timestamp": "2026-10-17T16:05:55.476Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.47234228571428577, "accuracy_std_dev": 0.009696421014466937, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0010906248196248197, "avg_duration_per_task_ms": 63821.896103896106, "avg_output_tokens_per_task": 2454.1998556998556, "last_run_timestamp": "2026-10-31T16:31:54.106Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.469697, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.006110727272727273, "avg_duration_per_task_ms": 42879.368686868685, "avg_output_tokens_per_task": 2318.919191919192, "last_run_timestamp": "2026-10-04T03:02:23.034Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Cloudflare", "accuracy": 0.46955114615384613, "accuracy_std_dev": 0.06298705219874558, "run_count": 8, "total_questions": 1560, "avg_cost_per_task_usd": 0.0048902935897435895, "avg_duration_per_task_ms": 46682.213461538464, "avg_output_tokens_per_task": 2063.5615384615385, "last_run_timestamp": "2026-10-31T16:05:36.136Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.468855, "accuracy_std_dev": 0.012625999999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.001083489898989899, "avg_duration_per_task_ms": 264261.16161616164, "avg_output_tokens_per_task": 2445.7929292929293, "last_run_timestamp": "2026-08-29T20:09:29.346Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Crusoe", "accuracy": 0.468855, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.002208848484848485, "avg_duration_per_task_ms": 36376.982323232325, "avg_output_tokens_per_task": 2683.0429292929293, "last_run_timestamp": "2026-08-29T16:31:42.504Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.46412025, "accuracy_std_dev": 0.011095032793439591, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0007977228535353536, "avg_duration_per_task_ms": 144411.35542929292, "avg_output_tokens_per_task": 2242.354797979798, "last_run_timestamp": "2026-10-31T16:02:14.361Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.461279, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0011232171717171719, "avg_duration_per_task_ms": 136788.7626262626, "avg_output_tokens_per_task": 2036.2272727272727, "last_run_timestamp": "2026-10-01T16:27:00.375Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.45959600000000006, "accuracy_std_dev": 0.01346799999999998, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010278964646464647, "avg_duration_per_task_ms": 96379.95959595959, "avg_output_tokens_per_task": 1848.388888888889, "last_run_timestamp": "2026-10-01T16:47:11.191Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Parasail", "accuracy": 0.457071, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0017076717171717172, "avg_duration_per_task_ms": 27081.62626262626, "avg_output_tokens_per_task": 1337.1363636363637, "last_run_timestamp": "2026-09-01T13:03:42.840Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "Nebius", "accuracy": 0.442761, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0013494898989898992, "avg_duration_per_task_ms": 69228.69696969698, "avg_output_tokens_per_task": 1528.5050505050506, "last_run_timestamp": "2026-09-01T12:41:25.955Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.43602700000000005, "accuracy_std_dev": 0.006734000000000018, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0010042146464646464, "avg_duration_per_task_ms": 21558.090909090908, "avg_output_tokens_per_task": 1481.030303030303, "last_run_timestamp": "2026-10-01T19:16:07.087Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "Azure", "accuracy": 0.43139700000000003, "accuracy_std_dev": 0.012951914704011906, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0010746982323232325, "avg_duration_per_task_ms": 24227.715909090908, "avg_output_tokens_per_task": 1509.2487373737374, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen2.5-vl-72b-instruct", "model_slug": "qwen/qwen2.5-vl-72b-instruct", "model_name": "Qwen: Qwen2.5 VL 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.429293, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0015700353535353536, "avg_duration_per_task_ms": 59122.85858585859, "avg_output_tokens_per_task": 1664.8737373737374, "last_run_timestamp": "2026-09-01T13:21:35.512Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.4276093333333334, "accuracy_std_dev": 0.00629925838668507, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0010292508417508418, "avg_duration_per_task_ms": 16725.542087542086, "avg_output_tokens_per_task": 1523.1902356902358, "last_run_timestamp": "2026-10-02T23:19:57.879Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.405724, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0003259848484848485, "avg_duration_per_task_ms": 29056.348484848484, "avg_output_tokens_per_task": 1917.1515151515152, "last_run_timestamp": "2026-10-08T22:01:16.852Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy-mt2-30b-a3b-20260521", "model_slug": "tencent/hy-mt2-30b-a3b", "model_name": "Tencent: Hy-MT2-30B-A3B", "provider_name": "Reka", "accuracy": 0.388889, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00023147979797979797, "avg_duration_per_task_ms": 8604.353535353535, "avg_output_tokens_per_task": 710.540404040404, "last_run_timestamp": "2026-09-24T06:00:03.613Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.3855215, "accuracy_std_dev": 0.011293261320362695, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0001567070707070707, "avg_duration_per_task_ms": 7111.70202020202, "avg_output_tokens_per_task": 932.0808080808081, "last_run_timestamp": "2026-10-01T19:39:32.831Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "NextBit", "accuracy": 0.38047125000000004, "accuracy_std_dev": 0.012081350470353048, "run_count": 4, "total_questions": 792, "avg_cost_per_task_usd": 0.0003843409090909091, "avg_duration_per_task_ms": 52179.61616161616, "avg_output_tokens_per_task": 2261.531565656566, "last_run_timestamp": "2026-11-01T11:50:48.836Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Io Net", "accuracy": 0.3787201875, "accuracy_std_dev": 0.3723545, "run_count": 2, "total_questions": 224, "avg_cost_per_task_usd": 0.0059331517857142855, "avg_duration_per_task_ms": 145883.42857142858, "avg_output_tokens_per_task": 5127.165178571428, "last_run_timestamp": "2026-11-11T05:27:07.237Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.3633958571428571, "accuracy_std_dev": 0.08617342972074823, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.008518821067821068, "avg_duration_per_task_ms": 483581.14285714284, "avg_output_tokens_per_task": 21062.341991341993, "last_run_timestamp": "2026-10-31T16:45:48.945Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.346801, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.008616373737373737, "avg_duration_per_task_ms": 1179181.5303030303, "avg_output_tokens_per_task": 32995.232323232325, "last_run_timestamp": "2026-10-03T00:19:24.096Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Groq", "accuracy": 0.2912128823529411, "accuracy_std_dev": 0.01689256477789889, "run_count": 7, "total_questions": 1377, "avg_cost_per_task_usd": 0.0029356129266521424, "avg_duration_per_task_ms": 41493.66521423384, "avg_output_tokens_per_task": 36404.61365286855, "last_run_timestamp": "2026-10-31T16:04:29.122Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "DeepInfra", "accuracy": 0.2873176666666667, "accuracy_std_dev": 0.0007938452130121062, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00006128619528619528, "avg_duration_per_task_ms": 10231.205387205388, "avg_output_tokens_per_task": 580.9023569023569, "last_run_timestamp": "2026-10-08T22:00:27.722Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "model_slug": "meta-llama/llama-3.2-3b-instruct", "model_name": "Meta: Llama 3.2 3B Instruct", "provider_name": "Cloudflare", "accuracy": 0.286195, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0005941969696969697, "avg_duration_per_task_ms": 14998.156565656565, "avg_output_tokens_per_task": 1642.1363636363637, "last_run_timestamp": "2026-10-13T18:56:05.001Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.2770562857142857, "accuracy_std_dev": 0.021947671114424378, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.0010397142857142857, "avg_duration_per_task_ms": 152294.55627705628, "avg_output_tokens_per_task": 15530.19769119769, "last_run_timestamp": "2026-10-31T16:08:31.064Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "auto-routing", "accuracy": 0.27665533333333336, "accuracy_std_dev": 0.0007938452130121063, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.00006201683501683503, "avg_duration_per_task_ms": 9496.395622895623, "avg_output_tokens_per_task": 580.2053872053872, "last_run_timestamp": "2026-10-01T20:43:57.173Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Novita", "accuracy": 0.2745902896174863, "accuracy_std_dev": 0.009941000000000005, "run_count": 2, "total_questions": 366, "avg_cost_per_task_usd": 0.0000641912568306011, "avg_duration_per_task_ms": 8433.43169398907, "avg_output_tokens_per_task": 532.0136612021857, "last_run_timestamp": "2026-10-01T16:01:04.877Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "sao10k/l3-lunaris-8b", "model_slug": "sao10k/l3-lunaris-8b", "model_name": "Sao10K: Llama 3 8B Lunaris", "provider_name": "Parasail", "accuracy": 0.27161569453924916, "accuracy_std_dev": 0.020450238993441835, "run_count": 3, "total_questions": 586, "avg_cost_per_task_usd": 0.000055156996587030725, "avg_duration_per_task_ms": 10158.571672354949, "avg_output_tokens_per_task": 550.3515358361775, "last_run_timestamp": "2026-10-01T20:47:49.128Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-4b-it", "model_slug": "google/gemma-3-4b-it", "model_name": "Google: Gemma 3 4B", "provider_name": "Parasail", "accuracy": 0.271044, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.00011306565656565657, "avg_duration_per_task_ms": 10846.11616161616, "avg_output_tokens_per_task": 1804.4646464646464, "last_run_timestamp": "2026-10-15T03:00:03.049Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "CoreWeave", "accuracy": 0.26953032780708364, "accuracy_std_dev": 0.014445882398999181, "run_count": 7, "total_questions": 1327, "avg_cost_per_task_usd": 0.011133856819894499, "avg_duration_per_task_ms": 459953.84174830443, "avg_output_tokens_per_task": 49838.27731725697, "last_run_timestamp": "2026-10-31T16:04:29.124Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "DeepInfra", "accuracy": 0.26473062500000005, "accuracy_std_dev": 0.010705448612009439, "run_count": 8, "total_questions": 1584, "avg_cost_per_task_usd": 0.0003948459595959595, "avg_duration_per_task_ms": 362944.5946969697, "avg_output_tokens_per_task": 9468.12436868687, "last_run_timestamp": "2026-10-31T16:08:31.068Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Novita", "accuracy": 0.25901885714285716, "accuracy_std_dev": 0.013953091766226396, "run_count": 7, "total_questions": 1386, "avg_cost_per_task_usd": 0.00042360028860028863, "avg_duration_per_task_ms": 73749.8658008658, "avg_output_tokens_per_task": 8148.3650793650795, "last_run_timestamp": "2026-10-31T16:55:07.715Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-nano-banana-2.1-20261006", "model_slug": "google/gemini-nano-banana-2.1", "model_name": "Google: Nano Banana 2.1", "provider_name": "Google", "accuracy": 0.213805, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.17523430303030305, "avg_duration_per_task_ms": 78031.78787878787, "avg_output_tokens_per_task": 14613.282828282829, "last_run_timestamp": "2026-11-12T21:01:32.027Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.17174282233502539, "accuracy_std_dev": 0.01788397279378631, "run_count": 3, "total_questions": 591, "avg_cost_per_task_usd": 0.019897788494077834, "avg_duration_per_task_ms": 1784246.4196277496, "avg_output_tokens_per_task": 110294.06598984772, "last_run_timestamp": "2026-10-08T23:13:58.126Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google AI Studio", "accuracy": 0.1161615, "accuracy_std_dev": 0.0016835000000000044, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017850737373737373, "avg_duration_per_task_ms": 46853.982323232325, "avg_output_tokens_per_task": 11776.444444444445, "last_run_timestamp": "2026-10-01T16:00:58.389Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "Novita", "accuracy": 0.11531999999999999, "accuracy_std_dev": 0.027778000000000004, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.0013887626262626262, "avg_duration_per_task_ms": 56176.38383838384, "avg_output_tokens_per_task": 1211.2525252525252, "last_run_timestamp": "2026-08-29T18:43:09.114Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "Google", "accuracy": 0.112795, "accuracy_std_dev": 0.006733999999999997, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.018038613636363634, "avg_duration_per_task_ms": 21125.79292929293, "avg_output_tokens_per_task": 11795.222222222223, "last_run_timestamp": "2026-10-01T16:00:58.364Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-flash-lite-image-20260630", "model_slug": "google/gemini-3.1-flash-lite-image", "model_name": "Google: Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)", "provider_name": "auto-routing", "accuracy": 0.102694, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 396, "avg_cost_per_task_usd": 0.017984406565656568, "avg_duration_per_task_ms": 21554.590909090908, "avg_output_tokens_per_task": 11812.823232323231, "last_run_timestamp": "2026-10-01T16:02:13.023Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "Cloudflare", "accuracy": 0.09483766666666665, "accuracy_std_dev": 0.1317391787847825, "run_count": 3, "total_questions": 594, "avg_cost_per_task_usd": 0.0006702474747474747, "avg_duration_per_task_ms": 243969.58922558924, "avg_output_tokens_per_task": 2175.9141414141413, "last_run_timestamp": "2026-10-24T16:03:19.773Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.0033670000000000006, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.0034259191919191923, "avg_duration_per_task_ms": 169432.20707070708, "avg_output_tokens_per_task": 3225.4444444444443, "last_run_timestamp": "2026-08-22T17:29:06.480Z" }, { "benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.001684, "accuracy_std_dev": null, "run_count": 1, "total_questions": 198, "avg_cost_per_task_usd": 0.003385919191919192, "avg_duration_per_task_ms": 136907.34343434343, "avg_output_tokens_per_task": 3185.4444444444443, "last_run_timestamp": "2026-10-31T16:04:43.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "OpenInference", "accuracy": 0.816667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05960144, "avg_duration_per_task_ms": 162346.76, "avg_output_tokens_per_task": 42454.8, "last_run_timestamp": "2026-10-17T19:19:25.280Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.8000000909090909, "accuracy_std_dev": 0.020695063581156234, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.8742483381818181, "avg_duration_per_task_ms": 136193.65272727274, "avg_output_tokens_per_task": 5706.489090909091, "last_run_timestamp": "2026-10-31T12:24:55.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "auto-routing", "accuracy": 0.8, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.39313749000000003, "avg_duration_per_task_ms": 477750.16, "avg_output_tokens_per_task": 34082.08, "last_run_timestamp": "2026-10-01T16:00:45.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DigitalOcean", "accuracy": 0.7995237142857143, "accuracy_std_dev": 0.010754053185583914, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.14178823428571427, "avg_duration_per_task_ms": 2712952.931428571, "avg_output_tokens_per_task": 28444.37714285714, "last_run_timestamp": "2026-10-31T20:31:41.691Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "BaseTen", "accuracy": 0.7981221830985916, "accuracy_std_dev": 0.05639978873679898, "run_count": 3, "total_questions": 142, "avg_cost_per_task_usd": 0.007624964788732395, "avg_duration_per_task_ms": 30939.802816901407, "avg_output_tokens_per_task": 8761.732394366198, "last_run_timestamp": "2026-10-31T11:56:20.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Azure", "accuracy": 0.798, "accuracy_std_dev": 0.022910589499181372, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.8759728400000001, "avg_duration_per_task_ms": 127096.316, "avg_output_tokens_per_task": 5702.834, "last_run_timestamp": "2026-10-31T11:20:38.480Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Phala", "accuracy": 0.7978929137931035, "accuracy_std_dev": 0.03017695806811711, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.20533228735632184, "avg_duration_per_task_ms": 772776.3793103448, "avg_output_tokens_per_task": 35305.502873563215, "last_run_timestamp": "2026-10-31T19:01:09.612Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Alibaba", "accuracy": 0.7936961919770773, "accuracy_std_dev": 0.02409553422592355, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.18211524641833812, "avg_duration_per_task_ms": 451041.38395415473, "avg_output_tokens_per_task": 30422.72206303725, "last_run_timestamp": "2026-10-31T19:12:41.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "Novita", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00828808, "avg_duration_per_task_ms": 196026.56, "avg_output_tokens_per_task": 16303, "last_run_timestamp": "2026-10-14T19:21:35.384Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.1-flash-20261002", "model_slug": "inclusionai/ling-3.1-flash", "model_name": "inclusionAI: Ling 3.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.793333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03146312, "avg_duration_per_task_ms": 636210.72, "avg_output_tokens_per_task": 16745.44, "last_run_timestamp": "2026-10-14T19:17:59.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Anthropic", "accuracy": 0.7926667, "accuracy_std_dev": 0.017751370527652206, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.81432549, "avg_duration_per_task_ms": 121567.298, "avg_output_tokens_per_task": 5714.138, "last_run_timestamp": "2026-10-31T11:18:52.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Alibaba", "accuracy": 0.7924998750000001, "accuracy_std_dev": 0.02591377395535768, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.082230975, "avg_duration_per_task_ms": 213864.49, "avg_output_tokens_per_task": 26801.935, "last_run_timestamp": "2026-10-31T19:22:16.313Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "auto-routing", "accuracy": 0.79166675, "accuracy_std_dev": 0.012801703448662584, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.71616581, "avg_duration_per_task_ms": 111830.815, "avg_output_tokens_per_task": 6900.955, "last_run_timestamp": "2026-10-01T08:03:01.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "auto-routing", "accuracy": 0.7914285714285715, "accuracy_std_dev": 0.01807021076529105, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.20560172000000002, "avg_duration_per_task_ms": 767128.9885714286, "avg_output_tokens_per_task": 32379.805714285714, "last_run_timestamp": "2026-10-31T19:47:27.263Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Anthropic", "accuracy": 0.7906666, "accuracy_std_dev": 0.009977824303925176, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.814870324, "avg_duration_per_task_ms": 182126.652, "avg_output_tokens_per_task": 7046.908, "last_run_timestamp": "2026-10-01T08:05:22.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "auto-routing", "accuracy": 0.79, "accuracy_std_dev": 0.026204223579415577, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.857535498, "avg_duration_per_task_ms": 115142.934, "avg_output_tokens_per_task": 5717.478, "last_run_timestamp": "2026-10-31T11:38:36.784Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Chutes", "accuracy": 0.789795887755102, "accuracy_std_dev": 0.02205636874011677, "run_count": 10, "total_questions": 490, "avg_cost_per_task_usd": 0.07972918775510204, "avg_duration_per_task_ms": 1143421.8387755102, "avg_output_tokens_per_task": 25766.961224489794, "last_run_timestamp": "2026-10-31T20:22:41.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "model_slug": "anthropic/claude-fable-5", "model_name": "Anthropic: Claude Fable 5", "provider_name": "Google", "accuracy": 0.7892064761904761, "accuracy_std_dev": 0.022488265074455106, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.8947322371428572, "avg_duration_per_task_ms": 91208.88952380953, "avg_output_tokens_per_task": 5717.874285714286, "last_run_timestamp": "2026-11-11T21:16:12.438Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Xiaomi", "accuracy": 0.788889, "accuracy_std_dev": 0.015234869040023505, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.057788776666666666, "avg_duration_per_task_ms": 893038.1233333333, "avg_output_tokens_per_task": 40447.416666666664, "last_run_timestamp": "2026-10-31T13:13:33.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Wafer", "accuracy": 0.7882883714036617, "accuracy_std_dev": 0.015477690995371508, "run_count": 23, "total_questions": 1147, "avg_cost_per_task_usd": 0.0821072258064516, "avg_duration_per_task_ms": 149668.3635571055, "avg_output_tokens_per_task": 12209.048823016565, "last_run_timestamp": "2026-10-31T12:32:50.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "auto-routing", "accuracy": 0.788, "accuracy_std_dev": 0.02246951515275753, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.15925035199999998, "avg_duration_per_task_ms": 71120.884, "avg_output_tokens_per_task": 6245.168, "last_run_timestamp": "2026-10-31T11:10:36.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-4-argon-20261013", "model_slug": "google/gemini-4-argon", "model_name": "Google: Gemini 4 Argon", "provider_name": "Google", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.49828548, "avg_duration_per_task_ms": 348417.68, "avg_output_tokens_per_task": 21971.04, "last_run_timestamp": "2026-10-14T15:16:05.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "DigitalOcean", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.37777382000000004, "avg_duration_per_task_ms": 196216.66, "avg_output_tokens_per_task": 12248.68, "last_run_timestamp": "2026-11-07T03:08:27.754Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Fireworks", "accuracy": 0.786667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.20422622, "avg_duration_per_task_ms": 176471.64, "avg_output_tokens_per_task": 10146.04, "last_run_timestamp": "2026-11-05T15:41:01.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DeepInfra", "accuracy": 0.786127239884393, "accuracy_std_dev": 0.023660230134130145, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.09665253179190753, "avg_duration_per_task_ms": 380894.4450867052, "avg_output_tokens_per_task": 16737.754335260117, "last_run_timestamp": "2026-10-31T12:56:48.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "BaseTen", "accuracy": 0.7859650631578948, "accuracy_std_dev": 0.03135199999999999, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.09807105263157893, "avg_duration_per_task_ms": 69953.93684210526, "avg_output_tokens_per_task": 8018.557894736842, "last_run_timestamp": "2026-08-29T17:15:26.910Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Reka", "accuracy": 0.7858794166666667, "accuracy_std_dev": 0.021093449409710124, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.07943493055555556, "avg_duration_per_task_ms": 334937.25694444444, "avg_output_tokens_per_task": 18674.54861111111, "last_run_timestamp": "2026-10-31T19:43:30.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Parasail", "accuracy": 0.7845118821548821, "accuracy_std_dev": 0.03356639684143456, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.19079432323232323, "avg_duration_per_task_ms": 638899.1245791246, "avg_output_tokens_per_task": 30741.198653198655, "last_run_timestamp": "2026-10-31T19:27:20.607Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Together", "accuracy": 0.7844443333333334, "accuracy_std_dev": 0.01632984089872824, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0785912688888889, "avg_duration_per_task_ms": 47362.52222222222, "avg_output_tokens_per_task": 4720.888888888889, "last_run_timestamp": "2026-10-31T11:58:26.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Azure", "accuracy": 0.7833333333333333, "accuracy_std_dev": 0.011706202852429232, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.1729425, "avg_duration_per_task_ms": 66541.35, "avg_output_tokens_per_task": 6203.445, "last_run_timestamp": "2026-10-31T12:38:55.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.7833330000000001, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07443635, "avg_duration_per_task_ms": 98397.15, "avg_output_tokens_per_task": 21518.67, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Novita", "accuracy": 0.7825, "accuracy_std_dev": 0.023790007240436065, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.08953825, "avg_duration_per_task_ms": 274829.5175, "avg_output_tokens_per_task": 19992.6725, "last_run_timestamp": "2026-10-31T12:57:09.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Novita", "accuracy": 0.7822223333333334, "accuracy_std_dev": 0.01571327975171178, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.011317603333333334, "avg_duration_per_task_ms": 599501.3033333333, "avg_output_tokens_per_task": 21464.33, "last_run_timestamp": "2026-10-31T14:08:11.994Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "auto-routing", "accuracy": 0.782, "accuracy_std_dev": 0.01521706644527781, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.470768748, "avg_duration_per_task_ms": 115538.554, "avg_output_tokens_per_task": 6965.978, "last_run_timestamp": "2026-10-31T12:17:12.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Novita", "accuracy": 0.7818181878787879, "accuracy_std_dev": 0.019584458716288278, "run_count": 10, "total_questions": 495, "avg_cost_per_task_usd": 0.19799780808080808, "avg_duration_per_task_ms": 511549.30303030304, "avg_output_tokens_per_task": 27725.535353535353, "last_run_timestamp": "2026-10-31T18:53:29.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "Novita", "accuracy": 0.781333, "accuracy_std_dev": 0.027129319932501044, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.0587014, "avg_duration_per_task_ms": 871009.496, "avg_output_tokens_per_task": 40629.384, "last_run_timestamp": "2026-10-31T13:41:45.309Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Fireworks", "accuracy": 0.7811976407035177, "accuracy_std_dev": 0.02110393286871771, "run_count": 16, "total_questions": 796, "avg_cost_per_task_usd": 0.09273568467336683, "avg_duration_per_task_ms": 66515.5364321608, "avg_output_tokens_per_task": 4679.810301507538, "last_run_timestamp": "2026-10-31T13:12:01.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Crusoe", "accuracy": 0.7808333749999999, "accuracy_std_dev": 0.010769114145758466, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0823930725, "avg_duration_per_task_ms": 41435.8875, "avg_output_tokens_per_task": 4744.8375, "last_run_timestamp": "2026-10-31T12:40:46.044Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7800001, "accuracy_std_dev": 0.02108170349592273, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.174296886, "avg_duration_per_task_ms": 72081.318, "avg_output_tokens_per_task": 6231.684, "last_run_timestamp": "2026-10-31T12:10:32.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Google", "accuracy": 0.78, "accuracy_std_dev": 0.021081938910409114, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.18010515333333332, "avg_duration_per_task_ms": 64530.58666666667, "avg_output_tokens_per_task": 6325.823333333334, "last_run_timestamp": "2026-10-31T12:07:29.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Together", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.08351372, "avg_duration_per_task_ms": 190050, "avg_output_tokens_per_task": 8493.48, "last_run_timestamp": "2026-08-22T08:39:22.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "PrimeIntellect", "accuracy": 0.78, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0042799399999999994, "avg_duration_per_task_ms": 627315.22, "avg_output_tokens_per_task": 12675.1, "last_run_timestamp": "2026-10-31T21:10:18.835Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "auto-routing", "accuracy": 0.7786668, "accuracy_std_dev": 0.01484714413481597, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.014840148000000001, "avg_duration_per_task_ms": 379677.016, "avg_output_tokens_per_task": 21736.348, "last_run_timestamp": "2026-10-31T12:28:58.923Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Near AI", "accuracy": 0.7786666, "accuracy_std_dev": 0.010666575000439441, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.031286299999999996, "avg_duration_per_task_ms": 221994.964, "avg_output_tokens_per_task": 12740.492, "last_run_timestamp": "2026-10-31T12:48:16.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "BaseTen", "accuracy": 0.7784256831875607, "accuracy_std_dev": 0.029691712716069543, "run_count": 21, "total_questions": 1029, "avg_cost_per_task_usd": 0.07680297376093294, "avg_duration_per_task_ms": 74568.79203109816, "avg_output_tokens_per_task": 4283.289601554908, "last_run_timestamp": "2026-10-31T13:24:01.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "ModelRun", "accuracy": 0.77833325, "accuracy_std_dev": 0.005527984369324845, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.12165100000000001, "avg_duration_per_task_ms": 123453.655, "avg_output_tokens_per_task": 4597.39, "last_run_timestamp": "2026-10-28T05:13:02.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Z.AI", "accuracy": 0.7779999000000001, "accuracy_std_dev": 0.020011142453393298, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.081736628, "avg_duration_per_task_ms": 135707.96, "avg_output_tokens_per_task": 4671.7, "last_run_timestamp": "2026-10-31T12:38:49.692Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "AtlasCloud", "accuracy": 0.777778, "accuracy_std_dev": 0.01257094435593444, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.016507473333333335, "avg_duration_per_task_ms": 92510.76666666666, "avg_output_tokens_per_task": 10861.266666666666, "last_run_timestamp": "2026-10-31T11:43:38.949Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Nebius", "accuracy": 0.777459306590258, "accuracy_std_dev": 0.02712463143584833, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.17213766475644698, "avg_duration_per_task_ms": 55665.0888252149, "avg_output_tokens_per_task": 4643.535816618911, "last_run_timestamp": "2026-10-31T12:26:01.294Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Anthropic", "accuracy": 0.7766666666666666, "accuracy_std_dev": 0.014782246611692315, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17363205, "avg_duration_per_task_ms": 73619.57333333333, "avg_output_tokens_per_task": 6340.086666666667, "last_run_timestamp": "2026-10-31T12:24:45.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DigitalOcean", "accuracy": 0.7766665833333334, "accuracy_std_dev": 0.0379571517689318, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.10135336166666668, "avg_duration_per_task_ms": 172908.82, "avg_output_tokens_per_task": 6639.301666666666, "last_run_timestamp": "2026-10-31T11:53:06.149Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "CoreWeave", "accuracy": 0.7766665, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.15718574, "avg_duration_per_task_ms": 517790.44, "avg_output_tokens_per_task": 27035.53, "last_run_timestamp": "2026-08-29T08:26:04.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "InferenceNet", "accuracy": 0.7766665, "accuracy_std_dev": 0.016666499999999973, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.017346550000000002, "avg_duration_per_task_ms": 335231.13, "avg_output_tokens_per_task": 20113.88, "last_run_timestamp": "2026-10-03T08:17:48.643Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Alibaba", "accuracy": 0.7764815, "accuracy_std_dev": 0.02583109973193382, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.13240519666666667, "avg_duration_per_task_ms": 319498.1977777778, "avg_output_tokens_per_task": 19319.296666666665, "last_run_timestamp": "2026-10-31T13:25:38.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "CoreWeave", "accuracy": 0.7760940000000001, "accuracy_std_dev": 0.020936325423770052, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.06970348989898989, "avg_duration_per_task_ms": 95095.22222222222, "avg_output_tokens_per_task": 4292.19696969697, "last_run_timestamp": "2026-10-31T12:44:06.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "DeepInfra", "accuracy": 0.7760001999999999, "accuracy_std_dev": 0.028472283599318148, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.06820797999999999, "avg_duration_per_task_ms": 658315.768, "avg_output_tokens_per_task": 13133.756, "last_run_timestamp": "2026-10-31T13:30:16.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Anthropic", "accuracy": 0.7759999000000001, "accuracy_std_dev": 0.01890341768278953, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.43835202, "avg_duration_per_task_ms": 86044.028, "avg_output_tokens_per_task": 5677.174, "last_run_timestamp": "2026-10-31T12:26:17.134Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "AtlasCloud", "accuracy": 0.7758335, "accuracy_std_dev": 0.016308646311082982, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.15398643500000003, "avg_duration_per_task_ms": 330861.565, "avg_output_tokens_per_task": 19948.0325, "last_run_timestamp": "2026-10-31T13:41:05.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cloudflare", "accuracy": 0.7757695461847389, "accuracy_std_dev": 0.016169521055368324, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.08735203012048193, "avg_duration_per_task_ms": 139136.04417670684, "avg_output_tokens_per_task": 4443.712851405622, "last_run_timestamp": "2026-10-31T12:29:23.419Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "StreamLake", "accuracy": 0.7757574545454545, "accuracy_std_dev": 0.02074823590031709, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.08103076363636365, "avg_duration_per_task_ms": 336159.44, "avg_output_tokens_per_task": 20208.643636363635, "last_run_timestamp": "2026-10-31T12:24:53.587Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "DeepInfra", "accuracy": 0.7757142857142857, "accuracy_std_dev": 0.033557166846673904, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03545617428571429, "avg_duration_per_task_ms": 502388.40285714285, "avg_output_tokens_per_task": 13844.954285714286, "last_run_timestamp": "2026-10-08T22:42:47.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "DeepInfra", "accuracy": 0.7753625384615385, "accuracy_std_dev": 0.017323958686038136, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.06710363879598663, "avg_duration_per_task_ms": 1125092.9698996656, "avg_output_tokens_per_task": 36573.67558528428, "last_run_timestamp": "2026-10-24T13:26:35.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "auto-routing", "accuracy": 0.7753334, "accuracy_std_dev": 0.024046292392799363, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.16927985399999998, "avg_duration_per_task_ms": 561845.348, "avg_output_tokens_per_task": 35586.574, "last_run_timestamp": "2026-10-31T19:08:11.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "auto-routing", "accuracy": 0.7753334, "accuracy_std_dev": 0.026339740994170767, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.065886822, "avg_duration_per_task_ms": 257998.344, "avg_output_tokens_per_task": 14164.592, "last_run_timestamp": "2026-10-31T12:39:11.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AtlasCloud", "accuracy": 0.775, "accuracy_std_dev": 0.01787294665828779, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09322250500000001, "avg_duration_per_task_ms": 167383.47, "avg_output_tokens_per_task": 5123.47, "last_run_timestamp": "2026-10-31T12:32:51.992Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Amazon Bedrock", "accuracy": 0.774712655172414, "accuracy_std_dev": 0.021326058387893743, "run_count": 29, "total_questions": 1450, "avg_cost_per_task_usd": 0.5006132268965519, "avg_duration_per_task_ms": 112500.0448275862, "avg_output_tokens_per_task": 7031.657931034482, "last_run_timestamp": "2026-10-31T11:36:20.802Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Claude Platform on AWS", "accuracy": 0.7745454545454545, "accuracy_std_dev": 0.02425759601131023, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.48932630545454553, "avg_duration_per_task_ms": 124123.69272727273, "avg_output_tokens_per_task": 7977.827272727272, "last_run_timestamp": "2026-10-31T11:58:26.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "DeepInfra", "accuracy": 0.7742857142857142, "accuracy_std_dev": 0.026945723560596418, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21623159142857146, "avg_duration_per_task_ms": 562306.5571428571, "avg_output_tokens_per_task": 12602.945714285714, "last_run_timestamp": "2026-10-10T11:06:43.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Friendli", "accuracy": 0.7739394545454545, "accuracy_std_dev": 0.019165412105066386, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06798592909090909, "avg_duration_per_task_ms": 45561.432727272724, "avg_output_tokens_per_task": 4731.818181818182, "last_run_timestamp": "2026-10-31T12:41:41.331Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Reka", "accuracy": 0.7733335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11407636, "avg_duration_per_task_ms": 360681.75, "avg_output_tokens_per_task": 19997.86, "last_run_timestamp": "2026-10-01T22:40:52.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Io Net", "accuracy": 0.7733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10867763, "avg_duration_per_task_ms": 521628.03, "avg_output_tokens_per_task": 12092.47, "last_run_timestamp": "2026-09-19T08:11:59.798Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Tenstorrent", "accuracy": 0.7733335, "accuracy_std_dev": 0.03711848895429339, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0407917975, "avg_duration_per_task_ms": 116084.2975, "avg_output_tokens_per_task": 4762.3475, "last_run_timestamp": "2026-11-08T04:06:44.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DekaLLM", "accuracy": 0.7733334, "accuracy_std_dev": 0.02108173511929225, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.05645720600000001, "avg_duration_per_task_ms": 156313.234, "avg_output_tokens_per_task": 16426.692, "last_run_timestamp": "2026-10-31T11:34:45.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DeepInfra", "accuracy": 0.773333375, "accuracy_std_dev": 0.02236071704428047, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04385152750000001, "avg_duration_per_task_ms": 107910.09, "avg_output_tokens_per_task": 4940.6725, "last_run_timestamp": "2026-10-31T12:00:41.768Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "model_slug": "anthropic/claude-sonnet-5.5", "model_name": "Anthropic: Claude Sonnet 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7733333333333333, "accuracy_std_dev": 0.02309381831909905, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.16708652333333332, "avg_duration_per_task_ms": 79748.64333333333, "avg_output_tokens_per_task": 6314.02, "last_run_timestamp": "2026-10-31T11:48:36.920Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "auto-routing", "accuracy": 0.7733333, "accuracy_std_dev": 0.026997983465622016, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.10150856999999999, "avg_duration_per_task_ms": 290647.242, "avg_output_tokens_per_task": 21805.618, "last_run_timestamp": "2026-10-31T11:29:11.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "ModelRun", "accuracy": 0.77333325, "accuracy_std_dev": 0.029814193116492343, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06547900499999999, "avg_duration_per_task_ms": 79239.345, "avg_output_tokens_per_task": 12411.3875, "last_run_timestamp": "2026-10-31T11:09:23.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Io Net", "accuracy": 0.7733331666666667, "accuracy_std_dev": 0.021081851068448003, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.05726335, "avg_duration_per_task_ms": 158508.54333333333, "avg_output_tokens_per_task": 4588.7266666666665, "last_run_timestamp": "2026-10-31T13:11:06.716Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Reka", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.012345539999999999, "avg_duration_per_task_ms": 657478.84, "avg_output_tokens_per_task": 14196.68, "last_run_timestamp": "2026-10-03T00:00:06.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Sail Research", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00804126, "avg_duration_per_task_ms": 386984.54, "avg_output_tokens_per_task": 13398.72, "last_run_timestamp": "2026-08-22T16:20:38.832Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "River", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.19214888, "avg_duration_per_task_ms": 135761, "avg_output_tokens_per_task": 9989.38, "last_run_timestamp": "2026-11-13T06:41:18.869Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Relace", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14084568, "avg_duration_per_task_ms": 130912.8, "avg_output_tokens_per_task": 10689.44, "last_run_timestamp": "2026-11-05T22:50:26.968Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Nebius", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2468412, "avg_duration_per_task_ms": 828355.82, "avg_output_tokens_per_task": 10570.46, "last_run_timestamp": "2026-11-04T22:42:55.359Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "GMICloud", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.26629643999999997, "avg_duration_per_task_ms": 438741.58, "avg_output_tokens_per_task": 27876.64, "last_run_timestamp": "2026-08-29T16:58:30.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Makora", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11485508, "avg_duration_per_task_ms": 381375.48, "avg_output_tokens_per_task": 14199.1, "last_run_timestamp": "2026-10-13T03:50:19.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "StreamLake", "accuracy": 0.773333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.057549260000000005, "avg_duration_per_task_ms": 140710.34, "avg_output_tokens_per_task": 4743.94, "last_run_timestamp": "2026-09-11T04:09:34.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Google", "accuracy": 0.7726666333333335, "accuracy_std_dev": 0.022400415672457697, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.49225354466666665, "avg_duration_per_task_ms": 105640.478, "avg_output_tokens_per_task": 7051.117333333334, "last_run_timestamp": "2026-10-31T12:26:08.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "auto-routing", "accuracy": 0.7725926666666666, "accuracy_std_dev": 0.033176625054396364, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05704026, "avg_duration_per_task_ms": 115075.10666666667, "avg_output_tokens_per_task": 6558.98, "last_run_timestamp": "2026-10-31T12:50:03.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Alibaba", "accuracy": 0.7723335, "accuracy_std_dev": 0.025212516733757476, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.18855182, "avg_duration_per_task_ms": 469904.514, "avg_output_tokens_per_task": 27186.904, "last_run_timestamp": "2026-10-31T19:55:50.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "model_slug": "google/gemini-3.7-flash", "model_name": "Google: Gemini 3.7 Flash", "provider_name": "Google", "accuracy": 0.7721088979591837, "accuracy_std_dev": 0.008055499999999993, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.18560600000000002, "avg_duration_per_task_ms": 213412.75510204083, "avg_output_tokens_per_task": 17105.88775510204, "last_run_timestamp": "2026-10-15T21:26:28.195Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Relace", "accuracy": 0.7720001999999999, "accuracy_std_dev": 0.024367767418456705, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.14564374000000002, "avg_duration_per_task_ms": 314489.068, "avg_output_tokens_per_task": 20798.24, "last_run_timestamp": "2026-10-31T13:16:57.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Azure", "accuracy": 0.772, "accuracy_std_dev": 0.02679938639596063, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.808827636, "avg_duration_per_task_ms": 183544.5, "avg_output_tokens_per_task": 6940.7, "last_run_timestamp": "2026-10-01T08:03:00.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Azure", "accuracy": 0.772, "accuracy_std_dev": 0.015719782301291577, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.438245934, "avg_duration_per_task_ms": 92813.308, "avg_output_tokens_per_task": 5707.604, "last_run_timestamp": "2026-10-31T12:52:03.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Parasail", "accuracy": 0.7718182727272727, "accuracy_std_dev": 0.031151820244762325, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.06332707454545455, "avg_duration_per_task_ms": 243359.18363636362, "avg_output_tokens_per_task": 11661.423636363636, "last_run_timestamp": "2026-10-31T12:57:09.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Baidu", "accuracy": 0.771666625, "accuracy_std_dev": 0.021537947777919213, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0856126475, "avg_duration_per_task_ms": 364320.3175, "avg_output_tokens_per_task": 20467.8275, "last_run_timestamp": "2026-11-10T19:01:03.247Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "StreamLake", "accuracy": 0.7716665, "accuracy_std_dev": 0.0016665000000000152, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.050993250000000004, "avg_duration_per_task_ms": 677393.46, "avg_output_tokens_per_task": 30421.97, "last_run_timestamp": "2026-10-31T13:28:05.897Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "SiliconFlow", "accuracy": 0.7715774062499999, "accuracy_std_dev": 0.022644299042712767, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.14963372544642858, "avg_duration_per_task_ms": 341867.6473214286, "avg_output_tokens_per_task": 20209.140625, "last_run_timestamp": "2026-10-31T14:04:49.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DeepInfra", "accuracy": 0.7714602976588629, "accuracy_std_dev": 0.028416062527791094, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.08344332775919731, "avg_duration_per_task_ms": 249520.2474916388, "avg_output_tokens_per_task": 12314.311036789297, "last_run_timestamp": "2026-10-31T19:22:52.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Venice", "accuracy": 0.771428857142857, "accuracy_std_dev": 0.019095107469331168, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3362465285714286, "avg_duration_per_task_ms": 611059.8085714285, "avg_output_tokens_per_task": 28432.711428571427, "last_run_timestamp": "2026-10-31T19:49:31.051Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Novita", "accuracy": 0.7713334, "accuracy_std_dev": 0.020011225740568715, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.051776854, "avg_duration_per_task_ms": 108112.036, "avg_output_tokens_per_task": 4665.94, "last_run_timestamp": "2026-10-31T13:28:16.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "model_slug": "anthropic/claude-fable-5.1", "model_name": "Anthropic: Claude Fable 5.1", "provider_name": "Google", "accuracy": 0.7711111666666667, "accuracy_std_dev": 0.014229407910107222, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.7819622033333333, "avg_duration_per_task_ms": 110376.29, "avg_output_tokens_per_task": 6891.35, "last_run_timestamp": "2026-11-11T21:09:52.293Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Azure", "accuracy": 0.771, "accuracy_std_dev": 0.018651732123317666, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.47884973, "avg_duration_per_task_ms": 100219.903, "avg_output_tokens_per_task": 7083.76, "last_run_timestamp": "2026-10-31T12:54:18.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DeepInfra", "accuracy": 0.7709447032085562, "accuracy_std_dev": 0.032091449469013114, "run_count": 15, "total_questions": 748, "avg_cost_per_task_usd": 0.03325723529411764, "avg_duration_per_task_ms": 455906.7553475936, "avg_output_tokens_per_task": 12250.7807486631, "last_run_timestamp": "2026-10-31T19:33:25.394Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Venice", "accuracy": 0.770833375, "accuracy_std_dev": 0.02283704805758343, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2110248925, "avg_duration_per_task_ms": 381381.2675, "avg_output_tokens_per_task": 17629.1275, "last_run_timestamp": "2026-10-31T12:14:30.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Parasail", "accuracy": 0.7706935637583894, "accuracy_std_dev": 0.013644816256244217, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.09587820805369127, "avg_duration_per_task_ms": 288021.6644295302, "avg_output_tokens_per_task": 13114.979865771813, "last_run_timestamp": "2026-08-29T08:48:12.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Phala", "accuracy": 0.7706668, "accuracy_std_dev": 0.027194791985966728, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.13800210599999999, "avg_duration_per_task_ms": 324537.226, "avg_output_tokens_per_task": 20086.83, "last_run_timestamp": "2026-10-31T13:20:51.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Anthropic", "accuracy": 0.7706667, "accuracy_std_dev": 0.025854399153915776, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.5051475320000001, "avg_duration_per_task_ms": 119868.748, "avg_output_tokens_per_task": 7988.984, "last_run_timestamp": "2026-10-31T11:50:39.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "NextBit", "accuracy": 0.7706666, "accuracy_std_dev": 0.0204828947622156, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10916058, "avg_duration_per_task_ms": 258128.42, "avg_output_tokens_per_task": 11359.708, "last_run_timestamp": "2026-10-03T16:07:05.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "CoreWeave", "accuracy": 0.7704079387755102, "accuracy_std_dev": 0.002986000000000044, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.1301587857142857, "avg_duration_per_task_ms": 225452.5, "avg_output_tokens_per_task": 42669.47959183674, "last_run_timestamp": "2026-08-29T17:18:52.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "auto-routing", "accuracy": 0.7700001, "accuracy_std_dev": 0.023333328572880462, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.145186808, "avg_duration_per_task_ms": 851216.382, "avg_output_tokens_per_task": 32358.94, "last_run_timestamp": "2026-10-31T20:06:11.663Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Wafer", "accuracy": 0.7700000714285714, "accuracy_std_dev": 0.014638549891415803, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.14041841571428573, "avg_duration_per_task_ms": 235377.64428571428, "avg_output_tokens_per_task": 25220.13142857143, "last_run_timestamp": "2026-10-31T13:20:49.855Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.77, "accuracy_std_dev": 0.01639789959720451, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.464249572, "avg_duration_per_task_ms": 116894.73, "avg_output_tokens_per_task": 7042.312, "last_run_timestamp": "2026-10-31T11:40:34.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "auto-routing", "accuracy": 0.77, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.35129818, "avg_duration_per_task_ms": 177012.35, "avg_output_tokens_per_task": 12679.64, "last_run_timestamp": "2026-10-01T19:06:01.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Moonshot AI", "accuracy": 0.77, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28240516, "avg_duration_per_task_ms": 307072.21, "avg_output_tokens_per_task": 10658.56, "last_run_timestamp": "2026-11-04T18:56:58.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "DeepInfra", "accuracy": 0.769999875, "accuracy_std_dev": 0.03382961802340332, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0406579, "avg_duration_per_task_ms": 682767.975, "avg_output_tokens_per_task": 12586.08, "last_run_timestamp": "2026-10-31T12:34:48.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "DeepInfra", "accuracy": 0.7699214781144781, "accuracy_std_dev": 0.02849514533750542, "run_count": 6, "total_questions": 297, "avg_cost_per_task_usd": 0.010507353535353536, "avg_duration_per_task_ms": 977066.3771043771, "avg_output_tokens_per_task": 18208.95286195286, "last_run_timestamp": "2026-10-31T12:24:58.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Parasail", "accuracy": 0.7696666, "accuracy_std_dev": 0.019913617899317042, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.07924518799999998, "avg_duration_per_task_ms": 90727.314, "avg_output_tokens_per_task": 4394.952, "last_run_timestamp": "2026-11-05T00:52:28.447Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Reka", "accuracy": 0.7695237142857143, "accuracy_std_dev": 0.02386662296246901, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11461242285714283, "avg_duration_per_task_ms": 472929.0742857143, "avg_output_tokens_per_task": 11699.722857142857, "last_run_timestamp": "2026-10-23T07:26:58.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Morph", "accuracy": 0.7693333, "accuracy_std_dev": 0.039350196385914006, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.087123052, "avg_duration_per_task_ms": 282737.078, "avg_output_tokens_per_task": 16723.342, "last_run_timestamp": "2026-10-31T19:58:49.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Modal", "accuracy": 0.7693333, "accuracy_std_dev": 0.027680754646685506, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.080204858, "avg_duration_per_task_ms": 55518.588, "avg_output_tokens_per_task": 4833.54, "last_run_timestamp": "2026-10-31T12:18:44.291Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Together", "accuracy": 0.769166625, "accuracy_std_dev": 0.028272094359710526, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.014342402500000002, "avg_duration_per_task_ms": 227243.235, "avg_output_tokens_per_task": 15618.9, "last_run_timestamp": "2026-10-17T11:00:48.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "AkashML", "accuracy": 0.7690024251781471, "accuracy_std_dev": 0.02344176596675554, "run_count": 17, "total_questions": 842, "avg_cost_per_task_usd": 0.05268543230403801, "avg_duration_per_task_ms": 269011.41448931117, "avg_output_tokens_per_task": 11083.264845605701, "last_run_timestamp": "2026-10-31T12:29:57.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "PrimaLabs", "accuracy": 0.7689656482758621, "accuracy_std_dev": 0.012187977929090621, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.5075254827586206, "avg_duration_per_task_ms": 227003.33793103448, "avg_output_tokens_per_task": 39823.972413793104, "last_run_timestamp": "2026-11-06T16:54:34.176Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Cloudflare", "accuracy": 0.7688888333333334, "accuracy_std_dev": 0.013698652724710655, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.15528754666666666, "avg_duration_per_task_ms": 428069.5, "avg_output_tokens_per_task": 20104.073333333334, "last_run_timestamp": "2026-10-31T20:34:04.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "auto-routing", "accuracy": 0.7685714285714286, "accuracy_std_dev": 0.020995671729853154, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.062093528571428566, "avg_duration_per_task_ms": 230920.66857142857, "avg_output_tokens_per_task": 13057.314285714287, "last_run_timestamp": "2026-10-31T20:15:14.788Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Chutes", "accuracy": 0.7684848181818182, "accuracy_std_dev": 0.034709847551636586, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.05697716181818182, "avg_duration_per_task_ms": 301468.96545454545, "avg_output_tokens_per_task": 12590.405454545455, "last_run_timestamp": "2026-10-31T11:18:52.425Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Phala", "accuracy": 0.7683335, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.18849928, "avg_duration_per_task_ms": 346868.4, "avg_output_tokens_per_task": 10339.12, "last_run_timestamp": "2026-11-05T18:26:07.849Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Parasail", "accuracy": 0.7683333999999999, "accuracy_std_dev": 0.023154082915114577, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.127404986, "avg_duration_per_task_ms": 482373.346, "avg_output_tokens_per_task": 21637.406, "last_run_timestamp": "2026-11-05T02:16:26.851Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Venice", "accuracy": 0.768333375, "accuracy_std_dev": 0.027638527357375157, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.12531909, "avg_duration_per_task_ms": 166385.7025, "avg_output_tokens_per_task": 7956.0025, "last_run_timestamp": "2026-10-31T13:24:47.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Phala", "accuracy": 0.7681482222222222, "accuracy_std_dev": 0.017435059205697385, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.052077991111111104, "avg_duration_per_task_ms": 111760.7, "avg_output_tokens_per_task": 4431.0888888888885, "last_run_timestamp": "2026-10-31T13:22:10.288Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "auto-routing", "accuracy": 0.7680001999999999, "accuracy_std_dev": 0.021457811839980336, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.49101839, "avg_duration_per_task_ms": 123065.604, "avg_output_tokens_per_task": 8037.726, "last_run_timestamp": "2026-10-31T12:30:31.663Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "auto-routing", "accuracy": 0.7677776666666667, "accuracy_std_dev": 0.043104839177934014, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06842919666666666, "avg_duration_per_task_ms": 974829.46, "avg_output_tokens_per_task": 44158.48333333333, "last_run_timestamp": "2026-10-31T13:18:29.756Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Decart", "accuracy": 0.7675, "accuracy_std_dev": 0.019561171859068173, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0578507225, "avg_duration_per_task_ms": 50974.42, "avg_output_tokens_per_task": 4677.1725, "last_run_timestamp": "2026-10-24T12:36:03.010Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Alibaba", "accuracy": 0.7674509411764706, "accuracy_std_dev": 0.021678839985437112, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.12603030588235295, "avg_duration_per_task_ms": 92367.94588235294, "avg_output_tokens_per_task": 4470.871764705882, "last_run_timestamp": "2026-10-31T12:50:48.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Google", "accuracy": 0.7673332999999999, "accuracy_std_dev": 0.02871832890118551, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.5323030653333332, "avg_duration_per_task_ms": 111144.59533333333, "avg_output_tokens_per_task": 8030.952666666667, "last_run_timestamp": "2026-10-31T12:31:31.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Claude Platform on AWS", "accuracy": 0.7672726363636364, "accuracy_std_dev": 0.030278820027065213, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.43524433636363635, "avg_duration_per_task_ms": 90898.93818181819, "avg_output_tokens_per_task": 5582.549090909091, "last_run_timestamp": "2026-10-31T12:33:27.768Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.7669999000000001, "accuracy_std_dev": 0.030657633815903015, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.449202301, "avg_duration_per_task_ms": 78155.357, "avg_output_tokens_per_task": 5628.041, "last_run_timestamp": "2026-10-31T12:41:58.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "auto-routing", "accuracy": 0.7666669999999999, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.2750355, "avg_duration_per_task_ms": 98712.99, "avg_output_tokens_per_task": 5708.24, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Novita", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0337152, "avg_duration_per_task_ms": 192245.9, "avg_output_tokens_per_task": 17702.16, "last_run_timestamp": "2026-09-01T12:41:31.535Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "ModelRun", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01300872, "avg_duration_per_task_ms": 90405.02, "avg_output_tokens_per_task": 13808.24, "last_run_timestamp": "2026-10-29T03:35:28.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Nebius", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0734938, "avg_duration_per_task_ms": 215054.84, "avg_output_tokens_per_task": 13096.66, "last_run_timestamp": "2026-11-04T18:29:36.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Featherless", "accuracy": 0.7666669999999999, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0650557, "avg_duration_per_task_ms": 127920.32, "avg_output_tokens_per_task": 4266.18, "last_run_timestamp": "2026-11-14T00:27:40.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "NextBit", "accuracy": 0.76666675, "accuracy_std_dev": 0.024267078752653787, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.036940722499999995, "avg_duration_per_task_ms": 209470.17, "avg_output_tokens_per_task": 17358.2575, "last_run_timestamp": "2026-10-03T08:39:48.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Azure", "accuracy": 0.7666667, "accuracy_std_dev": 0.02658317763943958, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.5167896780000001, "avg_duration_per_task_ms": 124380.271, "avg_output_tokens_per_task": 7924.367, "last_run_timestamp": "2026-10-31T13:09:20.558Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Relace", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.010886757164351354, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.031067933333333332, "avg_duration_per_task_ms": 60278.37, "avg_output_tokens_per_task": 4752.3966666666665, "last_run_timestamp": "2026-10-31T12:25:28.648Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Inceptron", "accuracy": 0.7666666666666667, "accuracy_std_dev": 0.023517736937610882, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04758358666666667, "avg_duration_per_task_ms": 110427.16444444444, "avg_output_tokens_per_task": 4684.946666666667, "last_run_timestamp": "2026-10-31T12:08:49.485Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google", "accuracy": 0.7666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.27949224, "avg_duration_per_task_ms": 363191.33, "avg_output_tokens_per_task": 35460.86, "last_run_timestamp": "2026-10-15T21:31:19.497Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Makora", "accuracy": 0.766068730941704, "accuracy_std_dev": 0.02506483163840778, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.06749501121076233, "avg_duration_per_task_ms": 62014.139013452914, "avg_output_tokens_per_task": 4517.719730941704, "last_run_timestamp": "2026-10-31T13:21:43.928Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DekaLLM", "accuracy": 0.7660001999999999, "accuracy_std_dev": 0.0260256591340162, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.12242998000000002, "avg_duration_per_task_ms": 384915.524, "avg_output_tokens_per_task": 20927.716, "last_run_timestamp": "2026-10-31T13:14:15.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Mistral", "accuracy": 0.7659459729729727, "accuracy_std_dev": 0.021296341804228094, "run_count": 37, "total_questions": 1850, "avg_cost_per_task_usd": 0.06441325405405407, "avg_duration_per_task_ms": 66619.0454054054, "avg_output_tokens_per_task": 4623.922162162162, "last_run_timestamp": "2026-11-11T03:15:32.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Together", "accuracy": 0.76583325, "accuracy_std_dev": 0.03273434952672042, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0269725475, "avg_duration_per_task_ms": 160221.945, "avg_output_tokens_per_task": 9025.64, "last_run_timestamp": "2026-10-24T19:16:17.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Venice", "accuracy": 0.7657144285714286, "accuracy_std_dev": 0.02990523505942035, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.14187513999999998, "avg_duration_per_task_ms": 291309.5857142857, "avg_output_tokens_per_task": 12316.085714285715, "last_run_timestamp": "2026-10-31T19:58:31.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "SiliconFlow", "accuracy": 0.7655555, "accuracy_std_dev": 0.04258615812972569, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10541816000000001, "avg_duration_per_task_ms": 274154.0233333333, "avg_output_tokens_per_task": 11441.496666666666, "last_run_timestamp": "2026-10-31T21:36:17.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Tencent", "accuracy": 0.7653333999999999, "accuracy_std_dev": 0.023999911113168737, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.088749776, "avg_duration_per_task_ms": 386636.328, "avg_output_tokens_per_task": 14600.352, "last_run_timestamp": "2026-10-31T12:27:43.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Sail Research", "accuracy": 0.7652380952380953, "accuracy_std_dev": 0.027344026724583557, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.044898270476190466, "avg_duration_per_task_ms": 87109.40476190476, "avg_output_tokens_per_task": 4336.9914285714285, "last_run_timestamp": "2026-11-06T23:32:54.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "GMICloud", "accuracy": 0.7651852222222222, "accuracy_std_dev": 0.027893614278292685, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.09080310222222222, "avg_duration_per_task_ms": 365591.88666666666, "avg_output_tokens_per_task": 18878.693333333333, "last_run_timestamp": "2026-11-10T08:00:49.752Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Darkbloom", "accuracy": 0.7651852222222222, "accuracy_std_dev": 0.023471135705030368, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.05947763333333333, "avg_duration_per_task_ms": 1101862.8644444444, "avg_output_tokens_per_task": 56201.85555555556, "last_run_timestamp": "2026-11-13T02:43:05.270Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "InferenceNet", "accuracy": 0.7648888, "accuracy_std_dev": 0.023153787094699365, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.03474511733333333, "avg_duration_per_task_ms": 66218.656, "avg_output_tokens_per_task": 4837.296, "last_run_timestamp": "2026-11-04T21:48:00.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Baidu", "accuracy": 0.7647619999999999, "accuracy_std_dev": 0.026116980863579387, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04505181428571428, "avg_duration_per_task_ms": 210436.56857142856, "avg_output_tokens_per_task": 13672.442857142858, "last_run_timestamp": "2026-10-31T20:06:41.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cloudflare", "accuracy": 0.7646666, "accuracy_std_dev": 0.016343743623784605, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.035043700000000004, "avg_duration_per_task_ms": 223770.176, "avg_output_tokens_per_task": 15530.04, "last_run_timestamp": "2026-10-31T12:46:55.135Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "model_slug": "anthropic/claude-opus-4.8", "model_name": "Anthropic: Claude Opus 4.8", "provider_name": "Amazon Bedrock", "accuracy": 0.7644444333333335, "accuracy_std_dev": 0.01919359203950342, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.5300209846666668, "avg_duration_per_task_ms": 117525.25933333334, "avg_output_tokens_per_task": 8055.547333333333, "last_run_timestamp": "2026-10-31T12:26:06.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "Alibaba", "accuracy": 0.7644443333333334, "accuracy_std_dev": 0.04565023765059233, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01549936, "avg_duration_per_task_ms": 297786.8, "avg_output_tokens_per_task": 16565.086666666666, "last_run_timestamp": "2026-10-28T04:10:41.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SambaNova", "accuracy": 0.76439777486911, "accuracy_std_dev": 0.015263579494993957, "run_count": 4, "total_questions": 191, "avg_cost_per_task_usd": 0.06284435078534031, "avg_duration_per_task_ms": 156780.0209424084, "avg_output_tokens_per_task": 9570.141361256545, "last_run_timestamp": "2026-10-31T19:43:06.929Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Friendli", "accuracy": 0.764166625, "accuracy_std_dev": 0.019418799956083154, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0292764325, "avg_duration_per_task_ms": 122942.345, "avg_output_tokens_per_task": 12042.5375, "last_run_timestamp": "2026-10-31T19:08:17.847Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7641665000000001, "accuracy_std_dev": 0.021586383312171588, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0185151775, "avg_duration_per_task_ms": 270096.6575, "avg_output_tokens_per_task": 14323.5575, "last_run_timestamp": "2026-10-31T12:30:14.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "auto-routing", "accuracy": 0.7640001, "accuracy_std_dev": 0.02235083095748345, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.431202254, "avg_duration_per_task_ms": 89253.752, "avg_output_tokens_per_task": 5552.148, "last_run_timestamp": "2026-10-31T11:48:47.453Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "model_slug": "anthropic/claude-opus-4.7", "model_name": "Anthropic: Claude Opus 4.7", "provider_name": "Google", "accuracy": 0.7640000666666666, "accuracy_std_dev": 0.021609382051836254, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.5330969666666666, "avg_duration_per_task_ms": 69978.30333333333, "avg_output_tokens_per_task": 5705.826, "last_run_timestamp": "2026-10-31T12:33:51.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "model_slug": "anthropic/claude-opus-5", "model_name": "Anthropic: Claude Opus 5", "provider_name": "Anthropic", "accuracy": 0.764, "accuracy_std_dev": 0.02091261182635972, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.476702984, "avg_duration_per_task_ms": 107995.534, "avg_output_tokens_per_task": 7135.662, "last_run_timestamp": "2026-10-31T11:27:21.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Novita", "accuracy": 0.764, "accuracy_std_dev": 0.020483142122242875, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.080228768, "avg_duration_per_task_ms": 494377.212, "avg_output_tokens_per_task": 14669.972, "last_run_timestamp": "2026-10-31T12:27:34.738Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SiliconFlow", "accuracy": 0.7637037777777778, "accuracy_std_dev": 0.021798347556015337, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06269429111111112, "avg_duration_per_task_ms": 113855.65555555555, "avg_output_tokens_per_task": 4583.311111111111, "last_run_timestamp": "2026-10-31T12:40:08.732Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google", "accuracy": 0.7633335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.178977705, "avg_duration_per_task_ms": 157076.125, "avg_output_tokens_per_task": 23034.1, "last_run_timestamp": "2026-10-01T19:04:58.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Xiaomi", "accuracy": 0.7633332500000001, "accuracy_std_dev": 0.013743584364622665, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01189864, "avg_duration_per_task_ms": 642869.1, "avg_output_tokens_per_task": 23140.98, "last_run_timestamp": "2026-10-24T13:28:08.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "CoreWeave", "accuracy": 0.763333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.025058120000000003, "avg_duration_per_task_ms": 106870.58, "avg_output_tokens_per_task": 12582.44, "last_run_timestamp": "2026-11-04T18:35:21.465Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "BaseTen", "accuracy": 0.763333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020903190000000002, "avg_duration_per_task_ms": 62428.04, "avg_output_tokens_per_task": 8093.56, "last_run_timestamp": "2026-10-31T12:43:49.743Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cloudflare", "accuracy": 0.7630302727272728, "accuracy_std_dev": 0.03170123269496259, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.15556449636363637, "avg_duration_per_task_ms": 593095.6363636364, "avg_output_tokens_per_task": 12420.629090909091, "last_run_timestamp": "2026-10-31T13:02:37.599Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "NextBit", "accuracy": 0.7628571428571429, "accuracy_std_dev": 0.023328310442945685, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11986269428571429, "avg_duration_per_task_ms": 415507.14, "avg_output_tokens_per_task": 18962.734285714287, "last_run_timestamp": "2026-10-03T08:33:31.578Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "model_slug": "xiaomi/mimo-v2.6-pro", "model_name": "Xiaomi: MiMo-V2.6-Pro", "provider_name": "GMICloud", "accuracy": 0.7627778333333333, "accuracy_std_dev": 0.03313374040368652, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07174823666666666, "avg_duration_per_task_ms": 893696.3066666666, "avg_output_tokens_per_task": 51834.78666666667, "last_run_timestamp": "2026-10-27T03:58:39.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Z.AI", "accuracy": 0.7626666, "accuracy_std_dev": 0.02954472165785287, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.009234846, "avg_duration_per_task_ms": 195074.734, "avg_output_tokens_per_task": 4475.264, "last_run_timestamp": "2026-10-31T12:25:34.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "SiliconFlow", "accuracy": 0.7626665, "accuracy_std_dev": 0.026025525932245826, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.022020616000000003, "avg_duration_per_task_ms": 287031.818, "avg_output_tokens_per_task": 15706.274, "last_run_timestamp": "2026-10-31T13:19:25.796Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "PrimeIntellect", "accuracy": 0.7625418160535117, "accuracy_std_dev": 0.02147753020972668, "run_count": 12, "total_questions": 598, "avg_cost_per_task_usd": 0.09006360702341139, "avg_duration_per_task_ms": 48936.09866220736, "avg_output_tokens_per_task": 4348.224080267559, "last_run_timestamp": "2026-10-31T12:39:14.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Crusoe", "accuracy": 0.7625417625418061, "accuracy_std_dev": 0.01910679234955175, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.03341546153846154, "avg_duration_per_task_ms": 144903.0602006689, "avg_output_tokens_per_task": 21777.678929765887, "last_run_timestamp": "2026-10-31T12:18:15.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "BaseTen", "accuracy": 0.7623932051282051, "accuracy_std_dev": 0.018251804130003154, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.24276578461538462, "avg_duration_per_task_ms": 125754.82051282052, "avg_output_tokens_per_task": 7748.235897435898, "last_run_timestamp": "2026-10-31T11:39:17.172Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Io Net", "accuracy": 0.7623223984962407, "accuracy_std_dev": 0.0190669296226313, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.011861127819548874, "avg_duration_per_task_ms": 357840.4411027569, "avg_output_tokens_per_task": 17112.51127819549, "last_run_timestamp": "2026-10-31T11:47:32.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "DeepInfra", "accuracy": 0.7622223333333333, "accuracy_std_dev": 0.030469494967188978, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029655940000000006, "avg_duration_per_task_ms": 256385.88, "avg_output_tokens_per_task": 18856.086666666666, "last_run_timestamp": "2026-10-08T22:27:05.549Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Cohere", "accuracy": 0.762177653295129, "accuracy_std_dev": 0.016008036097981727, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.027902553008595985, "avg_duration_per_task_ms": 104077.65616045846, "avg_output_tokens_per_task": 17491.627507163324, "last_run_timestamp": "2026-10-31T11:55:25.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DekaLLM", "accuracy": 0.7621775042979944, "accuracy_std_dev": 0.026194934880387766, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.06229761318051576, "avg_duration_per_task_ms": 372098.9226361032, "avg_output_tokens_per_task": 49911.94555873926, "last_run_timestamp": "2026-10-31T19:59:18.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Morph", "accuracy": 0.7619782998454405, "accuracy_std_dev": 0.039147489254698645, "run_count": 13, "total_questions": 647, "avg_cost_per_task_usd": 0.06417079598145287, "avg_duration_per_task_ms": 203709.1777434312, "avg_output_tokens_per_task": 11216.542503863988, "last_run_timestamp": "2026-10-31T14:18:08.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "DeepInfra", "accuracy": 0.7619046734693877, "accuracy_std_dev": 0.00894270029465374, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.03367711020408164, "avg_duration_per_task_ms": 2596340.9346938776, "avg_output_tokens_per_task": 51764.93877551021, "last_run_timestamp": "2026-10-09T04:05:56.641Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "DekaLLM", "accuracy": 0.761784404040404, "accuracy_std_dev": 0.046884570628299496, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.03727159090909091, "avg_duration_per_task_ms": 138498.65151515152, "avg_output_tokens_per_task": 4381.035353535353, "last_run_timestamp": "2026-10-31T12:40:46.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "GMICloud", "accuracy": 0.76166675, "accuracy_std_dev": 0.025404142635027726, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.023970375, "avg_duration_per_task_ms": 166710.515, "avg_output_tokens_per_task": 16443.75, "last_run_timestamp": "2026-11-10T06:15:38.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7612120909090909, "accuracy_std_dev": 0.02288640210595304, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.54388782, "avg_duration_per_task_ms": 180374.2618181818, "avg_output_tokens_per_task": 10717.156363636364, "last_run_timestamp": "2026-10-31T12:58:32.935Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Phala", "accuracy": 0.7611111666666667, "accuracy_std_dev": 0.013005103465135887, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.033468996666666674, "avg_duration_per_task_ms": 272357.00333333336, "avg_output_tokens_per_task": 13650.26, "last_run_timestamp": "2026-10-03T08:20:09.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "GMICloud", "accuracy": 0.7611111666666667, "accuracy_std_dev": 0.014098336301571996, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01045996, "avg_duration_per_task_ms": 648808.1933333334, "avg_output_tokens_per_task": 22246.43666666667, "last_run_timestamp": "2026-11-10T14:26:17.627Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Reka", "accuracy": 0.7609786762860727, "accuracy_std_dev": 0.02511202768812936, "run_count": 16, "total_questions": 797, "avg_cost_per_task_usd": 0.054708953575909665, "avg_duration_per_task_ms": 274645.58720200753, "avg_output_tokens_per_task": 12041.819322459221, "last_run_timestamp": "2026-10-31T12:07:50.786Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "AtlasCloud", "accuracy": 0.7609525714285714, "accuracy_std_dev": 0.07435901694742715, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.2507432571428571, "avg_duration_per_task_ms": 493938.8057142857, "avg_output_tokens_per_task": 29106.445714285714, "last_run_timestamp": "2026-10-31T19:26:18.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Novita", "accuracy": 0.7609524285714285, "accuracy_std_dev": 0.016106319095818472, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.12609460857142857, "avg_duration_per_task_ms": 262407.6285714286, "avg_output_tokens_per_task": 20587.46, "last_run_timestamp": "2026-10-31T19:46:00.618Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google", "accuracy": 0.7609522857142857, "accuracy_std_dev": 0.01649574672206044, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08085598000000001, "avg_duration_per_task_ms": 105369.85714285714, "avg_output_tokens_per_task": 22435.354285714286, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Novita", "accuracy": 0.7607886785714285, "accuracy_std_dev": 0.017367969160654122, "run_count": 9, "total_questions": 448, "avg_cost_per_task_usd": 0.018826165178571424, "avg_duration_per_task_ms": 120325.66071428571, "avg_output_tokens_per_task": 14433.707589285714, "last_run_timestamp": "2026-10-31T12:07:52.801Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "GMICloud", "accuracy": 0.7607406666666667, "accuracy_std_dev": 0.038289998766721775, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.008870675555555556, "avg_duration_per_task_ms": 170336.9, "avg_output_tokens_per_task": 15026.084444444445, "last_run_timestamp": "2026-11-10T06:27:33.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Relace", "accuracy": 0.7607406666666667, "accuracy_std_dev": 0.034633054297881376, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.006436806666666667, "avg_duration_per_task_ms": 125547.04444444444, "avg_output_tokens_per_task": 4262.8533333333335, "last_run_timestamp": "2026-10-24T14:25:23.734Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "auto-routing", "accuracy": 0.7606667, "accuracy_std_dev": 0.027398352749937356, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.535814928, "avg_duration_per_task_ms": 173096.566, "avg_output_tokens_per_task": 10678.3, "last_run_timestamp": "2026-10-31T11:55:38.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Sail Research", "accuracy": 0.76066665, "accuracy_std_dev": 0.019194930466857663, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.011314683, "avg_duration_per_task_ms": 308174.577, "avg_output_tokens_per_task": 15148.268, "last_run_timestamp": "2026-11-06T22:45:20.558Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AtlasCloud", "accuracy": 0.7606666, "accuracy_std_dev": 0.030470293002201318, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.047939388, "avg_duration_per_task_ms": 181256.216, "avg_output_tokens_per_task": 19517.584, "last_run_timestamp": "2026-10-31T12:26:48.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "DeepInfra", "accuracy": 0.7606664000000001, "accuracy_std_dev": 0.028315632117966207, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.224639588, "avg_duration_per_task_ms": 748495.616, "avg_output_tokens_per_task": 60239.316, "last_run_timestamp": "2026-09-26T08:05:12.736Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "AkashML", "accuracy": 0.7604166354166666, "accuracy_std_dev": 0.010476531320050537, "run_count": 4, "total_questions": 192, "avg_cost_per_task_usd": 0.29691326041666666, "avg_duration_per_task_ms": 310718.984375, "avg_output_tokens_per_task": 7030.182291666667, "last_run_timestamp": "2026-10-31T12:13:14.359Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Mancer 2", "accuracy": 0.7603703333333334, "accuracy_std_dev": 0.033126907327897526, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.08524436222222224, "avg_duration_per_task_ms": 566210.18, "avg_output_tokens_per_task": 12520.904444444444, "last_run_timestamp": "2026-10-31T12:26:16.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Alibaba", "accuracy": 0.7600002499999999, "accuracy_std_dev": 0.027080189561513434, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02989183, "avg_duration_per_task_ms": 125270.795, "avg_output_tokens_per_task": 12269.195, "last_run_timestamp": "2026-10-31T12:01:23.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "auto-routing", "accuracy": 0.7600001999999999, "accuracy_std_dev": 0.03126929626582598, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.114411384, "avg_duration_per_task_ms": 874872.748, "avg_output_tokens_per_task": 24203.156, "last_run_timestamp": "2026-10-31T12:44:40.958Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Cloudflare", "accuracy": 0.76, "accuracy_std_dev": 0.022253774427851893, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03227686, "avg_duration_per_task_ms": 210388.82571428572, "avg_output_tokens_per_task": 14164.22, "last_run_timestamp": "2026-10-31T19:55:08.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "SiliconFlow", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.054461335, "avg_duration_per_task_ms": 91417, "avg_output_tokens_per_task": 15101.33, "last_run_timestamp": "2026-10-03T08:13:12.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Pearl Research", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.11851576, "avg_duration_per_task_ms": 319323.58, "avg_output_tokens_per_task": 19182.22, "last_run_timestamp": "2026-11-04T15:08:37.819Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DekaLLM", "accuracy": 0.76, "accuracy_std_dev": 0.014401594564491825, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.019761673333333334, "avg_duration_per_task_ms": 291193.25333333336, "avg_output_tokens_per_task": 17811.393333333333, "last_run_timestamp": "2026-10-10T11:23:23.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3", "model_slug": "z-ai/glm-5.3", "model_name": "GLM 5.3", "provider_name": "digitalocean", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01597332, "avg_duration_per_task_ms": 42256.58, "avg_output_tokens_per_task": 1764.66, "last_run_timestamp": "2026-10-01T23:50:45.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google", "accuracy": 0.76, "accuracy_std_dev": 0.01333300000000004, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.3443486, "avg_duration_per_task_ms": 184596.7, "avg_output_tokens_per_task": 12502.76, "last_run_timestamp": "2026-10-01T19:04:40.361Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Decart", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14539668, "avg_duration_per_task_ms": 189204.7, "avg_output_tokens_per_task": 10161.9, "last_run_timestamp": "2026-11-11T15:19:13.582Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Morph", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.27690942, "avg_duration_per_task_ms": 230447.6, "avg_output_tokens_per_task": 10636.8, "last_run_timestamp": "2026-11-05T06:25:26.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "OpenAI", "accuracy": 0.76, "accuracy_std_dev": 0.005443582705045132, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.40005456666666667, "avg_duration_per_task_ms": 394539.76666666666, "avg_output_tokens_per_task": 34690.113333333335, "last_run_timestamp": "2026-10-02T23:12:57.355Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Novita", "accuracy": 0.76, "accuracy_std_dev": 0.025991443038046205, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.08891490399999999, "avg_duration_per_task_ms": 301406.882, "avg_output_tokens_per_task": 12044.934, "last_run_timestamp": "2026-10-31T12:46:14.833Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "GMICloud", "accuracy": 0.76, "accuracy_std_dev": 0.010886348913509402, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06985152, "avg_duration_per_task_ms": 702954.7, "avg_output_tokens_per_task": 12942.693333333333, "last_run_timestamp": "2026-11-10T07:27:08.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Cohere", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10151575999999998, "avg_duration_per_task_ms": 434936.03, "avg_output_tokens_per_task": 13962.46, "last_run_timestamp": "2026-10-24T12:32:07.608Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Makora", "accuracy": 0.76, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07038507, "avg_duration_per_task_ms": 256302.49, "avg_output_tokens_per_task": 10045.27, "last_run_timestamp": "2026-08-26T00:00:05.638Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "SambaNova", "accuracy": 0.76, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14464874, "avg_duration_per_task_ms": 109820.9, "avg_output_tokens_per_task": 4147.16, "last_run_timestamp": "2026-11-14T00:56:57.828Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Novita", "accuracy": 0.7599999000000001, "accuracy_std_dev": 0.030840204640858015, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.008153706000000002, "avg_duration_per_task_ms": 200971.108, "avg_output_tokens_per_task": 4219.412, "last_run_timestamp": "2026-10-31T12:49:39.925Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DeepInfra", "accuracy": 0.7599998, "accuracy_std_dev": 0.007302967435227963, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.007451167999999999, "avg_duration_per_task_ms": 324203.24, "avg_output_tokens_per_task": 12454.056, "last_run_timestamp": "2026-10-31T20:10:56.893Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "DeepInfra", "accuracy": 0.759846288184438, "accuracy_std_dev": 0.015452349289717374, "run_count": 7, "total_questions": 347, "avg_cost_per_task_usd": 0.15869831700288184, "avg_duration_per_task_ms": 791662.6916426513, "avg_output_tokens_per_task": 30669.08069164265, "last_run_timestamp": "2026-10-31T19:07:30.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Ionstream", "accuracy": 0.7595079765100671, "accuracy_std_dev": 0.030194432004533367, "run_count": 12, "total_questions": 596, "avg_cost_per_task_usd": 0.04601521476510067, "avg_duration_per_task_ms": 392651.7818791946, "avg_output_tokens_per_task": 12917.01845637584, "last_run_timestamp": "2026-11-10T17:13:11.399Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "CoreWeave", "accuracy": 0.7594444166666666, "accuracy_std_dev": 0.033715523315080306, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09267692833333334, "avg_duration_per_task_ms": 244983.77333333335, "avg_output_tokens_per_task": 12483.865, "last_run_timestamp": "2026-10-31T12:53:55.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Wafer", "accuracy": 0.7592155882352941, "accuracy_std_dev": 0.027618637596970693, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.011689743529411765, "avg_duration_per_task_ms": 166650.9011764706, "avg_output_tokens_per_task": 5474.670588235294, "last_run_timestamp": "2026-10-31T12:46:12.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "NextBit", "accuracy": 0.759157608058608, "accuracy_std_dev": 0.022493444068575088, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.01790187728937729, "avg_duration_per_task_ms": 191122.336996337, "avg_output_tokens_per_task": 6046.5641025641025, "last_run_timestamp": "2026-11-01T11:24:36.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Parasail", "accuracy": 0.7590475714285715, "accuracy_std_dev": 0.02882413325103582, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.021705314285714285, "avg_duration_per_task_ms": 232157.9742857143, "avg_output_tokens_per_task": 12154.794285714286, "last_run_timestamp": "2026-10-31T19:23:14.281Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "InferenceNet", "accuracy": 0.7590360803212852, "accuracy_std_dev": 0.01520064937560236, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.21346128514056226, "avg_duration_per_task_ms": 312634.5903614458, "avg_output_tokens_per_task": 10222.072289156626, "last_run_timestamp": "2026-11-05T00:24:27.672Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Alibaba", "accuracy": 0.7587878181818182, "accuracy_std_dev": 0.01653114932933463, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.17482494545454544, "avg_duration_per_task_ms": 315756.4018181818, "avg_output_tokens_per_task": 10740.534545454546, "last_run_timestamp": "2026-10-31T11:13:27.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Anthropic", "accuracy": 0.7586668, "accuracy_std_dev": 0.024184373284416548, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.5305409400000001, "avg_duration_per_task_ms": 181423.388, "avg_output_tokens_per_task": 10458.596, "last_run_timestamp": "2026-10-31T12:38:33.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "OpenInference", "accuracy": 0.7583335, "accuracy_std_dev": 0.011666500000000024, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01736758, "avg_duration_per_task_ms": 642594.16, "avg_output_tokens_per_task": 12204.57, "last_run_timestamp": "2026-08-29T16:47:05.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Relace", "accuracy": 0.7583333333333333, "accuracy_std_dev": 0.03125451671181126, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06620967333333333, "avg_duration_per_task_ms": 161431.49, "avg_output_tokens_per_task": 10316.483333333334, "last_run_timestamp": "2026-10-31T19:43:05.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "model_slug": "openai/gpt-5.1", "model_name": "OpenAI: GPT-5.1", "provider_name": "Azure", "accuracy": 0.7583332500000001, "accuracy_std_dev": 0.014433785597946933, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.42515429, "avg_duration_per_task_ms": 460450.54, "avg_output_tokens_per_task": 34342.89, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "AtlasCloud", "accuracy": 0.7580951428571429, "accuracy_std_dev": 0.032014759919354034, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10769543714285713, "avg_duration_per_task_ms": 240797.41714285716, "avg_output_tokens_per_task": 11172.16, "last_run_timestamp": "2026-10-31T20:03:31.357Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Alibaba", "accuracy": 0.7578571428571429, "accuracy_std_dev": 0.029048460957159017, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.09093985428571429, "avg_duration_per_task_ms": 189688.76714285713, "avg_output_tokens_per_task": 11033.007142857143, "last_run_timestamp": "2026-10-31T19:54:53.711Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "ParetoAlpha", "accuracy": 0.7577776666666667, "accuracy_std_dev": 0.013698571612479267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.28612407999999995, "avg_duration_per_task_ms": 2065607.3133333332, "avg_output_tokens_per_task": 10347.066666666668, "last_run_timestamp": "2026-10-31T11:54:15.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DigitalOcean", "accuracy": 0.7577317113402062, "accuracy_std_dev": 0.03673749999999998, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.07915294845360824, "avg_duration_per_task_ms": 496514.40206185565, "avg_output_tokens_per_task": 7669.20618556701, "last_run_timestamp": "2026-08-29T17:13:58.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "CoreWeave", "accuracy": 0.7575757272727273, "accuracy_std_dev": 0.0216999667829277, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.02146501818181818, "avg_duration_per_task_ms": 118448.94, "avg_output_tokens_per_task": 17795.93090909091, "last_run_timestamp": "2026-10-31T12:41:20.328Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Cerebras", "accuracy": 0.7575, "accuracy_std_dev": 0.05171382256708548, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.26214604, "avg_duration_per_task_ms": 26048.875, "avg_output_tokens_per_task": 12109.95, "last_run_timestamp": "2026-10-31T11:18:07.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7575, "accuracy_std_dev": 0.023078766095699303, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.012481125, "avg_duration_per_task_ms": 215467.87, "avg_output_tokens_per_task": 4890.4975, "last_run_timestamp": "2026-10-31T11:56:22.568Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "GMICloud", "accuracy": 0.7573335999999999, "accuracy_std_dev": 0.009043086986201126, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.015731108, "avg_duration_per_task_ms": 124933.624, "avg_output_tokens_per_task": 11968.716, "last_run_timestamp": "2026-11-10T06:08:17.524Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Phala", "accuracy": 0.7573335, "accuracy_std_dev": 0.016653408061114705, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.071170548, "avg_duration_per_task_ms": 532347.09, "avg_output_tokens_per_task": 31376.552, "last_run_timestamp": "2026-10-31T20:08:33.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "NextBit", "accuracy": 0.7573333999999999, "accuracy_std_dev": 0.03991088294989224, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.023026752, "avg_duration_per_task_ms": 161149.652, "avg_output_tokens_per_task": 18473.992, "last_run_timestamp": "2026-10-03T08:55:31.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "InferenceNet", "accuracy": 0.7568888666666667, "accuracy_std_dev": 0.02909656905746029, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.008429334666666666, "avg_duration_per_task_ms": 120051.87733333334, "avg_output_tokens_per_task": 4480.442666666667, "last_run_timestamp": "2026-11-04T21:55:13.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "auto-routing", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08839812, "avg_duration_per_task_ms": 73380.11, "avg_output_tokens_per_task": 8719.83, "last_run_timestamp": "2026-10-31T12:44:24.100Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Wafer", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17797051000000003, "avg_duration_per_task_ms": 171587.38, "avg_output_tokens_per_task": 10362.94, "last_run_timestamp": "2026-11-05T21:56:52.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "AtlasCloud", "accuracy": 0.7566669999999999, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06446149, "avg_duration_per_task_ms": 318929.97, "avg_output_tokens_per_task": 13284.66, "last_run_timestamp": "2026-10-31T12:25:36.318Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "CoreWeave", "accuracy": 0.7566668, "accuracy_std_dev": 0.01819650760338369, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.151380274, "avg_duration_per_task_ms": 165330.14, "avg_output_tokens_per_task": 19976.098, "last_run_timestamp": "2026-10-31T12:20:13.275Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Darkbloom", "accuracy": 0.7566666666666667, "accuracy_std_dev": 0.016216087122223912, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.02569377333333333, "avg_duration_per_task_ms": 474320.2833333333, "avg_output_tokens_per_task": 22069.101666666666, "last_run_timestamp": "2026-11-13T02:23:53.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "SiliconFlow", "accuracy": 0.7566666666666667, "accuracy_std_dev": 0.022026835486641215, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06927284666666667, "avg_duration_per_task_ms": 313911.26, "avg_output_tokens_per_task": 14319.526666666667, "last_run_timestamp": "2026-10-31T12:43:48.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "StreamLake", "accuracy": 0.756666625, "accuracy_std_dev": 0.03366510310001107, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01679233, "avg_duration_per_task_ms": 291946.0025, "avg_output_tokens_per_task": 10730.4225, "last_run_timestamp": "2026-10-31T20:10:47.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Baidu", "accuracy": 0.756666625, "accuracy_std_dev": 0.015985992133251373, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.052604632500000005, "avg_duration_per_task_ms": 71429.835, "avg_output_tokens_per_task": 4345.545, "last_run_timestamp": "2026-11-10T18:36:01.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "auto-routing", "accuracy": 0.7566665, "accuracy_std_dev": 0.0251662251622421, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03758577333333333, "avg_duration_per_task_ms": 199140.7, "avg_output_tokens_per_task": 13605.943333333333, "last_run_timestamp": "2026-10-03T08:15:57.213Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "AtlasCloud", "accuracy": 0.7566665, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07760444, "avg_duration_per_task_ms": 178261.76, "avg_output_tokens_per_task": 9204.98, "last_run_timestamp": "2026-08-29T16:11:58.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Crusoe", "accuracy": 0.7566665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09622449, "avg_duration_per_task_ms": 171236.24, "avg_output_tokens_per_task": 9651.1, "last_run_timestamp": "2026-08-29T16:31:41.554Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Venice", "accuracy": 0.7564471518624641, "accuracy_std_dev": 0.017879759130688233, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.010146441260744987, "avg_duration_per_task_ms": 321562.229226361, "avg_output_tokens_per_task": 12668.309455587392, "last_run_timestamp": "2026-10-31T19:48:48.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Reka", "accuracy": 0.7564444666666666, "accuracy_std_dev": 0.023958871344776582, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.04415017200000001, "avg_duration_per_task_ms": 138405.18933333334, "avg_output_tokens_per_task": 4510.322666666667, "last_run_timestamp": "2026-10-31T12:43:47.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "StreamLake", "accuracy": 0.7563635454545454, "accuracy_std_dev": 0.01641972318648099, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.005844881818181818, "avg_duration_per_task_ms": 265682.2036363636, "avg_output_tokens_per_task": 15989.905454545455, "last_run_timestamp": "2026-10-31T13:22:04.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Mancer 2", "accuracy": 0.7562962222222223, "accuracy_std_dev": 0.035011314652957475, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04747446444444444, "avg_duration_per_task_ms": 235698.6, "avg_output_tokens_per_task": 14674.364444444445, "last_run_timestamp": "2026-10-31T12:45:59.472Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Alibaba", "accuracy": 0.7562961111111112, "accuracy_std_dev": 0.02966657881426859, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.03134207111111111, "avg_duration_per_task_ms": 175492.92222222223, "avg_output_tokens_per_task": 14182.893333333333, "last_run_timestamp": "2026-10-31T11:54:37.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Darkbloom", "accuracy": 0.7562620173410404, "accuracy_std_dev": 0.01469578908589609, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.03183617341040463, "avg_duration_per_task_ms": 505021.5144508671, "avg_output_tokens_per_task": 12785.520231213874, "last_run_timestamp": "2026-11-13T02:24:13.374Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Azure", "accuracy": 0.7561904285714286, "accuracy_std_dev": 0.020735037227271297, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1716927257142857, "avg_duration_per_task_ms": 187054.57428571428, "avg_output_tokens_per_task": 11177.605714285713, "last_run_timestamp": "2026-10-31T19:18:33.522Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "auto-routing", "accuracy": 0.7561904285714286, "accuracy_std_dev": 0.03184434649468365, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03082173142857143, "avg_duration_per_task_ms": 402850.4971428571, "avg_output_tokens_per_task": 11747.214285714286, "last_run_timestamp": "2026-10-31T19:35:25.708Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Cloudflare", "accuracy": 0.7560607272727272, "accuracy_std_dev": 0.029432815279705333, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.18950213272727276, "avg_duration_per_task_ms": 498615.7418181818, "avg_output_tokens_per_task": 20980.29090909091, "last_run_timestamp": "2026-10-31T14:12:32.839Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "StreamLake", "accuracy": 0.755833375, "accuracy_std_dev": 0.03072051092892785, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0448964025, "avg_duration_per_task_ms": 187316.3775, "avg_output_tokens_per_task": 9579.4275, "last_run_timestamp": "2026-10-31T20:17:26.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Parasail", "accuracy": 0.7558333333333334, "accuracy_std_dev": 0.021131098726495254, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.01632131833333333, "avg_duration_per_task_ms": 115260.22166666666, "avg_output_tokens_per_task": 4430.61, "last_run_timestamp": "2026-10-31T12:09:39.076Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "auto-routing", "accuracy": 0.7557471896551724, "accuracy_std_dev": 0.051344765491603474, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.07958818965517242, "avg_duration_per_task_ms": 709509.3793103448, "avg_output_tokens_per_task": 47576.2183908046, "last_run_timestamp": "2026-10-31T18:44:20.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7556667499999999, "accuracy_std_dev": 0.03178233349342839, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.5530219200000001, "avg_duration_per_task_ms": 181245.853, "avg_output_tokens_per_task": 10575.78, "last_run_timestamp": "2026-10-31T12:37:18.963Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "auto-routing", "accuracy": 0.7555554444444444, "accuracy_std_dev": 0.018856167739030906, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0086613, "avg_duration_per_task_ms": 117282.64, "avg_output_tokens_per_task": 5380.108888888889, "last_run_timestamp": "2026-10-31T13:20:27.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Together", "accuracy": 0.7554545454545455, "accuracy_std_dev": 0.03830433230509108, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.16801238363636364, "avg_duration_per_task_ms": 309650.71636363637, "avg_output_tokens_per_task": 24145.685454545455, "last_run_timestamp": "2026-10-31T13:11:05.620Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Novita", "accuracy": 0.7553333999999999, "accuracy_std_dev": 0.017650954020675496, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.040253192, "avg_duration_per_task_ms": 172625.032, "avg_output_tokens_per_task": 16547.004, "last_run_timestamp": "2026-10-31T12:45:56.690Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Venice", "accuracy": 0.7552381428571429, "accuracy_std_dev": 0.023015459657919207, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.021888451428571428, "avg_duration_per_task_ms": 479502.3685714286, "avg_output_tokens_per_task": 13683.494285714285, "last_run_timestamp": "2026-10-31T19:04:16.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Makora", "accuracy": 0.7552378571428572, "accuracy_std_dev": 0.014126057041900291, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.009880265714285713, "avg_duration_per_task_ms": 245506.26571428572, "avg_output_tokens_per_task": 16074.99142857143, "last_run_timestamp": "2026-10-31T12:27:44.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Modal", "accuracy": 0.7551024285714285, "accuracy_std_dev": 0.011805499999999969, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.27501580612244897, "avg_duration_per_task_ms": 65997.72448979592, "avg_output_tokens_per_task": 8842.591836734693, "last_run_timestamp": "2026-10-31T12:36:21.331Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DeepInfra", "accuracy": 0.7550335167785235, "accuracy_std_dev": 0.023364430687122125, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.012232758389261746, "avg_duration_per_task_ms": 267603.8926174497, "avg_output_tokens_per_task": 4166.208053691275, "last_run_timestamp": "2026-10-31T12:14:04.084Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "DigitalOcean", "accuracy": 0.754999875, "accuracy_std_dev": 0.022791430530121965, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.011049522499999999, "avg_duration_per_task_ms": 1077704.9225, "avg_output_tokens_per_task": 13220.1325, "last_run_timestamp": "2026-10-31T20:29:45.923Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Modal", "accuracy": 0.7549998, "accuracy_std_dev": 0.029636688974310207, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.014858368, "avg_duration_per_task_ms": 71120.474, "avg_output_tokens_per_task": 4319.272, "last_run_timestamp": "2026-10-31T11:35:00.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Baidu", "accuracy": 0.7547045887096774, "accuracy_std_dev": 0.03743255869747622, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.02217318548387097, "avg_duration_per_task_ms": 116333.60483870968, "avg_output_tokens_per_task": 15344.233870967742, "last_run_timestamp": "2026-11-10T18:13:04.202Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Venice", "accuracy": 0.7547041935483871, "accuracy_std_dev": 0.023888446777469652, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.01950188306451613, "avg_duration_per_task_ms": 216554.71370967742, "avg_output_tokens_per_task": 4241.487903225807, "last_run_timestamp": "2026-10-31T11:51:07.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Anthropic", "accuracy": 0.7546666, "accuracy_std_dev": 0.025785348123304443, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.5201454280000001, "avg_duration_per_task_ms": 216038.234, "avg_output_tokens_per_task": 9883.556, "last_run_timestamp": "2026-10-31T12:37:18.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Alibaba", "accuracy": 0.75416675, "accuracy_std_dev": 0.019096569464219906, "run_count": 13, "total_questions": 640, "avg_cost_per_task_usd": 0.0324113640625, "avg_duration_per_task_ms": 163294.796875, "avg_output_tokens_per_task": 15840.8265625, "last_run_timestamp": "2026-10-31T11:41:35.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Google", "accuracy": 0.754, "accuracy_std_dev": 0.031895600533615925, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.535401688, "avg_duration_per_task_ms": 188314.086, "avg_output_tokens_per_task": 10622.22, "last_run_timestamp": "2026-10-31T12:48:37.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "GMICloud", "accuracy": 0.7538888333333333, "accuracy_std_dev": 0.021162299055290663, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.06070119499999999, "avg_duration_per_task_ms": 164537.05, "avg_output_tokens_per_task": 4635.8516666666665, "last_run_timestamp": "2026-11-10T06:54:00.557Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.16966541999999996, "avg_duration_per_task_ms": 147439, "avg_output_tokens_per_task": 22903.86, "last_run_timestamp": "2026-10-01T19:04:26.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "auto-routing", "accuracy": 0.7533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33087047999999997, "avg_duration_per_task_ms": 241177.17, "avg_output_tokens_per_task": 11982.77, "last_run_timestamp": "2026-10-01T16:01:02.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Azure", "accuracy": 0.7533334285714286, "accuracy_std_dev": 0.028507921484874254, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.02505947714285714, "avg_duration_per_task_ms": 206845.73428571428, "avg_output_tokens_per_task": 13108.18, "last_run_timestamp": "2026-10-31T19:02:15.155Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "auto-routing", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.01721340993398913, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.33945626, "avg_duration_per_task_ms": 116389.55333333333, "avg_output_tokens_per_task": 7044.336666666667, "last_run_timestamp": "2026-10-31T12:36:01.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Wafer", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.03925236487426234, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03720142666666666, "avg_duration_per_task_ms": 244561.18666666668, "avg_output_tokens_per_task": 13729.346666666666, "last_run_timestamp": "2026-10-31T19:28:27.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Relace", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.02277092037860767, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.020851383333333334, "avg_duration_per_task_ms": 133350.03333333333, "avg_output_tokens_per_task": 13555.86, "last_run_timestamp": "2026-10-31T19:21:19.082Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Parasail", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018967786666666667, "avg_duration_per_task_ms": 85971.64666666667, "avg_output_tokens_per_task": 11730.173333333334, "last_run_timestamp": "2026-10-31T12:40:46.055Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Alibaba", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.029186609056590034, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09943221, "avg_duration_per_task_ms": 284246.225, "avg_output_tokens_per_task": 12177.723333333333, "last_run_timestamp": "2026-10-31T12:41:58.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Sail Research", "accuracy": 0.7533333333333333, "accuracy_std_dev": 0.02000009259423011, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.009942297777777777, "avg_duration_per_task_ms": 208204.5977777778, "avg_output_tokens_per_task": 4241.026666666667, "last_run_timestamp": "2026-11-06T22:25:00.575Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7533332857142857, "accuracy_std_dev": 0.015118704911601446, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.3460215285714286, "avg_duration_per_task_ms": 119668.60857142857, "avg_output_tokens_per_task": 7072.9971428571425, "last_run_timestamp": "2026-10-31T12:24:55.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DeepInfra", "accuracy": 0.7533332727272728, "accuracy_std_dev": 0.023952899912084696, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.007874056363636363, "avg_duration_per_task_ms": 231905.71636363637, "avg_output_tokens_per_task": 15094.398181818182, "last_run_timestamp": "2026-10-31T13:30:46.598Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Decart", "accuracy": 0.753333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01197713, "avg_duration_per_task_ms": 212560.45, "avg_output_tokens_per_task": 13957.42, "last_run_timestamp": "2026-10-24T11:49:49.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-codex-20260224", "model_slug": "openai/gpt-5.3-codex", "model_name": "OpenAI: GPT-5.3-Codex", "provider_name": "OpenAI", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29971074000000003, "avg_duration_per_task_ms": 180697.5, "avg_output_tokens_per_task": 13569.24, "last_run_timestamp": "2026-10-02T23:12:05.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.7-20260916", "model_slug": "x-ai/grok-4.7", "model_name": "SpaceXAI: Grok 4.7", "provider_name": "Amazon Bedrock", "accuracy": 0.753333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29315726000000003, "avg_duration_per_task_ms": 206111.18, "avg_output_tokens_per_task": 11166.84, "last_run_timestamp": "2026-11-03T04:31:24.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "River", "accuracy": 0.753333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06050348, "avg_duration_per_task_ms": 93599.96, "avg_output_tokens_per_task": 4522.22, "last_run_timestamp": "2026-11-12T21:06:33.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Sail Research", "accuracy": 0.7532544781065089, "accuracy_std_dev": 0.023852454489848872, "run_count": 17, "total_questions": 845, "avg_cost_per_task_usd": 0.08912921893491124, "avg_duration_per_task_ms": 600544.8769230769, "avg_output_tokens_per_task": 21467.282840236687, "last_run_timestamp": "2026-11-07T02:09:29.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Together", "accuracy": 0.7527289420654912, "accuracy_std_dev": 0.024253123484087077, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.06776491435768262, "avg_duration_per_task_ms": 1043193.3853904282, "avg_output_tokens_per_task": 77465.15365239295, "last_run_timestamp": "2026-10-24T19:38:23.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Fireworks", "accuracy": 0.752715030075188, "accuracy_std_dev": 0.009940333684036977, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.02512772681704261, "avg_duration_per_task_ms": 75665.01253132832, "avg_output_tokens_per_task": 12251.558897243107, "last_run_timestamp": "2026-10-31T12:26:39.122Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "StreamLake", "accuracy": 0.7526667, "accuracy_std_dev": 0.02374861961062159, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.007248376000000002, "avg_duration_per_task_ms": 194889.812, "avg_output_tokens_per_task": 4384.802, "last_run_timestamp": "2026-10-31T13:46:40.932Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "InferenceNet", "accuracy": 0.7525287586206897, "accuracy_std_dev": 0.020184097544858358, "run_count": 29, "total_questions": 1450, "avg_cost_per_task_usd": 0.2696313268965517, "avg_duration_per_task_ms": 233088.7579310345, "avg_output_tokens_per_task": 11456.733793103449, "last_run_timestamp": "2026-11-04T21:30:17.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7523809047619048, "accuracy_std_dev": 0.017612644913472624, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.36889707904761904, "avg_duration_per_task_ms": 141180.9876190476, "avg_output_tokens_per_task": 7127.063809523809, "last_run_timestamp": "2026-10-31T12:26:17.125Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Wafer", "accuracy": 0.752222375, "accuracy_std_dev": 0.02258048225941397, "run_count": 24, "total_questions": 1200, "avg_cost_per_task_usd": 0.24382183083333336, "avg_duration_per_task_ms": 219317.55666666667, "avg_output_tokens_per_task": 10147.890833333333, "last_run_timestamp": "2026-10-31T12:43:33.299Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Venice", "accuracy": 0.752, "accuracy_std_dev": 0.0324961125890467, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.018297658, "avg_duration_per_task_ms": 209683.558, "avg_output_tokens_per_task": 15226.814, "last_run_timestamp": "2026-10-31T12:48:59.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Google", "accuracy": 0.752, "accuracy_std_dev": 0.018570921118781412, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.12117575600000001, "avg_duration_per_task_ms": 124707.216, "avg_output_tokens_per_task": 10483.008, "last_run_timestamp": "2026-10-17T19:14:08.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "Alibaba", "accuracy": 0.7517208112094396, "accuracy_std_dev": 0.045868971980311604, "run_count": 7, "total_questions": 339, "avg_cost_per_task_usd": 0.09701299410029499, "avg_duration_per_task_ms": 405555.37758112093, "avg_output_tokens_per_task": 19035.16814159292, "last_run_timestamp": "2026-10-31T19:29:13.230Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Nebius", "accuracy": 0.7516778724832214, "accuracy_std_dev": 0.030780262062995275, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.08526638255033557, "avg_duration_per_task_ms": 172059.79865771811, "avg_output_tokens_per_task": 8282.208053691274, "last_run_timestamp": "2026-10-31T11:50:00.858Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Alibaba", "accuracy": 0.7514694483627204, "accuracy_std_dev": 0.034873185045067215, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.10740512846347608, "avg_duration_per_task_ms": 153833.89168765742, "avg_output_tokens_per_task": 9153.158690176322, "last_run_timestamp": "2026-10-31T19:46:27.571Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "auto-routing", "accuracy": 0.7514287142857142, "accuracy_std_dev": 0.02844421520758676, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.032982819999999996, "avg_duration_per_task_ms": 221535.9714285714, "avg_output_tokens_per_task": 35206.688571428575, "last_run_timestamp": "2026-10-31T11:46:18.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Parasail", "accuracy": 0.751256015075377, "accuracy_std_dev": 0.006852721115549665, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.07795362814070352, "avg_duration_per_task_ms": 228603.50753768845, "avg_output_tokens_per_task": 10368.422110552763, "last_run_timestamp": "2026-10-10T19:17:08.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "GMICloud", "accuracy": 0.7511110833333333, "accuracy_std_dev": 0.03446230540619305, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.09647276666666667, "avg_duration_per_task_ms": 304912.8616666667, "avg_output_tokens_per_task": 18589.79, "last_run_timestamp": "2026-11-10T08:38:21.830Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7508046206896551, "accuracy_std_dev": 0.028089208249744292, "run_count": 29, "total_questions": 1450, "avg_cost_per_task_usd": 0.20928471310344823, "avg_duration_per_task_ms": 114151.32896551724, "avg_output_tokens_per_task": 7693.8193103448275, "last_run_timestamp": "2026-10-31T12:48:39.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Decart", "accuracy": 0.7506668, "accuracy_std_dev": 0.025854254755455656, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.264866352, "avg_duration_per_task_ms": 264537.464, "avg_output_tokens_per_task": 10440.212, "last_run_timestamp": "2026-10-24T11:49:28.605Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Parasail", "accuracy": 0.7506666, "accuracy_std_dev": 0.016384222765819578, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.036424934, "avg_duration_per_task_ms": 269977.466, "avg_output_tokens_per_task": 21216.128, "last_run_timestamp": "2026-10-31T20:23:01.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Near AI", "accuracy": 0.7506666, "accuracy_std_dev": 0.03785357285435553, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.007070539999999999, "avg_duration_per_task_ms": 292989.264, "avg_output_tokens_per_task": 3954.392, "last_run_timestamp": "2026-10-31T13:04:37.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "AtlasCloud", "accuracy": 0.7504761428571428, "accuracy_std_dev": 0.026869944603210353, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.011859305714285716, "avg_duration_per_task_ms": 203490.8142857143, "avg_output_tokens_per_task": 13513.24, "last_run_timestamp": "2026-10-31T20:16:35.156Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "AtlasCloud", "accuracy": 0.7504761428571428, "accuracy_std_dev": 0.04779025223509368, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.020752462857142853, "avg_duration_per_task_ms": 333009.8742857143, "avg_output_tokens_per_task": 11171.528571428571, "last_run_timestamp": "2026-10-31T13:00:18.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Darkbloom", "accuracy": 0.7500001506849315, "accuracy_std_dev": 0.02552374915424634, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.027398205479452053, "avg_duration_per_task_ms": 1306114.2808219178, "avg_output_tokens_per_task": 60026.84246575343, "last_run_timestamp": "2026-11-13T02:06:40.995Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "CoreWeave", "accuracy": 0.75, "accuracy_std_dev": 0.023333000000000048, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01955629, "avg_duration_per_task_ms": 235357.66, "avg_output_tokens_per_task": 12724.88, "last_run_timestamp": "2026-08-29T16:33:29.042Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Ionstream", "accuracy": 0.75, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08026519, "avg_duration_per_task_ms": 649894.08, "avg_output_tokens_per_task": 11165.01, "last_run_timestamp": "2026-08-29T17:19:42.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "BaseTen", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.28014248, "avg_duration_per_task_ms": 223020.56, "avg_output_tokens_per_task": 18715.38, "last_run_timestamp": "2026-09-12T08:12:13.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Together", "accuracy": 0.75, "accuracy_std_dev": 0.01666660000320003, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.041500640000000005, "avg_duration_per_task_ms": 199607.65333333332, "avg_output_tokens_per_task": 12674.073333333334, "last_run_timestamp": "2026-10-03T08:33:09.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Io Net", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.37141135, "avg_duration_per_task_ms": 606870.4, "avg_output_tokens_per_task": 13812.73, "last_run_timestamp": "2026-11-11T05:59:06.349Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "auto-routing", "accuracy": 0.75, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29130333999999997, "avg_duration_per_task_ms": 176962.37, "avg_output_tokens_per_task": 10386.26, "last_run_timestamp": "2026-10-01T19:04:36.169Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "DeepInfra", "accuracy": 0.75, "accuracy_std_dev": 0.028631403807707387, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04503071555555555, "avg_duration_per_task_ms": 450907.20666666667, "avg_output_tokens_per_task": 22758.244444444445, "last_run_timestamp": "2026-10-24T19:38:23.863Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Venice", "accuracy": 0.75, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01952235, "avg_duration_per_task_ms": 2267441.92, "avg_output_tokens_per_task": 19667.19, "last_run_timestamp": "2026-10-03T08:13:52.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "SiliconFlow", "accuracy": 0.7499999999999999, "accuracy_std_dev": 0.03349968490896593, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.028200379999999997, "avg_duration_per_task_ms": 163955.21, "avg_output_tokens_per_task": 15323.823333333334, "last_run_timestamp": "2026-10-31T12:28:07.595Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Azure", "accuracy": 0.749524, "accuracy_std_dev": 0.021633867377662402, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.35241304, "avg_duration_per_task_ms": 87048.52428571429, "avg_output_tokens_per_task": 6993.714285714285, "last_run_timestamp": "2026-10-31T11:29:30.760Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "CoreWeave", "accuracy": 0.7495238571428572, "accuracy_std_dev": 0.021338220105372136, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.016605834285714285, "avg_duration_per_task_ms": 72972.84285714285, "avg_output_tokens_per_task": 4253.202857142857, "last_run_timestamp": "2026-10-31T12:03:51.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "DeepInfra", "accuracy": 0.7493333, "accuracy_std_dev": 0.015549210771289982, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.012228522, "avg_duration_per_task_ms": 315949.068, "avg_output_tokens_per_task": 16300.002, "last_run_timestamp": "2026-10-31T11:27:38.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Fireworks", "accuracy": 0.7492774046242774, "accuracy_std_dev": 0.025230016427736986, "run_count": 14, "total_questions": 692, "avg_cost_per_task_usd": 0.013838482658959535, "avg_duration_per_task_ms": 56300.32225433526, "avg_output_tokens_per_task": 4085.725433526012, "last_run_timestamp": "2026-10-31T12:14:05.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Claude Platform on AWS", "accuracy": 0.749091, "accuracy_std_dev": 0.0224325111713902, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.1957560618181818, "avg_duration_per_task_ms": 114755.80545454545, "avg_output_tokens_per_task": 7618.518181818182, "last_run_timestamp": "2026-10-31T11:08:14.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Ionstream", "accuracy": 0.7490679239373601, "accuracy_std_dev": 0.028614120182218948, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.09606371140939597, "avg_duration_per_task_ms": 393542.0715883669, "avg_output_tokens_per_task": 24831.977628635348, "last_run_timestamp": "2026-10-31T12:39:48.592Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "DeepInfra", "accuracy": 0.748889, "accuracy_std_dev": 0.015713326891527487, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03620722, "avg_duration_per_task_ms": 114735.25333333333, "avg_output_tokens_per_task": 14087.753333333334, "last_run_timestamp": "2026-10-31T11:35:01.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "ModelRun", "accuracy": 0.7488888333333333, "accuracy_std_dev": 0.024993786884054944, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.08968307666666667, "avg_duration_per_task_ms": 142393.51, "avg_output_tokens_per_task": 12273.018333333333, "last_run_timestamp": "2026-10-31T18:56:42.115Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "DigitalOcean", "accuracy": 0.7487373080808083, "accuracy_std_dev": 0.03476268720171639, "run_count": 8, "total_questions": 396, "avg_cost_per_task_usd": 0.08973175505050504, "avg_duration_per_task_ms": 588220.7929292929, "avg_output_tokens_per_task": 9930.64898989899, "last_run_timestamp": "2026-10-31T20:25:10.002Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "auto-routing", "accuracy": 0.7486668, "accuracy_std_dev": 0.03141830181534325, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.200451768, "avg_duration_per_task_ms": 115869.634, "avg_output_tokens_per_task": 7737.526, "last_run_timestamp": "2026-10-31T11:54:55.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Parasail", "accuracy": 0.7486667, "accuracy_std_dev": 0.02733343658616678, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.018688805999999995, "avg_duration_per_task_ms": 207184.068, "avg_output_tokens_per_task": 15779.308, "last_run_timestamp": "2026-10-31T12:32:12.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "SiliconFlow", "accuracy": 0.7485715714285714, "accuracy_std_dev": 0.02069105868213987, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.013271285714285713, "avg_duration_per_task_ms": 244472.46, "avg_output_tokens_per_task": 14692.394285714287, "last_run_timestamp": "2026-10-31T19:19:01.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Anthropic", "accuracy": 0.7485714285714286, "accuracy_std_dev": 0.023560522731632144, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.34892749714285715, "avg_duration_per_task_ms": 98456.37428571428, "avg_output_tokens_per_task": 7086.754285714285, "last_run_timestamp": "2026-10-31T11:27:20.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Amazon Bedrock", "accuracy": 0.7485712857142858, "accuracy_std_dev": 0.027994178209632344, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.18495269714285717, "avg_duration_per_task_ms": 176865.04, "avg_output_tokens_per_task": 9737.897142857142, "last_run_timestamp": "2026-10-31T20:09:19.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Wafer", "accuracy": 0.748333375, "accuracy_std_dev": 0.024664456546503833, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0092696725, "avg_duration_per_task_ms": 91481.9675, "avg_output_tokens_per_task": 13410.6275, "last_run_timestamp": "2026-10-31T13:03:50.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Modal", "accuracy": 0.748322067114094, "accuracy_std_dev": 0.02148614665313445, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.029814261744966444, "avg_duration_per_task_ms": 257681.3489932886, "avg_output_tokens_per_task": 10900.818791946309, "last_run_timestamp": "2026-10-31T12:26:05.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Baidu", "accuracy": 0.7481482222222222, "accuracy_std_dev": 0.02927572719953733, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.012146982222222224, "avg_duration_per_task_ms": 103906.89555555556, "avg_output_tokens_per_task": 16269.204444444444, "last_run_timestamp": "2026-11-10T18:35:45.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "auto-routing", "accuracy": 0.7481482222222222, "accuracy_std_dev": 0.020068464436510978, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.19089935333333333, "avg_duration_per_task_ms": 286228.22222222225, "avg_output_tokens_per_task": 11572.273333333333, "last_run_timestamp": "2026-10-31T12:37:16.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Io Net", "accuracy": 0.748148, "accuracy_std_dev": null, "run_count": 1, "total_questions": 45, "avg_cost_per_task_usd": 0.021196088888888888, "avg_duration_per_task_ms": 92100.66666666667, "avg_output_tokens_per_task": 8167.488888888889, "last_run_timestamp": "2026-08-22T08:32:23.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "auto-routing", "accuracy": 0.7480002, "accuracy_std_dev": 0.020396130352593862, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.030171100000000003, "avg_duration_per_task_ms": 197857.312, "avg_output_tokens_per_task": 18663.832, "last_run_timestamp": "2026-10-03T08:18:33.163Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Anthropic", "accuracy": 0.7480001, "accuracy_std_dev": 0.037333425000795194, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.19706498, "avg_duration_per_task_ms": 95603.86, "avg_output_tokens_per_task": 7573.922, "last_run_timestamp": "2026-10-31T11:28:40.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "BaseTen", "accuracy": 0.74791675, "accuracy_std_dev": 0.031222055284486006, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01487202, "avg_duration_per_task_ms": 51526.7675, "avg_output_tokens_per_task": 4120.2525, "last_run_timestamp": "2026-10-31T13:13:13.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Inceptron", "accuracy": 0.7478510515759312, "accuracy_std_dev": 0.031000808296901307, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.018540833810888252, "avg_duration_per_task_ms": 126193.2005730659, "avg_output_tokens_per_task": 4035.12893982808, "last_run_timestamp": "2026-10-31T11:27:58.748Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Crusoe", "accuracy": 0.7477776666666667, "accuracy_std_dev": 0.028393017748891882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.020487799999999997, "avg_duration_per_task_ms": 112394.16, "avg_output_tokens_per_task": 6879.653333333334, "last_run_timestamp": "2026-10-31T12:42:03.915Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "xAI", "accuracy": 0.747619, "accuracy_std_dev": 0.025305279849978234, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.33171734571428574, "avg_duration_per_task_ms": 235739.5942857143, "avg_output_tokens_per_task": 12164.34, "last_run_timestamp": "2026-10-01T16:01:04.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Modal", "accuracy": 0.7472221666666666, "accuracy_std_dev": 0.011613686586906374, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.17439429333333334, "avg_duration_per_task_ms": 161420.59666666668, "avg_output_tokens_per_task": 12499.9, "last_run_timestamp": "2026-10-10T11:19:48.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Wafer", "accuracy": 0.7470370555555556, "accuracy_std_dev": 0.025285697851359517, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.02032471444444444, "avg_duration_per_task_ms": 102872.56777777777, "avg_output_tokens_per_task": 14307.606666666667, "last_run_timestamp": "2026-10-31T13:23:12.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Relace", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00930136, "avg_duration_per_task_ms": 89811.58, "avg_output_tokens_per_task": 12020.36, "last_run_timestamp": "2026-10-24T15:18:33.444Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "DeepInfra", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.22191687999999998, "avg_duration_per_task_ms": 2285676.56, "avg_output_tokens_per_task": 10495.84, "last_run_timestamp": "2026-11-05T06:48:08.646Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "AkashML", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.21601631999999998, "avg_duration_per_task_ms": 312892.28, "avg_output_tokens_per_task": 10774.7, "last_run_timestamp": "2026-11-05T06:21:39.264Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Alibaba", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.33231256000000003, "avg_duration_per_task_ms": 225567.04, "avg_output_tokens_per_task": 11123.54, "last_run_timestamp": "2026-11-04T21:00:38.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "InferenceNet", "accuracy": 0.746667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07074242, "avg_duration_per_task_ms": 800280.44, "avg_output_tokens_per_task": 15183.42, "last_run_timestamp": "2026-10-02T06:00:07.943Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Venice", "accuracy": 0.7466667499999999, "accuracy_std_dev": 0.012472414008021857, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.070158785, "avg_duration_per_task_ms": 1726878.96, "avg_output_tokens_per_task": 17573.88, "last_run_timestamp": "2026-10-03T08:06:43.536Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "model_slug": "anthropic/claude-opus-4.5", "model_name": "Anthropic: Claude Opus 4.5", "provider_name": "Azure", "accuracy": 0.7466667, "accuracy_std_dev": 0.02529832669189804, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.5260258940000001, "avg_duration_per_task_ms": 187136.702, "avg_output_tokens_per_task": 10321.958, "last_run_timestamp": "2026-10-31T11:53:29.846Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Phala", "accuracy": 0.7466666000000001, "accuracy_std_dev": 0.035527816958546725, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.032979988, "avg_duration_per_task_ms": 209798.362, "avg_output_tokens_per_task": 16884.604, "last_run_timestamp": "2026-10-31T12:25:12.423Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Reka", "accuracy": 0.7466665625000001, "accuracy_std_dev": 0.022360772946078904, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.013514558749999999, "avg_duration_per_task_ms": 139486.7775, "avg_output_tokens_per_task": 15486.175, "last_run_timestamp": "2026-10-31T13:03:01.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DeepInfra", "accuracy": 0.7466665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00520861, "avg_duration_per_task_ms": 148091.91, "avg_output_tokens_per_task": 12193.49, "last_run_timestamp": "2026-10-31T11:47:33.345Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-glimmer-30b-20260810", "model_slug": "meta/muse-glimmer-30b", "model_name": "Meta: Muse Glimmer 30B", "provider_name": "Fireworks", "accuracy": 0.7466665, "accuracy_std_dev": 0.01054097823970812, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.047252215, "avg_duration_per_task_ms": 265241.545, "avg_output_tokens_per_task": 13256.62, "last_run_timestamp": "2026-09-19T08:01:36.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.58250779, "avg_duration_per_task_ms": 102747.13, "avg_output_tokens_per_task": 7546.85, "last_run_timestamp": "2026-10-01T19:04:42.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "River", "accuracy": 0.7466665, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.007670949999999999, "avg_duration_per_task_ms": 53013.89, "avg_output_tokens_per_task": 4043, "last_run_timestamp": "2026-11-12T20:51:11.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Morph", "accuracy": 0.7466663333333334, "accuracy_std_dev": 0.033993463423951875, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01425808, "avg_duration_per_task_ms": 139346.08, "avg_output_tokens_per_task": 12850.746666666666, "last_run_timestamp": "2026-10-31T12:18:46.817Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Azure", "accuracy": 0.74633335, "accuracy_std_dev": 0.028478019009887265, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.201189951, "avg_duration_per_task_ms": 95684.807, "avg_output_tokens_per_task": 7554.449, "last_run_timestamp": "2026-10-31T12:38:13.068Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "DeepInfra", "accuracy": 0.7463126578171091, "accuracy_std_dev": 0.020825554044618038, "run_count": 7, "total_questions": 339, "avg_cost_per_task_usd": 0.15311067846607668, "avg_duration_per_task_ms": 683576.4837758112, "avg_output_tokens_per_task": 30552.457227138642, "last_run_timestamp": "2026-10-31T19:55:58.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Anthropic", "accuracy": 0.7460002, "accuracy_std_dev": 0.031191292232929373, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.34538107000000007, "avg_duration_per_task_ms": 224701.758, "avg_output_tokens_per_task": 10858.334, "last_run_timestamp": "2026-10-31T12:24:26.627Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Decart", "accuracy": 0.74583325, "accuracy_std_dev": 0.03402396010442498, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.02499106, "avg_duration_per_task_ms": 929627.78, "avg_output_tokens_per_task": 52531.49, "last_run_timestamp": "2026-10-24T12:31:49.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7457779, "accuracy_std_dev": 0.024569333893222824, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.3685412306666667, "avg_duration_per_task_ms": 148474.26666666666, "avg_output_tokens_per_task": 11703.939333333334, "last_run_timestamp": "2026-10-31T12:39:28.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "CoreWeave", "accuracy": 0.7457142857142857, "accuracy_std_dev": 0.02770085585430936, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.025092839999999998, "avg_duration_per_task_ms": 285709.0542857143, "avg_output_tokens_per_task": 10724.948571428571, "last_run_timestamp": "2026-10-31T19:15:14.763Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Venice", "accuracy": 0.7457142857142857, "accuracy_std_dev": 0.035353609893330494, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0900297257142857, "avg_duration_per_task_ms": 239520.15714285715, "avg_output_tokens_per_task": 6359.951428571429, "last_run_timestamp": "2026-10-31T19:33:49.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "model_slug": "anthropic/claude-sonnet-5", "model_name": "Anthropic: Claude Sonnet 5", "provider_name": "Google", "accuracy": 0.7453333, "accuracy_std_dev": 0.032313302845886877, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.20589792800000006, "avg_duration_per_task_ms": 90844.94666666667, "avg_output_tokens_per_task": 7426.582, "last_run_timestamp": "2026-10-31T13:20:20.506Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Novita", "accuracy": 0.7453333, "accuracy_std_dev": 0.02454934079379731, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.17119798400000003, "avg_duration_per_task_ms": 320387.726, "avg_output_tokens_per_task": 11023.856, "last_run_timestamp": "2026-10-31T12:25:36.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "StreamLake", "accuracy": 0.7451853333333333, "accuracy_std_dev": 0.031234315132203188, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.01557802666666667, "avg_duration_per_task_ms": 147555.71333333335, "avg_output_tokens_per_task": 15042.41111111111, "last_run_timestamp": "2026-10-31T12:25:02.563Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Together", "accuracy": 0.7451852222222222, "accuracy_std_dev": 0.017716056438394953, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.21677278888888887, "avg_duration_per_task_ms": 284791.18666666665, "avg_output_tokens_per_task": 11557.691111111111, "last_run_timestamp": "2026-10-31T11:21:59.486Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "SambaNova", "accuracy": 0.745, "accuracy_std_dev": 0.026822730826670128, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.15284815, "avg_duration_per_task_ms": 494527.49, "avg_output_tokens_per_task": 28246.935, "last_run_timestamp": "2026-10-17T19:47:21.934Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Phala", "accuracy": 0.744999875, "accuracy_std_dev": 0.030867001888900313, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0114373625, "avg_duration_per_task_ms": 404697.66, "avg_output_tokens_per_task": 5754.185, "last_run_timestamp": "2026-10-31T12:24:48.023Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Cohere", "accuracy": 0.744898, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.046561530612244904, "avg_duration_per_task_ms": 36257.63265306123, "avg_output_tokens_per_task": 3739.4285714285716, "last_run_timestamp": "2026-11-13T00:52:20.873Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "PrimaLabs", "accuracy": 0.7447914583333333, "accuracy_std_dev": 0.005435000000000023, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.13105378125, "avg_duration_per_task_ms": 273825.1666666667, "avg_output_tokens_per_task": 8026.104166666667, "last_run_timestamp": "2026-11-06T15:59:08.297Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Azure", "accuracy": 0.7446667, "accuracy_std_dev": 0.01956190515798501, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.3469682859999999, "avg_duration_per_task_ms": 215223.822, "avg_output_tokens_per_task": 11882.416, "last_run_timestamp": "2026-10-31T13:31:12.841Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Wafer", "accuracy": 0.7446667, "accuracy_std_dev": 0.01956180291818729, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.055664519999999995, "avg_duration_per_task_ms": 332682.316, "avg_output_tokens_per_task": 14458.648, "last_run_timestamp": "2026-10-31T11:42:33.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Together", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02140772, "avg_duration_per_task_ms": 53184.22, "avg_output_tokens_per_task": 14364.18, "last_run_timestamp": "2026-10-31T11:41:11.192Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Reka", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04041968666666667, "avg_duration_per_task_ms": 291588.68666666665, "avg_output_tokens_per_task": 24330.48, "last_run_timestamp": "2026-09-30T11:25:05.946Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DekaLLM", "accuracy": 0.7444443333333334, "accuracy_std_dev": 0.03823241466202323, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.012960726666666665, "avg_duration_per_task_ms": 94310.08666666667, "avg_output_tokens_per_task": 4403.8, "last_run_timestamp": "2026-11-11T15:58:45.893Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "model_slug": "qwen/qwen3.5-122b-a10b", "model_name": "Qwen: Qwen3.5-122B-A10B", "provider_name": "AtlasCloud", "accuracy": 0.7440000999999999, "accuracy_std_dev": 0.10836965996112564, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.150848968, "avg_duration_per_task_ms": 558886.734, "avg_output_tokens_per_task": 27996, "last_run_timestamp": "2026-10-31T19:37:30.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Phala", "accuracy": 0.744, "accuracy_std_dev": 0.04013309523821955, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.12688716400000002, "avg_duration_per_task_ms": 700323.218, "avg_output_tokens_per_task": 36239.088, "last_run_timestamp": "2026-10-31T12:47:12.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Io Net", "accuracy": 0.743738, "accuracy_std_dev": 0.01688456971344439, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.00938970520231214, "avg_duration_per_task_ms": 347914.32658959535, "avg_output_tokens_per_task": 3877.916184971098, "last_run_timestamp": "2026-10-03T08:46:08.334Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "PrimeIntellect", "accuracy": 0.743502793220339, "accuracy_std_dev": 0.018213736559866647, "run_count": 6, "total_questions": 295, "avg_cost_per_task_usd": 0.010024135593220337, "avg_duration_per_task_ms": 123315.84745762713, "avg_output_tokens_per_task": 13505.108474576271, "last_run_timestamp": "2026-11-02T17:51:25.369Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Ambient", "accuracy": 0.7433335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06358472, "avg_duration_per_task_ms": 288690.34, "avg_output_tokens_per_task": 8551, "last_run_timestamp": "2026-08-29T08:12:20.454Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "model_slug": "x-ai/grok-4.6", "model_name": "SpaceXAI: Grok 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7433335, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.64823893, "avg_duration_per_task_ms": 264133.83, "avg_output_tokens_per_task": 10511, "last_run_timestamp": "2026-10-01T16:00:37.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cloudflare", "accuracy": 0.7433333333333333, "accuracy_std_dev": 0.017532868606008445, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012350623333333333, "avg_duration_per_task_ms": 125234.99333333333, "avg_output_tokens_per_task": 4198.8, "last_run_timestamp": "2026-10-03T08:23:09.816Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "model_slug": "anthropic/claude-opus-5.5", "model_name": "Anthropic: Claude Opus 5.5", "provider_name": "Google", "accuracy": 0.7431746190476191, "accuracy_std_dev": 0.018008818983714862, "run_count": 21, "total_questions": 1050, "avg_cost_per_task_usd": 0.36043625523809525, "avg_duration_per_task_ms": 86437.01714285715, "avg_output_tokens_per_task": 7106.278095238095, "last_run_timestamp": "2026-10-31T12:25:51.224Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7430302727272727, "accuracy_std_dev": 0.027760033635986, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.5088923236363636, "avg_duration_per_task_ms": 192892.78363636363, "avg_output_tokens_per_task": 9984.365454545454, "last_run_timestamp": "2026-10-31T12:42:53.834Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AkashML", "accuracy": 0.7430122052565706, "accuracy_std_dev": 0.013125368013739409, "run_count": 16, "total_questions": 799, "avg_cost_per_task_usd": 0.04480421151439299, "avg_duration_per_task_ms": 334888.2340425532, "avg_output_tokens_per_task": 25744.07634543179, "last_run_timestamp": "2026-10-31T19:20:57.312Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Morph", "accuracy": 0.7427351282051282, "accuracy_std_dev": 0.028431218171008805, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.013693692307692308, "avg_duration_per_task_ms": 1000472.5128205129, "avg_output_tokens_per_task": 18094.641025641027, "last_run_timestamp": "2026-11-05T09:28:04.757Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "SiliconFlow", "accuracy": 0.7426666000000001, "accuracy_std_dev": 0.04434197309367279, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.16675142799999998, "avg_duration_per_task_ms": 302047.356, "avg_output_tokens_per_task": 9682.606, "last_run_timestamp": "2026-10-31T11:21:43.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Sail Research", "accuracy": 0.7425926666666667, "accuracy_std_dev": 0.03165311687654157, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.013586744444444443, "avg_duration_per_task_ms": 236886.98, "avg_output_tokens_per_task": 22693.746666666666, "last_run_timestamp": "2026-10-24T12:29:34.092Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Morph", "accuracy": 0.7425926111111111, "accuracy_std_dev": 0.0225683017579448, "run_count": 18, "total_questions": 900, "avg_cost_per_task_usd": 0.4177886388888889, "avg_duration_per_task_ms": 618406.2588888889, "avg_output_tokens_per_task": 10668.22, "last_run_timestamp": "2026-10-31T11:24:57.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Mara", "accuracy": 0.7425405959079282, "accuracy_std_dev": 0.026283120985422872, "run_count": 8, "total_questions": 391, "avg_cost_per_task_usd": 0.12785004603580563, "avg_duration_per_task_ms": 203562.63938618926, "avg_output_tokens_per_task": 10880.268542199488, "last_run_timestamp": "2026-10-31T19:27:07.786Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Baidu", "accuracy": 0.7425, "accuracy_std_dev": 0.02470671298959049, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.007974519999999999, "avg_duration_per_task_ms": 152350.215, "avg_output_tokens_per_task": 12723.1225, "last_run_timestamp": "2026-11-10T18:17:30.522Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Alibaba", "accuracy": 0.7424241818181818, "accuracy_std_dev": 0.03530779679235989, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.36784247818181814, "avg_duration_per_task_ms": 381853.54363636364, "avg_output_tokens_per_task": 9773.532727272728, "last_run_timestamp": "2026-10-31T12:43:33.294Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.016629618082071397, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.27693500666666665, "avg_duration_per_task_ms": 100083.4, "avg_output_tokens_per_task": 5197.013333333333, "last_run_timestamp": "2026-10-30T00:15:31.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Cohere", "accuracy": 0.7422223333333333, "accuracy_std_dev": 0.011330936570096763, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01793052666666667, "avg_duration_per_task_ms": 86484.18, "avg_output_tokens_per_task": 20778.286666666667, "last_run_timestamp": "2026-10-31T11:37:00.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Friendli", "accuracy": 0.7420636446886446, "accuracy_std_dev": 0.016469967104193704, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.01720449084249084, "avg_duration_per_task_ms": 82040.02197802198, "avg_output_tokens_per_task": 4114.483516483517, "last_run_timestamp": "2026-10-31T12:00:42.348Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "DeepInfra", "accuracy": 0.7420434974874371, "accuracy_std_dev": 0.018804943831343932, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.046914160804020104, "avg_duration_per_task_ms": 685430.3140703518, "avg_output_tokens_per_task": 14198.173366834171, "last_run_timestamp": "2026-10-31T19:48:13.441Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Venice", "accuracy": 0.742, "accuracy_std_dev": 0.02599994871918021, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.143269796, "avg_duration_per_task_ms": 585731.59, "avg_output_tokens_per_task": 25891.308, "last_run_timestamp": "2026-10-31T19:26:38.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "ModelRun", "accuracy": 0.7419049285714285, "accuracy_std_dev": 0.027305278452197487, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.05683837428571428, "avg_duration_per_task_ms": 141278.7842857143, "avg_output_tokens_per_task": 10137.55, "last_run_timestamp": "2026-10-31T19:13:33.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "GMICloud", "accuracy": 0.7418181818181818, "accuracy_std_dev": 0.027279408610685823, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.019760812727272727, "avg_duration_per_task_ms": 152641.17272727273, "avg_output_tokens_per_task": 15736.389090909091, "last_run_timestamp": "2026-11-10T06:11:05.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Phala", "accuracy": 0.741666625, "accuracy_std_dev": 0.025331164930464133, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021842495, "avg_duration_per_task_ms": 141327.99, "avg_output_tokens_per_task": 16250.775, "last_run_timestamp": "2026-10-31T12:24:54.205Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "CoreWeave", "accuracy": 0.741666625, "accuracy_std_dev": 0.030866893899198455, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.021115824999999998, "avg_duration_per_task_ms": 150544.0025, "avg_output_tokens_per_task": 8219.325, "last_run_timestamp": "2026-10-31T18:57:59.813Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Relace", "accuracy": 0.7416481202672606, "accuracy_std_dev": 0.022563192218990436, "run_count": 9, "total_questions": 449, "avg_cost_per_task_usd": 0.21138012694877506, "avg_duration_per_task_ms": 402109.59020044544, "avg_output_tokens_per_task": 9801.538975501113, "last_run_timestamp": "2026-10-31T11:26:03.627Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DigitalOcean", "accuracy": 0.7414841313868613, "accuracy_std_dev": 0.027678383181115592, "run_count": 11, "total_questions": 548, "avg_cost_per_task_usd": 0.04167549270072993, "avg_duration_per_task_ms": 154993.6697080292, "avg_output_tokens_per_task": 8579.950729927008, "last_run_timestamp": "2026-10-31T19:46:31.278Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Fireworks", "accuracy": 0.7413334, "accuracy_std_dev": 0.02436751571744648, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.097602524, "avg_duration_per_task_ms": 217804.16, "avg_output_tokens_per_task": 9920.02, "last_run_timestamp": "2026-10-31T14:40:25.043Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "auto-routing", "accuracy": 0.7413334, "accuracy_std_dev": 0.025086522278705756, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.025878452000000003, "avg_duration_per_task_ms": 132781.264, "avg_output_tokens_per_task": 9514.518, "last_run_timestamp": "2026-10-31T19:14:12.566Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "auto-routing", "accuracy": 0.7413333, "accuracy_std_dev": 0.025438450338218323, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.51193901, "avg_duration_per_task_ms": 192863.266, "avg_output_tokens_per_task": 10096.234, "last_run_timestamp": "2026-10-31T12:10:56.773Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DeepInfra", "accuracy": 0.7413333, "accuracy_std_dev": 0.03095523075685272, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.40647999399999996, "avg_duration_per_task_ms": 588538.78, "avg_output_tokens_per_task": 9896.416, "last_run_timestamp": "2026-10-31T12:27:47.241Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Modal", "accuracy": 0.74125, "accuracy_std_dev": 0.021980774429032325, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.35810955499999997, "avg_duration_per_task_ms": 189257.0575, "avg_output_tokens_per_task": 10848.9475, "last_run_timestamp": "2026-10-31T11:51:20.518Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Claude Platform on AWS", "accuracy": 0.7412121818181818, "accuracy_std_dev": 0.0275340250329006, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.3449351727272727, "avg_duration_per_task_ms": 200135.41454545455, "avg_output_tokens_per_task": 11800.205454545454, "last_run_timestamp": "2026-10-31T13:12:27.229Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Mistral", "accuracy": 0.7411111666666667, "accuracy_std_dev": 0.009749878126018938, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029943926666666662, "avg_duration_per_task_ms": 81311.67, "avg_output_tokens_per_task": 13159.393333333333, "last_run_timestamp": "2026-11-11T03:07:37.121Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Moonshot AI", "accuracy": 0.7411111666666667, "accuracy_std_dev": 0.04950373847975749, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.1105783, "avg_duration_per_task_ms": 224383.69333333333, "avg_output_tokens_per_task": 11802.466666666667, "last_run_timestamp": "2026-10-24T19:31:21.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Nebius", "accuracy": 0.7410254102564102, "accuracy_std_dev": 0.015687179899124656, "run_count": 4, "total_questions": 195, "avg_cost_per_task_usd": 0.010222671794871796, "avg_duration_per_task_ms": 114819.93846153846, "avg_output_tokens_per_task": 3734.005128205128, "last_run_timestamp": "2026-10-31T12:06:01.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Baidu", "accuracy": 0.7409909864864865, "accuracy_std_dev": 0.009883309915655236, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.014433466216216218, "avg_duration_per_task_ms": 89927.77702702703, "avg_output_tokens_per_task": 10790.993243243243, "last_run_timestamp": "2026-11-10T18:11:41.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Groq", "accuracy": 0.7409637389558233, "accuracy_std_dev": 0.023850320811259554, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.12855561847389557, "avg_duration_per_task_ms": 62087.51004016064, "avg_output_tokens_per_task": 9516.100401606425, "last_run_timestamp": "2026-10-31T19:12:18.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "GMICloud", "accuracy": 0.7409525714285714, "accuracy_std_dev": 0.01094194948492327, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.03426931142857143, "avg_duration_per_task_ms": 134514.65142857144, "avg_output_tokens_per_task": 15886.077142857142, "last_run_timestamp": "2026-11-10T06:15:52.514Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "auto-routing", "accuracy": 0.7409522857142857, "accuracy_std_dev": 0.03499266784307123, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0443475, "avg_duration_per_task_ms": 181001.05142857143, "avg_output_tokens_per_task": 9386.614285714286, "last_run_timestamp": "2026-10-31T20:15:42.614Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Baidu", "accuracy": 0.740833375, "accuracy_std_dev": 0.01898462645759391, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06345218250000001, "avg_duration_per_task_ms": 202212.49, "avg_output_tokens_per_task": 11837.4125, "last_run_timestamp": "2026-11-10T19:24:51.597Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Azure", "accuracy": 0.7406666000000001, "accuracy_std_dev": 0.024303563525540885, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.5231801380000001, "avg_duration_per_task_ms": 223642.216, "avg_output_tokens_per_task": 10003.868, "last_run_timestamp": "2026-10-31T13:21:45.027Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Google", "accuracy": 0.7406666000000001, "accuracy_std_dev": 0.026741542381470827, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.5445005100000001, "avg_duration_per_task_ms": 200696.784, "avg_output_tokens_per_task": 9760.559, "last_run_timestamp": "2026-10-31T13:01:06.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Together", "accuracy": 0.7406666000000001, "accuracy_std_dev": 0.018961987708043703, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.014243233999999999, "avg_duration_per_task_ms": 72756.944, "avg_output_tokens_per_task": 4429.298, "last_run_timestamp": "2026-10-31T12:42:18.186Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "AkashML", "accuracy": 0.74014702004008, "accuracy_std_dev": 0.022155950173486136, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.06005350100200401, "avg_duration_per_task_ms": 140448.37274549098, "avg_output_tokens_per_task": 4119.851703406814, "last_run_timestamp": "2026-10-31T12:27:45.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Cloudflare", "accuracy": 0.740000125, "accuracy_std_dev": 0.026034122908394196, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.080340995, "avg_duration_per_task_ms": 330284.965, "avg_output_tokens_per_task": 11392.905, "last_run_timestamp": "2026-10-31T21:26:40.342Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Anthropic", "accuracy": 0.74, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.23335095, "avg_duration_per_task_ms": 96091.26, "avg_output_tokens_per_task": 5555.9, "last_run_timestamp": "2026-10-01T19:04:26.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "auto-routing", "accuracy": 0.74, "accuracy_std_dev": 0.027325104453597256, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.328687276, "avg_duration_per_task_ms": 196120.964, "avg_output_tokens_per_task": 11587.26, "last_run_timestamp": "2026-10-31T12:07:21.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Crusoe", "accuracy": 0.74, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.14729807, "avg_duration_per_task_ms": 386775.27, "avg_output_tokens_per_task": 24280.02, "last_run_timestamp": "2026-10-31T19:58:44.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "AtlasCloud", "accuracy": 0.74, "accuracy_std_dev": 0.03511903567511501, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.023083885, "avg_duration_per_task_ms": 150345.7, "avg_output_tokens_per_task": 16221.6925, "last_run_timestamp": "2026-10-31T11:30:43.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Near AI", "accuracy": 0.74, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01862194, "avg_duration_per_task_ms": 339225.77, "avg_output_tokens_per_task": 12354.38, "last_run_timestamp": "2026-10-31T13:22:50.108Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "model_slug": "google/gemini-3.1-pro-preview", "model_name": "Google: Gemini 3.1 Pro Preview", "provider_name": "Google AI Studio", "accuracy": 0.74, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.37915447, "avg_duration_per_task_ms": 148084.91, "avg_output_tokens_per_task": 12525.4, "last_run_timestamp": "2026-10-01T19:04:55.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "KT", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01977924, "avg_duration_per_task_ms": 63877.22, "avg_output_tokens_per_task": 11452.5, "last_run_timestamp": "2026-11-13T08:16:50.682Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "Novita", "accuracy": 0.74, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00321776, "avg_duration_per_task_ms": 67631.78, "avg_output_tokens_per_task": 11380.95, "last_run_timestamp": "2026-10-01T16:01:02.671Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Nebius", "accuracy": 0.74, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06914866, "avg_duration_per_task_ms": 152605.49, "avg_output_tokens_per_task": 14774.68, "last_run_timestamp": "2026-10-31T12:25:52.830Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "DeepInfra", "accuracy": 0.74, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0873214, "avg_duration_per_task_ms": 347037.32, "avg_output_tokens_per_task": 8891.3, "last_run_timestamp": "2026-08-22T16:12:20.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Sail Research", "accuracy": 0.74, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.049803139999999996, "avg_duration_per_task_ms": 363448.84, "avg_output_tokens_per_task": 8150.99, "last_run_timestamp": "2026-08-29T08:25:32.049Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Reka", "accuracy": 0.74, "accuracy_std_dev": 0.019860653454128848, "run_count": 16, "total_questions": 800, "avg_cost_per_task_usd": 0.01624762875, "avg_duration_per_task_ms": 161764.35125, "avg_output_tokens_per_task": 4237.58, "last_run_timestamp": "2026-10-31T11:33:42.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Makora", "accuracy": 0.7399329261744967, "accuracy_std_dev": 0.017150616176167584, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.24555212751677855, "avg_duration_per_task_ms": 357614.23825503356, "avg_output_tokens_per_task": 12408.355704697986, "last_run_timestamp": "2026-10-31T12:24:39.056Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Venice", "accuracy": 0.739687695652174, "accuracy_std_dev": 0.02325959803799046, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.050316361204013386, "avg_duration_per_task_ms": 654799.304347826, "avg_output_tokens_per_task": 13915.548494983277, "last_run_timestamp": "2026-10-24T19:40:41.765Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "StreamLake", "accuracy": 0.73958325, "accuracy_std_dev": 0.022756993736816403, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02112607, "avg_duration_per_task_ms": 318544.7, "avg_output_tokens_per_task": 11340.385, "last_run_timestamp": "2026-10-17T11:09:59.070Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Amazon Bedrock", "accuracy": 0.7395555333333333, "accuracy_std_dev": 0.016947170784004716, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.013297554666666666, "avg_duration_per_task_ms": 100947.12933333333, "avg_output_tokens_per_task": 10746.964, "last_run_timestamp": "2026-10-31T12:45:41.031Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Alibaba", "accuracy": 0.7394557102040816, "accuracy_std_dev": 0.01129971226890311, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.010599469387755103, "avg_duration_per_task_ms": 125314.78775510204, "avg_output_tokens_per_task": 12060.526530612246, "last_run_timestamp": "2026-10-24T19:21:18.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7393938181818183, "accuracy_std_dev": 0.027627475312607935, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.34960052363636374, "avg_duration_per_task_ms": 219975.07454545455, "avg_output_tokens_per_task": 10861.59090909091, "last_run_timestamp": "2026-10-31T12:46:18.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.7392308461538462, "accuracy_std_dev": 0.0224239080065011, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.017090104615384617, "avg_duration_per_task_ms": 299674.4630769231, "avg_output_tokens_per_task": 22554.26307692308, "last_run_timestamp": "2026-11-04T21:17:38.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Novita", "accuracy": 0.739166625, "accuracy_std_dev": 0.02757564218172218, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.020439862500000003, "avg_duration_per_task_ms": 430063.965, "avg_output_tokens_per_task": 11905.0125, "last_run_timestamp": "2026-10-17T11:29:07.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Relace", "accuracy": 0.738889, "accuracy_std_dev": 0.024695368688615833, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01095846, "avg_duration_per_task_ms": 321436.35333333333, "avg_output_tokens_per_task": 22570.04, "last_run_timestamp": "2026-10-31T12:52:06.048Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "Azure", "accuracy": 0.7388888333333333, "accuracy_std_dev": 0.026504076091654706, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.4952929133333333, "avg_duration_per_task_ms": 170578.00666666665, "avg_output_tokens_per_task": 7379.15, "last_run_timestamp": "2026-10-01T19:05:47.886Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Decart", "accuracy": 0.7386668, "accuracy_std_dev": 0.03166503013357166, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.009092168000000001, "avg_duration_per_task_ms": 101935.708, "avg_output_tokens_per_task": 5027.116, "last_run_timestamp": "2026-10-17T11:17:02.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "auto-routing", "accuracy": 0.7386666000000001, "accuracy_std_dev": 0.027616400924812788, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.077249096, "avg_duration_per_task_ms": 122620.824, "avg_output_tokens_per_task": 8779.518, "last_run_timestamp": "2026-10-31T20:18:12.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "auto-routing", "accuracy": 0.7385184444444445, "accuracy_std_dev": 0.028765060349555692, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.24696524444444445, "avg_duration_per_task_ms": 265213.1511111111, "avg_output_tokens_per_task": 10562.944444444445, "last_run_timestamp": "2026-10-31T11:38:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "DeepInfra", "accuracy": 0.7383335, "accuracy_std_dev": 0.0016665000000000152, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00751252, "avg_duration_per_task_ms": 320365.75, "avg_output_tokens_per_task": 15536.72, "last_run_timestamp": "2026-10-08T22:38:42.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Morph", "accuracy": 0.7380000999999999, "accuracy_std_dev": 0.02844855708274146, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.016056571999999998, "avg_duration_per_task_ms": 534586.188, "avg_output_tokens_per_task": 17634.696, "last_run_timestamp": "2026-10-31T12:03:50.388Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Anthropic", "accuracy": 0.738, "accuracy_std_dev": 0.020232086728758366, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.345152742, "avg_duration_per_task_ms": 207631.326, "avg_output_tokens_per_task": 11819.48, "last_run_timestamp": "2026-10-31T13:36:32.496Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DeepInfra", "accuracy": 0.7379031612903226, "accuracy_std_dev": 0.033855790905545226, "run_count": 5, "total_questions": 248, "avg_cost_per_task_usd": 0.06981228629032259, "avg_duration_per_task_ms": 509430.0443548387, "avg_output_tokens_per_task": 10842.524193548386, "last_run_timestamp": "2026-10-24T19:42:30.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Parasail", "accuracy": 0.7378277303370787, "accuracy_std_dev": 0.033381118228698506, "run_count": 11, "total_questions": 534, "avg_cost_per_task_usd": 0.025130365168539327, "avg_duration_per_task_ms": 93504.49625468165, "avg_output_tokens_per_task": 8311.610486891386, "last_run_timestamp": "2026-11-06T06:42:03.203Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "SiliconFlow", "accuracy": 0.7377779999999999, "accuracy_std_dev": 0.027933050471917086, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02205162, "avg_duration_per_task_ms": 230910.26, "avg_output_tokens_per_task": 11297.92, "last_run_timestamp": "2026-10-31T12:27:40.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cloudflare", "accuracy": 0.7377779999999999, "accuracy_std_dev": 0.020608136952831695, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.009694766666666667, "avg_duration_per_task_ms": 179217.14666666667, "avg_output_tokens_per_task": 12873.993333333334, "last_run_timestamp": "2026-10-31T11:06:39.132Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Parasail", "accuracy": 0.7377776666666667, "accuracy_std_dev": 0.020366893981273733, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.3511924911111111, "avg_duration_per_task_ms": 195777.59333333332, "avg_output_tokens_per_task": 9122.282222222222, "last_run_timestamp": "2026-10-31T11:28:52.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Makora", "accuracy": 0.7376126486486487, "accuracy_std_dev": 0.027927098584230564, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.011551577702702704, "avg_duration_per_task_ms": 61368.75337837838, "avg_output_tokens_per_task": 4262.564189189189, "last_run_timestamp": "2026-10-31T12:25:51.222Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Novita", "accuracy": 0.73750025, "accuracy_std_dev": 0.047015028625828785, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0101418575, "avg_duration_per_task_ms": 462885.0975, "avg_output_tokens_per_task": 15094.72, "last_run_timestamp": "2026-10-17T11:26:35.790Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Morph", "accuracy": 0.737499875, "accuracy_std_dev": 0.022592425906913488, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01732587, "avg_duration_per_task_ms": 193821.065, "avg_output_tokens_per_task": 4493.1475, "last_run_timestamp": "2026-10-31T12:42:00.537Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "model_slug": "anthropic/claude-opus-4.6", "model_name": "Anthropic: Claude Opus 4.6", "provider_name": "Amazon Bedrock", "accuracy": 0.7373335, "accuracy_std_dev": 0.029393808872788148, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.5392687700000001, "avg_duration_per_task_ms": 178582.538, "avg_output_tokens_per_task": 10153.252, "last_run_timestamp": "2026-10-31T12:28:38.525Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Ionstream", "accuracy": 0.7373334, "accuracy_std_dev": 0.020482933819157834, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.039714615999999994, "avg_duration_per_task_ms": 174773.632, "avg_output_tokens_per_task": 27822.08, "last_run_timestamp": "2026-10-21T12:36:51.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "auto-routing", "accuracy": 0.737143, "accuracy_std_dev": 0.024907972114508722, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04305575428571429, "avg_duration_per_task_ms": 613136.2114285714, "avg_output_tokens_per_task": 16198.417142857143, "last_run_timestamp": "2026-10-31T19:47:52.982Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "SiliconFlow", "accuracy": 0.7371428571428571, "accuracy_std_dev": 0.03452293332359381, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05083306285714286, "avg_duration_per_task_ms": 397117.17714285717, "avg_output_tokens_per_task": 11318.654285714285, "last_run_timestamp": "2026-10-31T21:59:10.427Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "GMICloud", "accuracy": 0.7371427142857143, "accuracy_std_dev": 0.04561487206325298, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.006840197142857143, "avg_duration_per_task_ms": 433214.3028571429, "avg_output_tokens_per_task": 14593.82, "last_run_timestamp": "2026-10-10T15:31:23.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "StreamLake", "accuracy": 0.7370185075376885, "accuracy_std_dev": 0.03341638611800355, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.004073236180904522, "avg_duration_per_task_ms": 239891.2864321608, "avg_output_tokens_per_task": 12177.208542713568, "last_run_timestamp": "2026-10-31T19:49:31.140Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Inceptron", "accuracy": 0.7367941894353369, "accuracy_std_dev": 0.02953072467422912, "run_count": 11, "total_questions": 549, "avg_cost_per_task_usd": 0.01213822586520947, "avg_duration_per_task_ms": 405302.62841530057, "avg_output_tokens_per_task": 14294.963570127504, "last_run_timestamp": "2026-10-31T11:58:45.245Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "StreamLake", "accuracy": 0.736667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02084315, "avg_duration_per_task_ms": 478625.81, "avg_output_tokens_per_task": 14704.35, "last_run_timestamp": "2026-08-29T16:31:42.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "auto-routing", "accuracy": 0.736667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.47698523000000004, "avg_duration_per_task_ms": 128774.34, "avg_output_tokens_per_task": 7206.5, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Together", "accuracy": 0.7366666000000001, "accuracy_std_dev": 0.027039148574613085, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.29887413199999996, "avg_duration_per_task_ms": 294262.764, "avg_output_tokens_per_task": 9417.472, "last_run_timestamp": "2026-10-31T13:04:39.070Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Fireworks", "accuracy": 0.7366665, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.17216302, "avg_duration_per_task_ms": 234612.69, "avg_output_tokens_per_task": 17080.8, "last_run_timestamp": "2026-09-19T08:12:02.180Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Phala", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.020005850000000002, "avg_duration_per_task_ms": 329585.74, "avg_output_tokens_per_task": 11689.47, "last_run_timestamp": "2026-08-29T16:43:42.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DeepInfra", "accuracy": 0.7366665, "accuracy_std_dev": 0.036666500000000046, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03538476, "avg_duration_per_task_ms": 350978.71, "avg_output_tokens_per_task": 8276.33, "last_run_timestamp": "2026-08-29T16:11:10.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "Io Net", "accuracy": 0.7366665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02539905, "avg_duration_per_task_ms": 216631.23, "avg_output_tokens_per_task": 14336.27, "last_run_timestamp": "2026-10-24T12:02:36.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "DigitalOcean", "accuracy": 0.7365900689655173, "accuracy_std_dev": 0.040120795309227075, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.07452031609195402, "avg_duration_per_task_ms": 904419.2068965518, "avg_output_tokens_per_task": 14631.514367816091, "last_run_timestamp": "2026-10-24T20:35:30.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Phala", "accuracy": 0.7361905714285714, "accuracy_std_dev": 0.03340122609750018, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.14018482571428573, "avg_duration_per_task_ms": 334656.22, "avg_output_tokens_per_task": 11876.488571428572, "last_run_timestamp": "2026-10-31T21:07:15.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "SiliconFlow", "accuracy": 0.7361904285714286, "accuracy_std_dev": 0.03649010551547657, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07199701142857143, "avg_duration_per_task_ms": 222190.2542857143, "avg_output_tokens_per_task": 9529.64, "last_run_timestamp": "2026-10-31T19:53:34.188Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "AtlasCloud", "accuracy": 0.736111045045045, "accuracy_std_dev": 0.04484669172680976, "run_count": 9, "total_questions": 444, "avg_cost_per_task_usd": 0.02811942117117117, "avg_duration_per_task_ms": 111185.40990990991, "avg_output_tokens_per_task": 10611.61936936937, "last_run_timestamp": "2026-10-31T19:48:53.852Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.736, "accuracy_std_dev": 0.011623577573191504, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.02217724, "avg_duration_per_task_ms": 74896.184, "avg_output_tokens_per_task": 4355.024, "last_run_timestamp": "2026-10-31T12:05:37.260Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "model_slug": "anthropic/claude-sonnet-4.6", "model_name": "Anthropic: Claude Sonnet 4.6", "provider_name": "Google", "accuracy": 0.7359999666666666, "accuracy_std_dev": 0.028627806219924643, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.3641733786666666, "avg_duration_per_task_ms": 196840.98666666666, "avg_output_tokens_per_task": 11882.770666666667, "last_run_timestamp": "2026-10-31T13:17:20.726Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Google", "accuracy": 0.7359999333333334, "accuracy_std_dev": 0.0231964569463145, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.013030309333333335, "avg_duration_per_task_ms": 73246.75466666666, "avg_output_tokens_per_task": 10779.536, "last_run_timestamp": "2026-10-31T11:42:11.484Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Moonshot AI", "accuracy": 0.7359360302267003, "accuracy_std_dev": 0.023828191210399825, "run_count": 8, "total_questions": 397, "avg_cost_per_task_usd": 0.28617903778337533, "avg_duration_per_task_ms": 407022.72040302266, "avg_output_tokens_per_task": 9678.146095717884, "last_run_timestamp": "2026-10-24T12:29:09.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Makora", "accuracy": 0.7359261111111111, "accuracy_std_dev": 0.02318882707227026, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04132428666666666, "avg_duration_per_task_ms": 135748.28, "avg_output_tokens_per_task": 25779.64, "last_run_timestamp": "2026-10-31T12:16:10.183Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "auto-routing", "accuracy": 0.735833375, "accuracy_std_dev": 0.03463102033559473, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.030614205000000002, "avg_duration_per_task_ms": 479244.2275, "avg_output_tokens_per_task": 15408.6125, "last_run_timestamp": "2026-10-17T13:55:17.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "Xiaomi", "accuracy": 0.73583325, "accuracy_std_dev": 0.025590284073012965, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0082457875, "avg_duration_per_task_ms": 453019.37, "avg_output_tokens_per_task": 14752.6375, "last_run_timestamp": "2026-10-17T12:09:02.862Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "NextBit", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.01369867976201439, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01667712, "avg_duration_per_task_ms": 656503.2166666667, "avg_output_tokens_per_task": 11802.333333333334, "last_run_timestamp": "2026-11-03T00:15:40.640Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "model_slug": "openai/gpt-5.5", "model_name": "OpenAI: GPT-5.5", "provider_name": "OpenAI", "accuracy": 0.7355556666666666, "accuracy_std_dev": 0.04157400066013479, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.48569376, "avg_duration_per_task_ms": 125885.67333333334, "avg_output_tokens_per_task": 7296.606666666667, "last_run_timestamp": "2026-10-02T23:08:08.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Relace", "accuracy": 0.7352940588235294, "accuracy_std_dev": 0.03872239054432369, "run_count": 17, "total_questions": 850, "avg_cost_per_task_usd": 0.009413557647058823, "avg_duration_per_task_ms": 215238.48941176472, "avg_output_tokens_per_task": 15440.829411764706, "last_run_timestamp": "2026-11-02T18:06:38.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Novita", "accuracy": 0.7352381428571428, "accuracy_std_dev": 0.029106006893170788, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.051658899999999994, "avg_duration_per_task_ms": 348562.78857142857, "avg_output_tokens_per_task": 9821.065714285714, "last_run_timestamp": "2026-10-31T21:01:26.628Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Fireworks", "accuracy": 0.7349398099062917, "accuracy_std_dev": 0.02455646627618342, "run_count": 15, "total_questions": 747, "avg_cost_per_task_usd": 0.05549164792503348, "avg_duration_per_task_ms": 157974.1485943775, "avg_output_tokens_per_task": 30263.555555555555, "last_run_timestamp": "2026-10-31T12:41:59.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Phala", "accuracy": 0.7349397911646587, "accuracy_std_dev": 0.047428285455095254, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.2969604819277108, "avg_duration_per_task_ms": 386565.9397590361, "avg_output_tokens_per_task": 11351.530120481928, "last_run_timestamp": "2026-10-31T12:42:56.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "Google AI Studio", "accuracy": 0.734694, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.25339910204081634, "avg_duration_per_task_ms": 146531.16326530612, "avg_output_tokens_per_task": 29831.102040816328, "last_run_timestamp": "2026-10-01T08:03:06.861Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Minimax", "accuracy": 0.7346668, "accuracy_std_dev": 0.03066679130525396, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.029810941999999993, "avg_duration_per_task_ms": 101760.856, "avg_output_tokens_per_task": 10584.172, "last_run_timestamp": "2026-10-31T19:09:49.337Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "SiliconFlow", "accuracy": 0.7344446666666666, "accuracy_std_dev": 0.03183677894965017, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07159142333333333, "avg_duration_per_task_ms": 344988.6066666667, "avg_output_tokens_per_task": 11458.973333333333, "last_run_timestamp": "2026-10-24T22:27:21.158Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "SiliconFlow", "accuracy": 0.7344443333333334, "accuracy_std_dev": 0.019404823126108514, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.056632510000000004, "avg_duration_per_task_ms": 908364.2133333334, "avg_output_tokens_per_task": 15944.676666666666, "last_run_timestamp": "2026-10-31T20:02:53.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "CoreWeave", "accuracy": 0.7342859999999999, "accuracy_std_dev": 0.027701017933849065, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.08147224571428571, "avg_duration_per_task_ms": 77662.75714285714, "avg_output_tokens_per_task": 11731.842857142858, "last_run_timestamp": "2026-10-31T19:56:16.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Novita", "accuracy": 0.7342857142857143, "accuracy_std_dev": 0.03331958219903675, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09951040285714285, "avg_duration_per_task_ms": 257635.84285714285, "avg_output_tokens_per_task": 12551.72, "last_run_timestamp": "2026-10-31T20:07:04.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "auto-routing", "accuracy": 0.7340741111111111, "accuracy_std_dev": 0.05052255713253118, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.009341564444444445, "avg_duration_per_task_ms": 247292.54444444444, "avg_output_tokens_per_task": 15978.137777777778, "last_run_timestamp": "2026-10-31T11:40:30.784Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Inceptron", "accuracy": 0.7338094285714286, "accuracy_std_dev": 0.034752133656416974, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06415493714285714, "avg_duration_per_task_ms": 159387.71142857143, "avg_output_tokens_per_task": 11195.86, "last_run_timestamp": "2026-10-31T19:24:16.029Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "model_slug": "qwen/qwen3.5-397b-a17b", "model_name": "Qwen: Qwen3.5 397B A17B", "provider_name": "Novita", "accuracy": 0.733796, "accuracy_std_dev": 0.0059017228557995535, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.20467558333333333, "avg_duration_per_task_ms": 370954.7361111111, "avg_output_tokens_per_task": 23212.9375, "last_run_timestamp": "2026-10-24T19:18:46.938Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Mistral", "accuracy": 0.73375, "accuracy_std_dev": 0.027810549558575794, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.018745945, "avg_duration_per_task_ms": 93462.3375, "avg_output_tokens_per_task": 4950.955, "last_run_timestamp": "2026-11-11T03:16:45.253Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "auto-routing", "accuracy": 0.7336684371859297, "accuracy_std_dev": 0.0449684893006689, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.033184525125628146, "avg_duration_per_task_ms": 1207877.3844221106, "avg_output_tokens_per_task": 52852.572864321606, "last_run_timestamp": "2026-10-31T20:03:31.183Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "Novita", "accuracy": 0.7333335, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06130771, "avg_duration_per_task_ms": 487797.15, "avg_output_tokens_per_task": 14346.49, "last_run_timestamp": "2026-08-29T16:33:37.924Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google", "accuracy": 0.7333335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12301775, "avg_duration_per_task_ms": 155307.55, "avg_output_tokens_per_task": 18812.75, "last_run_timestamp": "2026-10-01T19:05:47.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Together", "accuracy": 0.7333335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08901925000000001, "avg_duration_per_task_ms": 93612.32, "avg_output_tokens_per_task": 8634.62, "last_run_timestamp": "2026-10-31T11:29:29.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Phala", "accuracy": 0.7333335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08394800999999999, "avg_duration_per_task_ms": 134148.76, "avg_output_tokens_per_task": 8723.94, "last_run_timestamp": "2026-10-17T19:55:39.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "auto-routing", "accuracy": 0.7333335, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12687428, "avg_duration_per_task_ms": 110480.49, "avg_output_tokens_per_task": 5152.54, "last_run_timestamp": "2026-10-01T19:04:55.696Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "AtlasCloud", "accuracy": 0.7333334285714286, "accuracy_std_dev": 0.03342828381500203, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10974002285714286, "avg_duration_per_task_ms": 171262.26571428572, "avg_output_tokens_per_task": 9719.968571428572, "last_run_timestamp": "2026-10-31T20:27:27.677Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Amazon Bedrock", "accuracy": 0.7333334, "accuracy_std_dev": 0.03451234883690184, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.30572972400000004, "avg_duration_per_task_ms": 282521.796, "avg_output_tokens_per_task": 10813.856, "last_run_timestamp": "2026-10-31T12:51:11.835Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "StreamLake", "accuracy": 0.733333375, "accuracy_std_dev": 0.03109126351138492, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.06630269500000001, "avg_duration_per_task_ms": 258633.95, "avg_output_tokens_per_task": 9560.47, "last_run_timestamp": "2026-10-31T20:31:43.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Mancer 2", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06451945333333334, "avg_duration_per_task_ms": 876085.0533333333, "avg_output_tokens_per_task": 15144.64, "last_run_timestamp": "2026-10-31T12:31:12.958Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Crusoe", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.03569390445004426, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.021479246666666667, "avg_duration_per_task_ms": 120781.28, "avg_output_tokens_per_task": 11581.666666666666, "last_run_timestamp": "2026-10-31T11:22:01.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "SiliconFlow", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.06834903523337645, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.15942455777777778, "avg_duration_per_task_ms": 1424663.4244444445, "avg_output_tokens_per_task": 28406.333333333332, "last_run_timestamp": "2026-10-24T20:45:53.033Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "GMICloud", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.035136551965154465, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04706574222222222, "avg_duration_per_task_ms": 216014.41555555555, "avg_output_tokens_per_task": 10077.462222222222, "last_run_timestamp": "2026-11-10T07:12:51.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "InferenceNet", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.05760637824832321, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04898614, "avg_duration_per_task_ms": 106829.63333333333, "avg_output_tokens_per_task": 9897.326666666666, "last_run_timestamp": "2026-10-10T11:51:48.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "model_slug": "z-ai/glm-5.3", "model_name": "Z.ai: GLM 5.3", "provider_name": "Amazon Bedrock", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.024944382578492966, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09260214, "avg_duration_per_task_ms": 326558.9733333333, "avg_output_tokens_per_task": 4729.153333333334, "last_run_timestamp": "2026-10-31T12:28:59.426Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Cohere", "accuracy": 0.7333333333333333, "accuracy_std_dev": 0.009428090415820642, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.024898453333333334, "avg_duration_per_task_ms": 37126.9, "avg_output_tokens_per_task": 4153.473333333333, "last_run_timestamp": "2026-10-31T12:07:50.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Relace", "accuracy": 0.73333325, "accuracy_std_dev": 0.0368176738970226, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.013768695000000003, "avg_duration_per_task_ms": 81021.62, "avg_output_tokens_per_task": 13622.935, "last_run_timestamp": "2026-11-01T12:08:57.072Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Anthropic", "accuracy": 0.7333332, "accuracy_std_dev": 0.032659904062320846, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.012294704000000002, "avg_duration_per_task_ms": 85675.876, "avg_output_tokens_per_task": 10698.828, "last_run_timestamp": "2026-10-31T11:35:21.238Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "NextBit", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01186947, "avg_duration_per_task_ms": 176514.74, "avg_output_tokens_per_task": 14201.61, "last_run_timestamp": "2026-08-29T16:50:07.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Decart", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00602072, "avg_duration_per_task_ms": 517268.78, "avg_output_tokens_per_task": 18678.62, "last_run_timestamp": "2026-08-22T08:38:00.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.058005, "avg_duration_per_task_ms": 191243.62, "avg_output_tokens_per_task": 30812.32, "last_run_timestamp": "2026-09-12T08:09:12.993Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DekaLLM", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01115704, "avg_duration_per_task_ms": 102700.12, "avg_output_tokens_per_task": 12077.2, "last_run_timestamp": "2026-11-04T09:13:50.416Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "auto-routing", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12365041999999998, "avg_duration_per_task_ms": 156560.04, "avg_output_tokens_per_task": 19704.13, "last_run_timestamp": "2026-10-01T19:04:26.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "model_slug": "google/gemini-3.5-flash-lite", "model_name": "Google: Gemini 3.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10928178, "avg_duration_per_task_ms": 75479.44, "avg_output_tokens_per_task": 18824.05, "last_run_timestamp": "2026-10-01T19:05:46.177Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Parasail", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2484202, "avg_duration_per_task_ms": 207733.62, "avg_output_tokens_per_task": 10608.24, "last_run_timestamp": "2026-11-04T21:15:40.182Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Together", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.2865894, "avg_duration_per_task_ms": 249378.06, "avg_output_tokens_per_task": 16191.68, "last_run_timestamp": "2026-11-04T18:44:37.678Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "AtlasCloud", "accuracy": 0.7333330000000001, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.055889249999999994, "avg_duration_per_task_ms": 201945.25, "avg_output_tokens_per_task": 9141.8, "last_run_timestamp": "2026-10-04T00:00:34.896Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flashx-20260918", "model_slug": "z-ai/glm-5.3-flashx", "model_name": "Z.ai: GLM 5.3 FlashX", "provider_name": "Novita", "accuracy": 0.7333330000000001, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.027065839999999997, "avg_duration_per_task_ms": 104408.46, "avg_output_tokens_per_task": 4213.84, "last_run_timestamp": "2026-11-12T21:20:43.637Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Sail Research", "accuracy": 0.7330556666666667, "accuracy_std_dev": 0.016468863254301695, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.1880071, "avg_duration_per_task_ms": 216665.43166666667, "avg_output_tokens_per_task": 9600.853333333333, "last_run_timestamp": "2026-11-06T22:35:15.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Moonshot AI", "accuracy": 0.733019156654888, "accuracy_std_dev": 0.02874817787961395, "run_count": 17, "total_questions": 849, "avg_cost_per_task_usd": 0.12840728386336864, "avg_duration_per_task_ms": 141161.32508833922, "avg_output_tokens_per_task": 9290.04593639576, "last_run_timestamp": "2026-10-24T19:48:18.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Tenstorrent", "accuracy": 0.7329929795918368, "accuracy_std_dev": 0.05409699999999995, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.003721785714285715, "avg_duration_per_task_ms": 82527.76530612246, "avg_output_tokens_per_task": 3412.2551020408164, "last_run_timestamp": "2026-11-08T04:25:13.787Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Modal", "accuracy": 0.732916625, "accuracy_std_dev": 0.018888392049996622, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0482453475, "avg_duration_per_task_ms": 162563.185, "avg_output_tokens_per_task": 29218.5525, "last_run_timestamp": "2026-10-31T12:25:59.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Inceptron", "accuracy": 0.7325939704251386, "accuracy_std_dev": 0.03159346942038068, "run_count": 11, "total_questions": 541, "avg_cost_per_task_usd": 0.06591540850277265, "avg_duration_per_task_ms": 143626.1885397412, "avg_output_tokens_per_task": 8437.54898336414, "last_run_timestamp": "2026-10-31T19:10:53.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Z.AI", "accuracy": 0.732381, "accuracy_std_dev": 0.016106167044245587, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.11714905142857145, "avg_duration_per_task_ms": 334799.0685714286, "avg_output_tokens_per_task": 9713.528571428571, "last_run_timestamp": "2026-10-31T20:34:53.249Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Parasail", "accuracy": 0.7321772312138729, "accuracy_std_dev": 0.08403648462496778, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.1502556156069364, "avg_duration_per_task_ms": 2018284.2369942197, "avg_output_tokens_per_task": 112906.43641618497, "last_run_timestamp": "2026-10-31T19:11:32.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "CoreWeave", "accuracy": 0.732, "accuracy_std_dev": 0.02124973202654565, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.036015575999999994, "avg_duration_per_task_ms": 148623.836, "avg_output_tokens_per_task": 32577.608, "last_run_timestamp": "2026-10-31T12:40:24.644Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "auto-routing", "accuracy": 0.73166675, "accuracy_std_dev": 0.04146614750537478, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.010049615000000001, "avg_duration_per_task_ms": 411186.61, "avg_output_tokens_per_task": 14357.37, "last_run_timestamp": "2026-10-17T13:13:33.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Crusoe", "accuracy": 0.7316665, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01274185, "avg_duration_per_task_ms": 154997.2, "avg_output_tokens_per_task": 13696.96, "last_run_timestamp": "2026-10-31T19:54:14.421Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "auto-routing", "accuracy": 0.7314284285714286, "accuracy_std_dev": 0.019424805892445475, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06280418, "avg_duration_per_task_ms": 232822.65142857144, "avg_output_tokens_per_task": 12506.968571428572, "last_run_timestamp": "2026-10-31T20:18:31.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Azure", "accuracy": 0.7313332, "accuracy_std_dev": 0.020231941744676914, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.012560144, "avg_duration_per_task_ms": 74716.332, "avg_output_tokens_per_task": 10525.228, "last_run_timestamp": "2026-10-31T11:19:23.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "DigitalOcean", "accuracy": 0.7311827782258065, "accuracy_std_dev": 0.07837008853695393, "run_count": 10, "total_questions": 496, "avg_cost_per_task_usd": 0.009883709677419354, "avg_duration_per_task_ms": 170348.1431451613, "avg_output_tokens_per_task": 4275.5161290322585, "last_run_timestamp": "2026-10-31T12:31:51.998Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Wafer", "accuracy": 0.7311637573964497, "accuracy_std_dev": 0.02813377440238986, "run_count": 17, "total_questions": 845, "avg_cost_per_task_usd": 0.024811487573964494, "avg_duration_per_task_ms": 209572.125443787, "avg_output_tokens_per_task": 21468.289940828403, "last_run_timestamp": "2026-10-31T11:27:16.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "auto-routing", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.029731211518088756, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012172700000000002, "avg_duration_per_task_ms": 217797.45666666667, "avg_output_tokens_per_task": 14731.27, "last_run_timestamp": "2026-10-31T19:46:05.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "OpenInference", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.020608136952831695, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028709900000000003, "avg_duration_per_task_ms": 638312.6933333334, "avg_output_tokens_per_task": 61738.18, "last_run_timestamp": "2026-10-31T12:28:49.803Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Azure", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.014740588127118051, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.29708465, "avg_duration_per_task_ms": 186510.67, "avg_output_tokens_per_task": 10592.956666666667, "last_run_timestamp": "2026-10-01T19:05:18.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "OpenAI", "accuracy": 0.7311110000000001, "accuracy_std_dev": 0.013698950130089036, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12928292, "avg_duration_per_task_ms": 109953.54, "avg_output_tokens_per_task": 5131.753333333333, "last_run_timestamp": "2026-10-02T23:08:37.533Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "Xiaomi", "accuracy": 0.730833375, "accuracy_std_dev": 0.028074913427905263, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.025226807499999997, "avg_duration_per_task_ms": 435604.5225, "avg_output_tokens_per_task": 16317.23, "last_run_timestamp": "2026-10-17T11:15:55.168Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Crusoe", "accuracy": 0.7304761428571429, "accuracy_std_dev": 0.032043090320150264, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09725585142857143, "avg_duration_per_task_ms": 317413.97142857144, "avg_output_tokens_per_task": 11001.088571428572, "last_run_timestamp": "2026-10-31T20:47:51.733Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "GMICloud", "accuracy": 0.7303704444444444, "accuracy_std_dev": 0.04158720192054578, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.028855546666666666, "avg_duration_per_task_ms": 158061.0911111111, "avg_output_tokens_per_task": 13689.44, "last_run_timestamp": "2026-11-10T06:43:24.233Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Google", "accuracy": 0.7303333, "accuracy_std_dev": 0.025427430945929256, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.3636253240000001, "avg_duration_per_task_ms": 225306.613, "avg_output_tokens_per_task": 10854.177, "last_run_timestamp": "2026-10-31T12:43:31.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Reka", "accuracy": 0.7300002499999999, "accuracy_std_dev": 0.028867484592314224, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03407485, "avg_duration_per_task_ms": 264292.92, "avg_output_tokens_per_task": 34254.08, "last_run_timestamp": "2026-10-31T13:21:05.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "Novita", "accuracy": 0.73, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06303523999999999, "avg_duration_per_task_ms": 517277.01, "avg_output_tokens_per_task": 14652.21, "last_run_timestamp": "2026-08-30T00:01:09.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "auto-routing", "accuracy": 0.73, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06290177, "avg_duration_per_task_ms": 485845.77, "avg_output_tokens_per_task": 14483.09, "last_run_timestamp": "2026-08-29T16:22:37.842Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Nebius", "accuracy": 0.73, "accuracy_std_dev": 0.0268328157299975, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.074570136, "avg_duration_per_task_ms": 246781.984, "avg_output_tokens_per_task": 30037.412, "last_run_timestamp": "2026-10-31T11:52:51.154Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "model_slug": "google/gemini-3-flash-preview", "model_name": "Google: Gemini 3 Flash Preview", "provider_name": "Google AI Studio", "accuracy": 0.73, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12856867000000002, "avg_duration_per_task_ms": 121477.86, "avg_output_tokens_per_task": 19799.68, "last_run_timestamp": "2026-10-01T19:05:00.104Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "model_slug": "google/gemini-3.6-flash", "model_name": "Google: Gemini 3.6 Flash", "provider_name": "Google AI Studio", "accuracy": 0.73, "accuracy_std_dev": 0.04333300000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.24468226999999998, "avg_duration_per_task_ms": 158524.06, "avg_output_tokens_per_task": 22552.35, "last_run_timestamp": "2026-10-01T19:04:55.721Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Sail Research", "accuracy": 0.73, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.22321662, "avg_duration_per_task_ms": 277142.38, "avg_output_tokens_per_task": 10411.9, "last_run_timestamp": "2026-11-06T22:14:35.733Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3.1-pro-20261023", "model_slug": "moonshotai/kimi-k3.1-pro", "model_name": "MoonshotAI: Kimi K3.1 Pro", "provider_name": "Modal", "accuracy": 0.73, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14279748, "avg_duration_per_task_ms": 129435.32, "avg_output_tokens_per_task": 7981.12, "last_run_timestamp": "2026-11-04T21:11:46.281Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "DigitalOcean", "accuracy": 0.73, "accuracy_std_dev": 0.036667000000000005, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06843740999999999, "avg_duration_per_task_ms": 1343672.46, "avg_output_tokens_per_task": 8825.5, "last_run_timestamp": "2026-08-29T16:20:58.762Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "KT", "accuracy": 0.73, "accuracy_std_dev": 0.0033330000000000304, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01420625, "avg_duration_per_task_ms": 63360.51, "avg_output_tokens_per_task": 4011.33, "last_run_timestamp": "2026-11-11T02:06:30.445Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google", "accuracy": 0.7299997500000001, "accuracy_std_dev": 0.017320556188745798, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.32059346000000005, "avg_duration_per_task_ms": 156347.09, "avg_output_tokens_per_task": 21297.645, "last_run_timestamp": "2026-10-01T19:05:00.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Nebius", "accuracy": 0.729654525083612, "accuracy_std_dev": 0.01804542249436126, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.18312612040133777, "avg_duration_per_task_ms": 543481.8829431438, "avg_output_tokens_per_task": 8776.65551839465, "last_run_timestamp": "2026-10-31T21:08:40.848Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "Claude Platform on AWS", "accuracy": 0.7293334, "accuracy_std_dev": 0.026533159130416407, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.012403696, "avg_duration_per_task_ms": 92314.648, "avg_output_tokens_per_task": 10764.472, "last_run_timestamp": "2026-10-31T12:41:01.831Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Together", "accuracy": 0.7291665, "accuracy_std_dev": 0.023318429765959808, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0316769325, "avg_duration_per_task_ms": 89151.6375, "avg_output_tokens_per_task": 22436.51, "last_run_timestamp": "2026-10-31T11:37:01.556Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "ModelRun", "accuracy": 0.7288890000000001, "accuracy_std_dev": 0.03142665378305492, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02386675333333333, "avg_duration_per_task_ms": 151228.52, "avg_output_tokens_per_task": 22554.02666666667, "last_run_timestamp": "2026-10-28T06:40:49.113Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Novita", "accuracy": 0.728889, "accuracy_std_dev": 0.01911599421078241, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.017266966666666664, "avg_duration_per_task_ms": 122400.14, "avg_output_tokens_per_task": 11879.513333333334, "last_run_timestamp": "2026-10-31T11:33:22.952Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Together", "accuracy": 0.728889, "accuracy_std_dev": 0.01257094435593444, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04855964666666667, "avg_duration_per_task_ms": 115027.81333333334, "avg_output_tokens_per_task": 15070.966666666667, "last_run_timestamp": "2026-10-31T12:30:34.970Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "auto-routing", "accuracy": 0.7288886666666667, "accuracy_std_dev": 0.017497738831695454, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.033088113333333335, "avg_duration_per_task_ms": 134354.10666666666, "avg_output_tokens_per_task": 14586.926666666666, "last_run_timestamp": "2026-10-31T12:32:08.127Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "PrimeIntellect", "accuracy": 0.7283335, "accuracy_std_dev": 0.024626886091898288, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04604961333333333, "avg_duration_per_task_ms": 118630.39, "avg_output_tokens_per_task": 26239.556666666667, "last_run_timestamp": "2026-10-31T12:19:28.999Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Parasail", "accuracy": 0.7283333333333334, "accuracy_std_dev": 0.03857423338908441, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07535822, "avg_duration_per_task_ms": 168347.59666666668, "avg_output_tokens_per_task": 11190.87, "last_run_timestamp": "2026-10-31T20:16:51.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "model_slug": "anthropic/claude-haiku-5.5", "model_name": "Anthropic: Claude Haiku 5.5", "provider_name": "auto-routing", "accuracy": 0.72833325, "accuracy_std_dev": 0.041197373838480274, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.011627464999999998, "avg_duration_per_task_ms": 86463.365, "avg_output_tokens_per_task": 10669.235, "last_run_timestamp": "2026-10-31T12:31:32.731Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Phala", "accuracy": 0.72833325, "accuracy_std_dev": 0.03411588502717615, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.17819147999999999, "avg_duration_per_task_ms": 442725.015, "avg_output_tokens_per_task": 9652.53, "last_run_timestamp": "2026-10-31T22:21:14.715Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Friendli", "accuracy": 0.728, "accuracy_std_dev": 0.018086336157442175, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10671685199999999, "avg_duration_per_task_ms": 361548.212, "avg_output_tokens_per_task": 14424.78, "last_run_timestamp": "2026-10-17T19:16:53.251Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "PrimeIntellect", "accuracy": 0.728, "accuracy_std_dev": 0.02362657637492154, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.027497304, "avg_duration_per_task_ms": 93734.552, "avg_output_tokens_per_task": 12939.792, "last_run_timestamp": "2026-10-31T12:24:43.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Venice", "accuracy": 0.728, "accuracy_std_dev": 0.03194425700497665, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.080962124, "avg_duration_per_task_ms": 531036.2, "avg_output_tokens_per_task": 12180.928, "last_run_timestamp": "2026-10-24T23:08:37.435Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Relace", "accuracy": 0.7279999333333333, "accuracy_std_dev": 0.024092455251846428, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.024600684000000005, "avg_duration_per_task_ms": 183990.44, "avg_output_tokens_per_task": 23628.08, "last_run_timestamp": "2026-11-02T17:49:35.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Alibaba", "accuracy": 0.7277167582417582, "accuracy_std_dev": 0.03220902548783545, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.0777849358974359, "avg_duration_per_task_ms": 190736.38461538462, "avg_output_tokens_per_task": 8353.329670329671, "last_run_timestamp": "2026-10-31T19:01:31.914Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Baidu", "accuracy": 0.7275001249999999, "accuracy_std_dev": 0.027373361190386816, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.065442425, "avg_duration_per_task_ms": 198229.1, "avg_output_tokens_per_task": 13024.92, "last_run_timestamp": "2026-11-10T19:01:19.915Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "Fireworks", "accuracy": 0.7271345697356426, "accuracy_std_dev": 0.02666397891606509, "run_count": 22, "total_questions": 1097, "avg_cost_per_task_usd": 0.4311616608933454, "avg_duration_per_task_ms": 251082.97538742024, "avg_output_tokens_per_task": 10876.747493163171, "last_run_timestamp": "2026-10-31T12:24:25.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "BaseTen", "accuracy": 0.7270409540816326, "accuracy_std_dev": 0.02754153279934688, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.04606436989795917, "avg_duration_per_task_ms": 81576.88520408163, "avg_output_tokens_per_task": 19891.04081632653, "last_run_timestamp": "2026-10-31T12:26:53.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "River", "accuracy": 0.7268042474226803, "accuracy_std_dev": 0.04829799999999995, "run_count": 2, "total_questions": 97, "avg_cost_per_task_usd": 0.005463453608247422, "avg_duration_per_task_ms": 80743.0412371134, "avg_output_tokens_per_task": 7162.484536082474, "last_run_timestamp": "2026-11-12T20:24:40.836Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "model_slug": "deepseek/deepseek-v4-pro", "model_name": "DeepSeek: DeepSeek V4 Pro 0423", "provider_name": "Nebius", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07819403999999999, "avg_duration_per_task_ms": 528014.2, "avg_output_tokens_per_task": 10299.58, "last_run_timestamp": "2026-11-03T22:19:06.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Nebius", "accuracy": 0.726667, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.24019618, "avg_duration_per_task_ms": 932058.35, "avg_output_tokens_per_task": 24816.04, "last_run_timestamp": "2026-10-31T11:58:06.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Inferact vLLM", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00647162, "avg_duration_per_task_ms": 74169.8, "avg_output_tokens_per_task": 10062.22, "last_run_timestamp": "2026-11-13T21:15:53.722Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Fireworks", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.21341358, "avg_duration_per_task_ms": 101630.84, "avg_output_tokens_per_task": 8827.48, "last_run_timestamp": "2026-08-26T21:00:03.913Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "auto-routing", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 2.91364046, "avg_duration_per_task_ms": 331183.04, "avg_output_tokens_per_task": 12422.4, "last_run_timestamp": "2026-10-01T08:02:52.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "Alibaba", "accuracy": 0.726667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05448868, "avg_duration_per_task_ms": 350679.6, "avg_output_tokens_per_task": 17629, "last_run_timestamp": "2026-10-01T08:04:38.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Sail Research", "accuracy": 0.72666675, "accuracy_std_dev": 0.014907082584714534, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0051741124999999995, "avg_duration_per_task_ms": 172373.7025, "avg_output_tokens_per_task": 11668.895, "last_run_timestamp": "2026-11-06T22:18:15.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Novita", "accuracy": 0.7266667, "accuracy_std_dev": 0.04442226168724414, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.028172072, "avg_duration_per_task_ms": 117004.198, "avg_output_tokens_per_task": 10046.564, "last_run_timestamp": "2026-10-31T19:22:00.212Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Novita", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.024944382578492966, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01469154, "avg_duration_per_task_ms": 161228.33333333334, "avg_output_tokens_per_task": 15812.58, "last_run_timestamp": "2026-10-03T16:15:22.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "OpenAI", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.014401903168987377, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.30249026, "avg_duration_per_task_ms": 183822.13333333333, "avg_output_tokens_per_task": 10747.233333333334, "last_run_timestamp": "2026-10-02T23:13:07.333Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "OpenAI", "accuracy": 0.7266666666666667, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013071513333333333, "avg_duration_per_task_ms": 97597.57333333333, "avg_output_tokens_per_task": 4935.306666666666, "last_run_timestamp": "2026-10-02T23:07:05.152Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Venice", "accuracy": 0.726666625, "accuracy_std_dev": 0.030550477356898618, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02624409, "avg_duration_per_task_ms": 262347.155, "avg_output_tokens_per_task": 8877.63, "last_run_timestamp": "2026-10-24T19:14:23.110Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google", "accuracy": 0.7266665714285715, "accuracy_std_dev": 0.02225403120372413, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09929396571428571, "avg_duration_per_task_ms": 163639.28, "avg_output_tokens_per_task": 52902.33142857143, "last_run_timestamp": "2026-10-01T19:06:49.017Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Modal", "accuracy": 0.7266665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018476000000000003, "avg_duration_per_task_ms": 104934.72, "avg_output_tokens_per_task": 10917.83, "last_run_timestamp": "2026-10-24T11:48:43.775Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "auto-routing", "accuracy": 0.7266665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.43167194999999997, "avg_duration_per_task_ms": 195075.98, "avg_output_tokens_per_task": 31329.14, "last_run_timestamp": "2026-10-01T19:04:40.362Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "model_slug": "tencent/hy4-preview", "model_name": "Tencent: Hy4 preview", "provider_name": "Io Net", "accuracy": 0.7266665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06725601, "avg_duration_per_task_ms": 640810.38, "avg_output_tokens_per_task": 15599.02, "last_run_timestamp": "2026-10-27T14:13:59.381Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "CoreWeave", "accuracy": 0.7266663333333333, "accuracy_std_dev": 0.018856180831641284, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.024500659999999997, "avg_duration_per_task_ms": 72543.00666666667, "avg_output_tokens_per_task": 15175.193333333333, "last_run_timestamp": "2026-10-31T11:34:42.984Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Nebius", "accuracy": 0.7264369310344827, "accuracy_std_dev": 0.006057869592521776, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.012712737931034484, "avg_duration_per_task_ms": 160320.86896551723, "avg_output_tokens_per_task": 8003.303448275862, "last_run_timestamp": "2026-10-31T12:11:53.396Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Darkbloom", "accuracy": 0.7261904081632652, "accuracy_std_dev": 0.026678340975171223, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.016112545918367346, "avg_duration_per_task_ms": 812871.9897959183, "avg_output_tokens_per_task": 11587.295918367347, "last_run_timestamp": "2026-11-13T04:05:40.636Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Cloudflare", "accuracy": 0.7260605454545455, "accuracy_std_dev": 0.026431521795571147, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07336239636363638, "avg_duration_per_task_ms": 233666.03636363638, "avg_output_tokens_per_task": 8533.292727272727, "last_run_timestamp": "2026-10-31T20:10:58.659Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StepFun", "accuracy": 0.7255556666666666, "accuracy_std_dev": 0.016960557004087128, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012908883333333333, "avg_duration_per_task_ms": 53544.27, "avg_output_tokens_per_task": 10836.613333333333, "last_run_timestamp": "2026-11-02T17:45:53.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Venice", "accuracy": 0.7253333, "accuracy_std_dev": 0.02454917785609123, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.09142225, "avg_duration_per_task_ms": 693347.86, "avg_output_tokens_per_task": 9534.488, "last_run_timestamp": "2026-10-31T19:40:42.793Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "model_slug": "openai/gpt-5.6-sol", "model_name": "OpenAI: GPT-5.6 Sol", "provider_name": "Azure", "accuracy": 0.725, "accuracy_std_dev": 0.019220966564318938, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.30439079, "avg_duration_per_task_ms": 120258.43333333333, "avg_output_tokens_per_task": 5186.016666666666, "last_run_timestamp": "2026-10-01T19:05:16.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "Particle AI", "accuracy": 0.725, "accuracy_std_dev": 0.017240198505237704, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01070377, "avg_duration_per_task_ms": 64833.36, "avg_output_tokens_per_task": 4228.4, "last_run_timestamp": "2026-11-03T02:02:55.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "StreamLake", "accuracy": 0.7244444444444444, "accuracy_std_dev": 0.022662439694648512, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.06561171777777777, "avg_duration_per_task_ms": 200024.29111111112, "avg_output_tokens_per_task": 9471.135555555555, "last_run_timestamp": "2026-10-31T20:17:42.375Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Phala", "accuracy": 0.7244443333333334, "accuracy_std_dev": 0.006285236475706843, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018751046666666667, "avg_duration_per_task_ms": 183683.96666666667, "avg_output_tokens_per_task": 11588.8, "last_run_timestamp": "2026-10-31T11:59:31.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "OpenAI", "accuracy": 0.7244443333333334, "accuracy_std_dev": 0.02199873172606905, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12152117999999999, "avg_duration_per_task_ms": 471173.63333333336, "avg_output_tokens_per_task": 54771.27333333333, "last_run_timestamp": "2026-10-02T23:27:09.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-2.4t-a95b-20260812", "model_slug": "qwen/qwen3.8-2.4t-a95b", "model_name": "Qwen: Qwen3.8 2.4T A95B", "provider_name": "Venice", "accuracy": 0.724, "accuracy_std_dev": 0.02351363670723862, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.26574487199999997, "avg_duration_per_task_ms": 177434.862, "avg_output_tokens_per_task": 11402.85, "last_run_timestamp": "2026-10-31T12:56:09.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Z.AI", "accuracy": 0.724, "accuracy_std_dev": 0.028158109155268212, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.05089722, "avg_duration_per_task_ms": 400509.008, "avg_output_tokens_per_task": 9631.752, "last_run_timestamp": "2026-10-24T20:04:11.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "SiliconFlow", "accuracy": 0.7238094285714286, "accuracy_std_dev": 0.03062471250410662, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.1589893657142857, "avg_duration_per_task_ms": 227082.73428571428, "avg_output_tokens_per_task": 9576.722857142857, "last_run_timestamp": "2026-10-31T19:57:59.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "Makora", "accuracy": 0.723360643442623, "accuracy_std_dev": 0.019132270021092652, "run_count": 5, "total_questions": 244, "avg_cost_per_task_usd": 0.0756424631147541, "avg_duration_per_task_ms": 76631.30327868853, "avg_output_tokens_per_task": 8337.151639344262, "last_run_timestamp": "2026-10-31T11:46:18.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "auto-routing", "accuracy": 0.7233335, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11131322, "avg_duration_per_task_ms": 83286.18, "avg_output_tokens_per_task": 3736.91, "last_run_timestamp": "2026-10-01T19:04:40.360Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "StreamLake", "accuracy": 0.7233335, "accuracy_std_dev": 0.04333349999999997, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.11253119, "avg_duration_per_task_ms": 335490.34, "avg_output_tokens_per_task": 10275.49, "last_run_timestamp": "2026-08-30T00:01:09.490Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "InferenceNet", "accuracy": 0.723333375, "accuracy_std_dev": 0.026457591854406843, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.007071109999999999, "avg_duration_per_task_ms": 91771.1525, "avg_output_tokens_per_task": 9121.6525, "last_run_timestamp": "2026-11-04T21:12:04.761Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Azure", "accuracy": 0.7233333, "accuracy_std_dev": 0.031162630755602133, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.3478321060000001, "avg_duration_per_task_ms": 232644.894, "avg_output_tokens_per_task": 10841.002, "last_run_timestamp": "2026-10-31T12:28:19.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "auto-routing", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02918011, "avg_duration_per_task_ms": 177614.61, "avg_output_tokens_per_task": 35023.7, "last_run_timestamp": "2026-10-31T11:38:56.927Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "model_slug": "openai/gpt-5.4", "model_name": "OpenAI: GPT-5.4", "provider_name": "Amazon Bedrock", "accuracy": 0.7233330000000001, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.33860155, "avg_duration_per_task_ms": 113561.71, "avg_output_tokens_per_task": 10711.4, "last_run_timestamp": "2026-10-01T19:04:58.091Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "auto-routing", "accuracy": 0.7228916987951808, "accuracy_std_dev": 0.08587293804779245, "run_count": 10, "total_questions": 498, "avg_cost_per_task_usd": 0.040060716867469884, "avg_duration_per_task_ms": 271599.5261044177, "avg_output_tokens_per_task": 22537.02610441767, "last_run_timestamp": "2026-10-31T19:30:45.204Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "model_slug": "meta/muse-spark-open", "model_name": "Meta: Muse Spark Open", "provider_name": "BaseTen", "accuracy": 0.7228067157894738, "accuracy_std_dev": 0.002917999999999976, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.06760965263157895, "avg_duration_per_task_ms": 49831.26315789474, "avg_output_tokens_per_task": 5886.4315789473685, "last_run_timestamp": "2026-10-31T12:12:21.006Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "OpenInference", "accuracy": 0.722650186440678, "accuracy_std_dev": 0.03806732610044826, "run_count": 13, "total_questions": 649, "avg_cost_per_task_usd": 0.024282243451463792, "avg_duration_per_task_ms": 277851.95069337444, "avg_output_tokens_per_task": 9180.11093990755, "last_run_timestamp": "2026-10-31T13:24:52.766Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Venice", "accuracy": 0.7225, "accuracy_std_dev": 0.029615564602924586, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.05079202000000001, "avg_duration_per_task_ms": 151914.885, "avg_output_tokens_per_task": 28118.625, "last_run_timestamp": "2026-10-31T12:23:57.101Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Io Net", "accuracy": 0.7222223333333333, "accuracy_std_dev": 0.011330936570096713, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.018792733333333332, "avg_duration_per_task_ms": 301192.17333333334, "avg_output_tokens_per_task": 19692.3, "last_run_timestamp": "2026-10-01T16:00:06.404Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "auto-routing", "accuracy": 0.722, "accuracy_std_dev": 0.030265381765310677, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.332162946, "avg_duration_per_task_ms": 207983.252, "avg_output_tokens_per_task": 10350.054, "last_run_timestamp": "2026-10-31T12:49:56.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "model_slug": "anthropic/claude-sonnet-4.5", "model_name": "Anthropic: Claude Sonnet 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.722, "accuracy_std_dev": 0.03148898925815181, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.350364916, "avg_duration_per_task_ms": 202841.697, "avg_output_tokens_per_task": 10188.288, "last_run_timestamp": "2026-10-31T12:42:19.688Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DeepInfra", "accuracy": 0.722, "accuracy_std_dev": 0.040309938119525816, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.054631136, "avg_duration_per_task_ms": 317758.82, "avg_output_tokens_per_task": 11839.54, "last_run_timestamp": "2026-09-26T08:12:00.292Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Nebius", "accuracy": 0.72166675, "accuracy_std_dev": 0.012801898733683979, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.024317345, "avg_duration_per_task_ms": 116922.04, "avg_output_tokens_per_task": 11395.83, "last_run_timestamp": "2026-10-31T12:46:14.030Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Particle AI", "accuracy": 0.7216665, "accuracy_std_dev": 0.031754396022440755, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.018527804999999998, "avg_duration_per_task_ms": 90190.985, "avg_output_tokens_per_task": 8919.29, "last_run_timestamp": "2026-11-03T01:55:17.248Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Ionstream", "accuracy": 0.7213335999999999, "accuracy_std_dev": 0.027455065511486212, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.060558396, "avg_duration_per_task_ms": 335487.684, "avg_output_tokens_per_task": 77986.164, "last_run_timestamp": "2026-11-10T17:50:20.037Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Reka", "accuracy": 0.7213334, "accuracy_std_dev": 0.01146975927559073, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.012899843999999999, "avg_duration_per_task_ms": 152142.784, "avg_output_tokens_per_task": 13063.108, "last_run_timestamp": "2026-10-31T13:10:04.012Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "OpenInference", "accuracy": 0.7211110000000001, "accuracy_std_dev": 0.04794518990124728, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.011416513333333333, "avg_duration_per_task_ms": 885570.78, "avg_output_tokens_per_task": 25900.693333333333, "last_run_timestamp": "2026-10-17T10:32:07.729Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Makora", "accuracy": 0.7210882448979592, "accuracy_std_dev": 0.0076780598851532815, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.009489710204081632, "avg_duration_per_task_ms": 145499.9755102041, "avg_output_tokens_per_task": 16260.81224489796, "last_run_timestamp": "2026-09-26T08:06:43.933Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "auto-routing", "accuracy": 0.7209522857142857, "accuracy_std_dev": 0.02348344522493778, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06015369714285714, "avg_duration_per_task_ms": 338155.90571428573, "avg_output_tokens_per_task": 10030, "last_run_timestamp": "2026-10-31T20:35:24.659Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "DeepInfra", "accuracy": 0.7206894068965518, "accuracy_std_dev": 0.04434366958954421, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.015247455172413793, "avg_duration_per_task_ms": 270487.4206896552, "avg_output_tokens_per_task": 5712.675862068965, "last_run_timestamp": "2026-10-09T02:01:31.527Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "AtlasCloud", "accuracy": 0.7206666, "accuracy_std_dev": 0.037172284778313, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.07504917600000001, "avg_duration_per_task_ms": 152339.734, "avg_output_tokens_per_task": 8284.966, "last_run_timestamp": "2026-10-31T19:53:09.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "model_slug": "moonshotai/kimi-k3", "model_name": "MoonshotAI: Kimi K3", "provider_name": "DigitalOcean", "accuracy": 0.7206058181818182, "accuracy_std_dev": 0.06497575192305928, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.2521624945454546, "avg_duration_per_task_ms": 431862.3145454546, "avg_output_tokens_per_task": 10421.192727272728, "last_run_timestamp": "2026-10-31T12:33:51.242Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Fireworks", "accuracy": 0.72, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.026063740000000002, "avg_duration_per_task_ms": 127597.92, "avg_output_tokens_per_task": 12968.38, "last_run_timestamp": "2026-09-19T08:13:24.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "Io Net", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12671924, "avg_duration_per_task_ms": 566714.78, "avg_output_tokens_per_task": 16714.02, "last_run_timestamp": "2026-09-28T18:00:05.892Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "Novita", "accuracy": 0.72, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00317077, "avg_duration_per_task_ms": 128719.23, "avg_output_tokens_per_task": 15118.03, "last_run_timestamp": "2026-10-01T08:00:27.045Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Venice", "accuracy": 0.72, "accuracy_std_dev": 0.01247214674785381, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08792842000000001, "avg_duration_per_task_ms": 120135.21, "avg_output_tokens_per_task": 16462.21, "last_run_timestamp": "2026-10-31T11:24:22.174Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "PrimeIntellect", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00807376, "avg_duration_per_task_ms": 275288.98, "avg_output_tokens_per_task": 19939.9, "last_run_timestamp": "2026-11-01T18:47:31.518Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "model_slug": "qwen/qwen3.7-max", "model_name": "Qwen: Qwen3.7 Max", "provider_name": "AtlasCloud", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.402901, "avg_duration_per_task_ms": 196016.4, "avg_output_tokens_per_task": 16256.52, "last_run_timestamp": "2026-09-27T00:00:03.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "model_slug": "xiaomi/mimo-v2.6-flash", "model_name": "Xiaomi: MiMo-V2.6-Flash", "provider_name": "Tenstorrent", "accuracy": 0.72, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01515436, "avg_duration_per_task_ms": 412586.86, "avg_output_tokens_per_task": 21062.22, "last_run_timestamp": "2026-11-08T06:16:39.274Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Parasail", "accuracy": 0.7196295555555555, "accuracy_std_dev": 0.023540924295603994, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.04364436, "avg_duration_per_task_ms": 154693.13555555555, "avg_output_tokens_per_task": 29598.053333333333, "last_run_timestamp": "2026-11-05T00:27:56.809Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Phala", "accuracy": 0.7190477142857142, "accuracy_std_dev": 0.026047291719454555, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.21749334, "avg_duration_per_task_ms": 1099812.5457142857, "avg_output_tokens_per_task": 14108.717142857144, "last_run_timestamp": "2026-10-31T19:47:23.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "auto-routing", "accuracy": 0.7190477142857142, "accuracy_std_dev": 0.014443512619010297, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.052984574285714285, "avg_duration_per_task_ms": 384227.2342857143, "avg_output_tokens_per_task": 9361.582857142857, "last_run_timestamp": "2026-10-31T20:46:45.843Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Novita", "accuracy": 0.7190475714285715, "accuracy_std_dev": 0.021507714169951352, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09843814571428573, "avg_duration_per_task_ms": 197643.15714285715, "avg_output_tokens_per_task": 13285.197142857143, "last_run_timestamp": "2026-10-31T20:36:53.013Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "DeepInfra", "accuracy": 0.71833325, "accuracy_std_dev": 0.03378866920563018, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.03334888, "avg_duration_per_task_ms": 414661.68, "avg_output_tokens_per_task": 8985.465, "last_run_timestamp": "2026-10-31T21:01:10.349Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Novita", "accuracy": 0.718, "accuracy_std_dev": 0.022120970444354387, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.081670762, "avg_duration_per_task_ms": 224518.662, "avg_output_tokens_per_task": 9223.09, "last_run_timestamp": "2026-10-31T20:11:34.826Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Novita", "accuracy": 0.718, "accuracy_std_dev": 0.027170482351257574, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.049269784, "avg_duration_per_task_ms": 173368.438, "avg_output_tokens_per_task": 9812.098, "last_run_timestamp": "2026-10-31T20:01:58.593Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "StreamLake", "accuracy": 0.717778, "accuracy_std_dev": 0.046930323977857506, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.045013793333333336, "avg_duration_per_task_ms": 196458.52, "avg_output_tokens_per_task": 18662.633333333335, "last_run_timestamp": "2026-10-31T19:03:35.054Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "SiliconFlow", "accuracy": 0.7173333000000001, "accuracy_std_dev": 0.023702007417305403, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.08080122399999999, "avg_duration_per_task_ms": 234769.354, "avg_output_tokens_per_task": 8765.008, "last_run_timestamp": "2026-10-31T20:37:11.918Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Novita", "accuracy": 0.717143, "accuracy_std_dev": 0.02802638374817556, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.026408662857142856, "avg_duration_per_task_ms": 224207.81142857144, "avg_output_tokens_per_task": 10096.908571428572, "last_run_timestamp": "2026-10-31T20:20:16.475Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "auto-routing", "accuracy": 0.7171427142857143, "accuracy_std_dev": 0.03802358976708421, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10077471142857143, "avg_duration_per_task_ms": 217139.6, "avg_output_tokens_per_task": 10145.26857142857, "last_run_timestamp": "2026-10-31T20:09:58.111Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "DekaLLM", "accuracy": 0.716667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01519146, "avg_duration_per_task_ms": 819823.24, "avg_output_tokens_per_task": 11470.58, "last_run_timestamp": "2026-10-31T20:19:18.995Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "auto-routing", "accuracy": 0.7166669999999999, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1280142, "avg_duration_per_task_ms": 531647.04, "avg_output_tokens_per_task": 56970.52, "last_run_timestamp": "2026-10-01T19:04:42.411Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "StreamLake", "accuracy": 0.716666625, "accuracy_std_dev": 0.029627375668195347, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0693026925, "avg_duration_per_task_ms": 283497.055, "avg_output_tokens_per_task": 12604.555, "last_run_timestamp": "2026-10-31T19:48:30.443Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Ambient", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00699117, "avg_duration_per_task_ms": 151328.11, "avg_output_tokens_per_task": 13794.25, "last_run_timestamp": "2026-08-29T09:31:13.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Venice", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.022047789999999998, "avg_duration_per_task_ms": 63248.07, "avg_output_tokens_per_task": 8992.32, "last_run_timestamp": "2026-10-31T12:43:48.267Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "StreamLake", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.015074639999999999, "avg_duration_per_task_ms": 124785.23, "avg_output_tokens_per_task": 11839.88, "last_run_timestamp": "2026-10-31T12:28:39.529Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Novita", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.043333500000000025, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.048402060000000004, "avg_duration_per_task_ms": 204313.4, "avg_output_tokens_per_task": 8310.75, "last_run_timestamp": "2026-08-29T17:21:26.058Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "OpenAI", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.71669186, "avg_duration_per_task_ms": 277432.68, "avg_output_tokens_per_task": 22919.22, "last_run_timestamp": "2026-10-02T23:08:29.026Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Relace", "accuracy": 0.7166665000000001, "accuracy_std_dev": 0.02603405888927041, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.13610847, "avg_duration_per_task_ms": 230555.42, "avg_output_tokens_per_task": 11694.86, "last_run_timestamp": "2026-10-31T19:46:31.005Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Novita", "accuracy": 0.7161904285714286, "accuracy_std_dev": 0.02133854528350812, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.053568791428571426, "avg_duration_per_task_ms": 335614.3914285714, "avg_output_tokens_per_task": 9870.974285714286, "last_run_timestamp": "2026-10-31T20:51:04.120Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Venice", "accuracy": 0.7159998000000001, "accuracy_std_dev": 0.03295793548085197, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.24450749400000002, "avg_duration_per_task_ms": 314057.314, "avg_output_tokens_per_task": 21575.564, "last_run_timestamp": "2026-10-31T12:46:35.433Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Together", "accuracy": 0.7155556666666666, "accuracy_std_dev": 0.03189478845969807, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01325666, "avg_duration_per_task_ms": 86099.67333333334, "avg_output_tokens_per_task": 8966.793333333333, "last_run_timestamp": "2026-10-31T13:08:15.957Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Venice", "accuracy": 0.7153333000000001, "accuracy_std_dev": 0.03183982342303425, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.09070920600000001, "avg_duration_per_task_ms": 284476.36, "avg_output_tokens_per_task": 9245.778, "last_run_timestamp": "2026-10-31T19:57:59.098Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Wafer", "accuracy": 0.715278, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.009582875, "avg_duration_per_task_ms": 164124.5, "avg_output_tokens_per_task": 18445.333333333332, "last_run_timestamp": "2026-10-10T11:44:02.317Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Venice", "accuracy": 0.7152381428571428, "accuracy_std_dev": 0.022172375614898876, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.04021302285714286, "avg_duration_per_task_ms": 623020.8714285714, "avg_output_tokens_per_task": 9178.234285714285, "last_run_timestamp": "2026-10-31T19:41:08.087Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "model_slug": "z-ai/glm-5", "model_name": "Z.ai: GLM 5", "provider_name": "Z.AI", "accuracy": 0.7152381428571428, "accuracy_std_dev": 0.03866263877783441, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.10695628285714286, "avg_duration_per_task_ms": 188318.44857142857, "avg_output_tokens_per_task": 12311.014285714286, "last_run_timestamp": "2026-10-31T20:14:27.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Chutes", "accuracy": 0.715238, "accuracy_std_dev": 0.0354049972784311, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.05114133714285715, "avg_duration_per_task_ms": 426873.76857142855, "avg_output_tokens_per_task": 11755.502857142857, "last_run_timestamp": "2026-10-31T20:58:51.733Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Baidu", "accuracy": 0.7147368947368421, "accuracy_std_dev": 0.0235532410014658, "run_count": 19, "total_questions": 950, "avg_cost_per_task_usd": 0.07154260947368421, "avg_duration_per_task_ms": 115435.01157894736, "avg_output_tokens_per_task": 8767.831578947369, "last_run_timestamp": "2026-11-10T19:27:07.187Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Fireworks", "accuracy": 0.7143545606936417, "accuracy_std_dev": 0.030307639134686294, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.08907920231213873, "avg_duration_per_task_ms": 87722.67919075144, "avg_output_tokens_per_task": 7278.56647398844, "last_run_timestamp": "2026-09-26T08:26:47.303Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "model_slug": "z-ai/glm-5.3-flash", "model_name": "Z.ai: GLM 5.3 Flash", "provider_name": "GMICloud", "accuracy": 0.7139393636363637, "accuracy_std_dev": 0.07661811901688757, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.010148356363636365, "avg_duration_per_task_ms": 242007.0509090909, "avg_output_tokens_per_task": 8765.507272727273, "last_run_timestamp": "2026-11-10T08:18:49.964Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Baidu", "accuracy": 0.71375, "accuracy_std_dev": 0.03794860559691226, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.09171088000000001, "avg_duration_per_task_ms": 170840.095, "avg_output_tokens_per_task": 11418.88, "last_run_timestamp": "2026-11-10T20:09:17.079Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Alibaba", "accuracy": 0.7135715, "accuracy_std_dev": 0.03209528384302411, "run_count": 28, "total_questions": 1400, "avg_cost_per_task_usd": 0.1159875742857143, "avg_duration_per_task_ms": 125459.01928571428, "avg_output_tokens_per_task": 8558.635714285714, "last_run_timestamp": "2026-10-31T19:56:11.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "Minimax", "accuracy": 0.7133333571428572, "accuracy_std_dev": 0.024168666585380864, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.038873661428571425, "avg_duration_per_task_ms": 230582.26428571428, "avg_output_tokens_per_task": 9732.062857142857, "last_run_timestamp": "2026-10-31T20:38:36.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Novita", "accuracy": 0.7133332857142858, "accuracy_std_dev": 0.026427450856552726, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.13545089142857142, "avg_duration_per_task_ms": 267525.3171428571, "avg_output_tokens_per_task": 10590.037142857143, "last_run_timestamp": "2026-10-31T20:31:35.900Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.2-20261109", "model_slug": "anthropic/claude-fable-5.2", "model_name": "Anthropic: Claude Fable 5.2", "provider_name": "Anthropic", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.78360724, "avg_duration_per_task_ms": 157424.24, "avg_output_tokens_per_task": 9303.06, "last_run_timestamp": "2026-11-10T18:06:53.999Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "River", "accuracy": 0.713333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.011907570000000001, "avg_duration_per_task_ms": 93169.09, "avg_output_tokens_per_task": 17553.86, "last_run_timestamp": "2026-11-12T21:15:48.475Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cloudflare", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05452906, "avg_duration_per_task_ms": 272872.18, "avg_output_tokens_per_task": 39719.44, "last_run_timestamp": "2026-10-13T20:01:29.821Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08007529, "avg_duration_per_task_ms": 140906.22, "avg_output_tokens_per_task": 42659.23, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "DeepInfra", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.006318460000000001, "avg_duration_per_task_ms": 234030.92, "avg_output_tokens_per_task": 14773.36, "last_run_timestamp": "2026-10-01T08:05:57.971Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Fireworks", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01617958, "avg_duration_per_task_ms": 141600.68, "avg_output_tokens_per_task": 14742.64, "last_run_timestamp": "2026-10-31T12:42:39.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "GMICloud", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.09868048, "avg_duration_per_task_ms": 318461, "avg_output_tokens_per_task": 11794.24, "last_run_timestamp": "2026-08-29T17:18:09.782Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "model_slug": "qwen/qwen3.8-flash", "model_name": "Qwen: Qwen3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.713333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01665106, "avg_duration_per_task_ms": 327560.6, "avg_output_tokens_per_task": 17227.24, "last_run_timestamp": "2026-10-01T08:00:41.459Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "model_slug": "google/gemini-3.8-flash", "model_name": "Google: Gemini 3.8 Flash", "provider_name": "auto-routing", "accuracy": 0.712766, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.21022357446808512, "avg_duration_per_task_ms": 261220.25531914894, "avg_output_tokens_per_task": 27609.17021276596, "last_run_timestamp": "2026-10-01T08:02:55.088Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "SiliconFlow", "accuracy": 0.7126666, "accuracy_std_dev": 0.028975068211136257, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.07464725, "avg_duration_per_task_ms": 125244.052, "avg_output_tokens_per_task": 9446.146, "last_run_timestamp": "2026-10-31T19:18:12.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "ModelRun", "accuracy": 0.7125, "accuracy_std_dev": 0.006401128377091036, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.066283255, "avg_duration_per_task_ms": 74239.48, "avg_output_tokens_per_task": 8403.98, "last_run_timestamp": "2026-09-26T08:08:19.642Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Alibaba", "accuracy": 0.712222, "accuracy_std_dev": 0.01640534390577251, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04505285333333333, "avg_duration_per_task_ms": 337842.44, "avg_output_tokens_per_task": 13141.253333333334, "last_run_timestamp": "2026-10-04T00:00:34.898Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "OpenInference", "accuracy": 0.7117822371364653, "accuracy_std_dev": 0.056118924341169225, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.0462677941834452, "avg_duration_per_task_ms": 1033622.2237136465, "avg_output_tokens_per_task": 60076.49888143177, "last_run_timestamp": "2026-10-31T13:23:14.602Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Cohere", "accuracy": 0.7111113333333333, "accuracy_std_dev": 0.006285236475706792, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06754876666666666, "avg_duration_per_task_ms": 139695.35333333333, "avg_output_tokens_per_task": 33581.94, "last_run_timestamp": "2026-10-31T12:01:01.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Crusoe", "accuracy": 0.7111113333333333, "accuracy_std_dev": 0.0332591470759883, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10563259333333332, "avg_duration_per_task_ms": 87886.20666666667, "avg_output_tokens_per_task": 8897.613333333333, "last_run_timestamp": "2026-09-12T08:12:28.966Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "GMICloud", "accuracy": 0.7103703333333333, "accuracy_std_dev": 0.018354812708266867, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.022485688888888886, "avg_duration_per_task_ms": 293573.69333333336, "avg_output_tokens_per_task": 10080.99111111111, "last_run_timestamp": "2026-11-10T07:15:16.990Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "AkashML", "accuracy": 0.71, "accuracy_std_dev": 0.036667000000000005, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.011451959999999999, "avg_duration_per_task_ms": 477432.72, "avg_output_tokens_per_task": 17288.47, "last_run_timestamp": "2026-08-29T08:43:07.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Cohere", "accuracy": 0.71, "accuracy_std_dev": 0.016666999999999987, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.015407820000000001, "avg_duration_per_task_ms": 62181.78, "avg_output_tokens_per_task": 11850.44, "last_run_timestamp": "2026-10-31T11:31:06.991Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "ModelRun", "accuracy": 0.71, "accuracy_std_dev": 0.01795033271279393, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0134254, "avg_duration_per_task_ms": 120168.52, "avg_output_tokens_per_task": 11670.58, "last_run_timestamp": "2026-10-31T12:25:39.138Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.71, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12724396999999998, "avg_duration_per_task_ms": 73218.95, "avg_output_tokens_per_task": 3826.45, "last_run_timestamp": "2026-10-01T19:04:38.304Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "AtlasCloud", "accuracy": 0.7095238571428572, "accuracy_std_dev": 0.057993713007602476, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.030501814285714284, "avg_duration_per_task_ms": 305801.77142857143, "avg_output_tokens_per_task": 10004.028571428571, "last_run_timestamp": "2026-10-31T20:10:49.937Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "Venice", "accuracy": 0.7095237142857143, "accuracy_std_dev": 0.03614029729387676, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06154300571428571, "avg_duration_per_task_ms": 247462.57428571428, "avg_output_tokens_per_task": 25975.411428571428, "last_run_timestamp": "2026-10-31T19:22:37.818Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "AtlasCloud", "accuracy": 0.7093334, "accuracy_std_dev": 0.07236922366199598, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.10543298799999999, "avg_duration_per_task_ms": 199005.244, "avg_output_tokens_per_task": 11270.708, "last_run_timestamp": "2026-10-17T19:18:15.219Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "auto-routing", "accuracy": 0.7093333, "accuracy_std_dev": 0.02924233156931914, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.047131844, "avg_duration_per_task_ms": 100777.312, "avg_output_tokens_per_task": 8394.926, "last_run_timestamp": "2026-10-31T19:54:42.760Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "CoreWeave", "accuracy": 0.709091, "accuracy_std_dev": 0.0405291288424421, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07860464545454546, "avg_duration_per_task_ms": 111991.2290909091, "avg_output_tokens_per_task": 8595.541818181819, "last_run_timestamp": "2026-10-31T19:45:55.386Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Novita", "accuracy": 0.708889, "accuracy_std_dev": 0.0031423825355929936, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.017471633333333333, "avg_duration_per_task_ms": 219758.70666666667, "avg_output_tokens_per_task": 16040.473333333333, "last_run_timestamp": "2026-10-31T12:30:16.930Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "auto-routing", "accuracy": 0.7085714285714285, "accuracy_std_dev": 0.012706521619761615, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.027396339999999998, "avg_duration_per_task_ms": 231738.34285714285, "avg_output_tokens_per_task": 10497.88, "last_run_timestamp": "2026-10-31T19:47:12.389Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Wafer", "accuracy": 0.7083333, "accuracy_std_dev": 0.024843901555713817, "run_count": 20, "total_questions": 1000, "avg_cost_per_task_usd": 0.09730978499999998, "avg_duration_per_task_ms": 109439.124, "avg_output_tokens_per_task": 9725.372, "last_run_timestamp": "2026-10-31T19:37:51.988Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Nebius", "accuracy": 0.7083331666666667, "accuracy_std_dev": 0.017400462957602263, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.25373078000000004, "avg_duration_per_task_ms": 176420.9, "avg_output_tokens_per_task": 10605.743333333334, "last_run_timestamp": "2026-10-31T19:48:27.256Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "model_slug": "qwen/qwen3.6-27b", "model_name": "Qwen: Qwen3.6 27B", "provider_name": "Phala", "accuracy": 0.7080952857142857, "accuracy_std_dev": 0.122226283771085, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.23844771999999997, "avg_duration_per_task_ms": 1324378.657142857, "avg_output_tokens_per_task": 65425.98, "last_run_timestamp": "2026-10-10T19:16:51.412Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "Azure", "accuracy": 0.7077778333333333, "accuracy_std_dev": 0.012422535280914084, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.11194547666666667, "avg_duration_per_task_ms": 92493.15666666666, "avg_output_tokens_per_task": 3647.7966666666666, "last_run_timestamp": "2026-10-01T19:04:52.651Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "auto-routing", "accuracy": 0.707483, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.004842469387755102, "avg_duration_per_task_ms": 161558.55102040817, "avg_output_tokens_per_task": 17024.020408163266, "last_run_timestamp": "2026-10-01T08:01:49.153Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "model_slug": "google/gemini-3.5-flash", "model_name": "Google: Gemini 3.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.706667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.44632813, "avg_duration_per_task_ms": 132811.5, "avg_output_tokens_per_task": 20668.24, "last_run_timestamp": "2026-10-01T19:04:52.658Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "auto-routing", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.71176192, "avg_duration_per_task_ms": 285827.28, "avg_output_tokens_per_task": 22895.76, "last_run_timestamp": "2026-10-01T16:00:34.660Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "DeepInfra", "accuracy": 0.706667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.023244639999999997, "avg_duration_per_task_ms": 174558.52, "avg_output_tokens_per_task": 8529.56, "last_run_timestamp": "2026-11-13T12:51:19.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "OpenAI", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 3.04321209, "avg_duration_per_task_ms": 317897.29, "avg_output_tokens_per_task": 13272.53, "last_run_timestamp": "2026-10-02T23:13:16.494Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.7066669999999999, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1042244, "avg_duration_per_task_ms": 226024.67, "avg_output_tokens_per_task": 3543.23, "last_run_timestamp": "2026-10-30T00:17:12.390Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "DigitalOcean", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.015417799999999999, "avg_duration_per_task_ms": 75266.33333333333, "avg_output_tokens_per_task": 12018.02, "last_run_timestamp": "2026-10-31T11:20:39.476Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.005443174461857933, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.5994524266666668, "avg_duration_per_task_ms": 636541.5466666666, "avg_output_tokens_per_task": 54354.86, "last_run_timestamp": "2026-10-02T23:22:41.519Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.7066666666666667, "accuracy_std_dev": 0.01962587106064057, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08575882, "avg_duration_per_task_ms": 316120.8933333333, "avg_output_tokens_per_task": 32977.22666666667, "last_run_timestamp": "2026-10-02T23:12:53.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Phala", "accuracy": 0.706666625, "accuracy_std_dev": 0.020000041668315968, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.10048072499999999, "avg_duration_per_task_ms": 168728.9075, "avg_output_tokens_per_task": 11337.64, "last_run_timestamp": "2026-10-17T19:53:39.136Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Novita", "accuracy": 0.7063038567335244, "accuracy_std_dev": 0.03586910330405172, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.02153851862464183, "avg_duration_per_task_ms": 628235.6618911175, "avg_output_tokens_per_task": 9341.048710601719, "last_run_timestamp": "2026-10-31T19:57:15.103Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Makora", "accuracy": 0.7061111666666666, "accuracy_std_dev": 0.023602995992152234, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.023716863333333334, "avg_duration_per_task_ms": 111467.81333333334, "avg_output_tokens_per_task": 29595.083333333332, "last_run_timestamp": "2026-10-31T19:52:48.234Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "SambaNova", "accuracy": 0.7060303366834171, "accuracy_std_dev": 0.0301180502813512, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.030429407035175878, "avg_duration_per_task_ms": 140666.3216080402, "avg_output_tokens_per_task": 10689.924623115578, "last_run_timestamp": "2026-11-11T19:39:53.430Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Azure", "accuracy": 0.7055558333333333, "accuracy_std_dev": 0.032754340019894884, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.012937629999999999, "avg_duration_per_task_ms": 95536.39333333333, "avg_output_tokens_per_task": 4842.643333333333, "last_run_timestamp": "2026-10-01T19:05:16.550Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Chutes", "accuracy": 0.7055555, "accuracy_std_dev": 0.03408387439709088, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.09377007666666666, "avg_duration_per_task_ms": 923124.9966666667, "avg_output_tokens_per_task": 9445.45, "last_run_timestamp": "2026-10-31T21:09:07.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "model_slug": "openai/gpt-5-mini", "model_name": "OpenAI: GPT-5 Mini", "provider_name": "Azure", "accuracy": 0.70499975, "accuracy_std_dev": 0.011902485735656231, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.13260128, "avg_duration_per_task_ms": 477326.39, "avg_output_tokens_per_task": 55745.89, "last_run_timestamp": "2026-10-01T19:05:16.547Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "model_slug": "qwen/qwen3.8-27b", "model_name": "Qwen: Qwen3.8 27B", "provider_name": "Tenstorrent", "accuracy": 0.7048148888888889, "accuracy_std_dev": 0.14567945297081722, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.0241112, "avg_duration_per_task_ms": 290526.02444444445, "avg_output_tokens_per_task": 11444.402222222223, "last_run_timestamp": "2026-11-08T03:52:11.162Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Venice", "accuracy": 0.704762, "accuracy_std_dev": 0.03486274480219332, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.07204742, "avg_duration_per_task_ms": 281672.5857142857, "avg_output_tokens_per_task": 11399.114285714286, "last_run_timestamp": "2026-10-31T21:34:09.712Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Together", "accuracy": 0.7046667, "accuracy_std_dev": 0.036642272492436935, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.09915010799999999, "avg_duration_per_task_ms": 70205.334, "avg_output_tokens_per_task": 8678.166, "last_run_timestamp": "2026-10-31T19:33:25.981Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20261017", "model_slug": "deepseek/deepseek-v4.1-flash-1017", "model_name": "DeepSeek: DeepSeek V4.1 Flash 1017", "provider_name": "Makora", "accuracy": 0.7039998000000001, "accuracy_std_dev": 0.025508095447524113, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.021704752, "avg_duration_per_task_ms": 119683.016, "avg_output_tokens_per_task": 11904.78, "last_run_timestamp": "2026-10-31T12:25:42.323Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "DigitalOcean", "accuracy": 0.7033335, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.053050639999999996, "avg_duration_per_task_ms": 750494.53, "avg_output_tokens_per_task": 9139.19, "last_run_timestamp": "2026-08-29T16:47:05.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.7033335, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08534018, "avg_duration_per_task_ms": 347658.09, "avg_output_tokens_per_task": 32908.85, "last_run_timestamp": "2026-10-01T16:00:47.532Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Moonshot AI", "accuracy": 0.703333, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.045981190000000005, "avg_duration_per_task_ms": 151977.07, "avg_output_tokens_per_task": 8039.38, "last_run_timestamp": "2026-08-29T17:36:31.073Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "model_slug": "qwen/qwen3.7-plus", "model_name": "Qwen: Qwen3.7 Plus", "provider_name": "auto-routing", "accuracy": 0.703333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05425128, "avg_duration_per_task_ms": 396980.46, "avg_output_tokens_per_task": 20673.9, "last_run_timestamp": "2026-10-01T08:05:20.277Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Nebius", "accuracy": 0.7025, "accuracy_std_dev": 0.030173028796426787, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.2793706775, "avg_duration_per_task_ms": 496473.2875, "avg_output_tokens_per_task": 9455.475, "last_run_timestamp": "2026-10-31T22:14:51.015Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Io Net", "accuracy": 0.7023811326530611, "accuracy_std_dev": 0.047340802625087566, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.014528989795918367, "avg_duration_per_task_ms": 320290.7346938776, "avg_output_tokens_per_task": 12552.510204081633, "last_run_timestamp": "2026-11-11T06:41:01.794Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "model_slug": "openai/gpt-5.6-terra", "model_name": "OpenAI: GPT-5.6 Terra", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.021999169459070246, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.10818771333333332, "avg_duration_per_task_ms": 79804.25333333333, "avg_output_tokens_per_task": 3542.24, "last_run_timestamp": "2026-10-02T23:10:36.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "OpenAI", "accuracy": 0.7022223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.13357544, "avg_duration_per_task_ms": 105586.74666666667, "avg_output_tokens_per_task": 5260.826666666667, "last_run_timestamp": "2026-10-02T23:07:17.250Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "Nebius", "accuracy": 0.7018973089430894, "accuracy_std_dev": 0.04349030049378826, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.029020682926829265, "avg_duration_per_task_ms": 391991.35365853657, "avg_output_tokens_per_task": 21482.62601626016, "last_run_timestamp": "2026-11-12T15:21:11.557Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "DeepInfra", "accuracy": 0.7018181818181818, "accuracy_std_dev": 0.033525796984135756, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.04067215272727272, "avg_duration_per_task_ms": 142417.50727272726, "avg_output_tokens_per_task": 8505.803636363637, "last_run_timestamp": "2026-10-31T20:18:13.909Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "model_slug": "z-ai/glm-4.7", "model_name": "Z.ai: GLM 4.7", "provider_name": "Z.AI", "accuracy": 0.7009522857142857, "accuracy_std_dev": 0.04471111302801668, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06413979428571429, "avg_duration_per_task_ms": 345333.78285714285, "avg_output_tokens_per_task": 10011.842857142858, "last_run_timestamp": "2026-10-31T20:44:35.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "AtlasCloud", "accuracy": 0.7006667, "accuracy_std_dev": 0.16001239243699222, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.061942158000000004, "avg_duration_per_task_ms": 147418.05, "avg_output_tokens_per_task": 16768.652, "last_run_timestamp": "2026-10-31T19:40:48.790Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "StreamLake", "accuracy": 0.7006060000000001, "accuracy_std_dev": 0.025968777609765438, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.03481650181818182, "avg_duration_per_task_ms": 169300.62181818183, "avg_output_tokens_per_task": 8963.998181818182, "last_run_timestamp": "2026-10-31T20:27:27.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Z.AI", "accuracy": 0.7000001, "accuracy_std_dev": 0.02170498150863068, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.124576664, "avg_duration_per_task_ms": 163368.62, "avg_output_tokens_per_task": 11905.278, "last_run_timestamp": "2026-10-31T19:55:04.254Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.29769207999999997, "avg_duration_per_task_ms": 128375.6, "avg_output_tokens_per_task": 8500.86, "last_run_timestamp": "2026-09-01T12:41:24.616Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Phala", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02395168, "avg_duration_per_task_ms": 160119.2, "avg_output_tokens_per_task": 14308.04, "last_run_timestamp": "2026-08-29T16:33:36.025Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Google", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0355865, "avg_duration_per_task_ms": 154491.8, "avg_output_tokens_per_task": 12574.4, "last_run_timestamp": "2026-10-31T11:15:31.400Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash", "model_slug": "z-ai/glm-5.3-flash", "model_name": "GLM 5.3 Flash", "provider_name": "digitalocean", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00357748, "avg_duration_per_task_ms": 48793.08, "avg_output_tokens_per_task": 1567.76, "last_run_timestamp": "2026-10-01T23:50:45.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-vl-20260910", "model_slug": "inclusionai/ling-3.0-flash-vl", "model_name": "inclusionAI: Ling 3.0 Flash VL", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0030046400000000003, "avg_duration_per_task_ms": 98848.12, "avg_output_tokens_per_task": 17113.74, "last_run_timestamp": "2026-10-01T08:00:19.901Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "PrimeIntellect", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12726106, "avg_duration_per_task_ms": 181226.44, "avg_output_tokens_per_task": 9838.56, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "StreamLake", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0161093, "avg_duration_per_task_ms": 215051.72, "avg_output_tokens_per_task": 15584.08, "last_run_timestamp": "2026-10-31T12:24:55.227Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": 0.03266000904120248, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.055787651428571436, "avg_duration_per_task_ms": 343671.10285714286, "avg_output_tokens_per_task": 9284.377142857144, "last_run_timestamp": "2026-10-31T22:10:13.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Phala", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07376152, "avg_duration_per_task_ms": 750952.84, "avg_output_tokens_per_task": 11282.28, "last_run_timestamp": "2026-08-29T16:44:16.336Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "Azure", "accuracy": 0.7, "accuracy_std_dev": 0.004714280910170736, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.634261665, "avg_duration_per_task_ms": 673734.145, "avg_output_tokens_per_task": 52923.67, "last_run_timestamp": "2026-10-01T16:01:02.669Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "auto-routing", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.1012818, "avg_duration_per_task_ms": 144103.72, "avg_output_tokens_per_task": 3638.14, "last_run_timestamp": "2026-10-01T08:03:00.822Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "OpenAI", "accuracy": 0.7, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.05778106, "avg_duration_per_task_ms": 103456.94, "avg_output_tokens_per_task": 4693.4, "last_run_timestamp": "2026-10-28T00:06:02.481Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Parasail", "accuracy": 0.6977776666666667, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013071946666666667, "avg_duration_per_task_ms": 88036.78, "avg_output_tokens_per_task": 11978.486666666666, "last_run_timestamp": "2026-10-31T11:55:17.583Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Amazon Bedrock", "accuracy": 0.6977776666666667, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.14221055333333335, "avg_duration_per_task_ms": 130884.03333333334, "avg_output_tokens_per_task": 4996.426666666666, "last_run_timestamp": "2026-10-30T00:14:00.742Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Parasail", "accuracy": 0.6976666, "accuracy_std_dev": 0.024722754252712212, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.11522000200000002, "avg_duration_per_task_ms": 90418.644, "avg_output_tokens_per_task": 9963.432, "last_run_timestamp": "2026-10-31T19:57:32.812Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "BaseTen", "accuracy": 0.6972222117283948, "accuracy_std_dev": 0.0265134534342492, "run_count": 33, "total_questions": 1620, "avg_cost_per_task_usd": 0.10315858827160493, "avg_duration_per_task_ms": 72358.56851851851, "avg_output_tokens_per_task": 8108.612962962963, "last_run_timestamp": "2026-10-31T20:07:10.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "auto-routing", "accuracy": 0.696667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01321477, "avg_duration_per_task_ms": 116452.62, "avg_output_tokens_per_task": 5139.02, "last_run_timestamp": "2026-10-01T19:04:36.191Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Decart", "accuracy": 0.6966665384615385, "accuracy_std_dev": 0.021162820164747506, "run_count": 13, "total_questions": 650, "avg_cost_per_task_usd": 0.1543893523076923, "avg_duration_per_task_ms": 76695.08, "avg_output_tokens_per_task": 10682.713846153847, "last_run_timestamp": "2026-10-24T19:06:47.526Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Azure", "accuracy": 0.6966665000000001, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.62488186, "avg_duration_per_task_ms": 152081.62, "avg_output_tokens_per_task": 3927.45, "last_run_timestamp": "2026-10-01T08:05:41.499Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "Venice", "accuracy": 0.6961905714285714, "accuracy_std_dev": 0.027338489642977408, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.045278854285714286, "avg_duration_per_task_ms": 549308.6714285715, "avg_output_tokens_per_task": 9042.874285714286, "last_run_timestamp": "2026-10-31T20:03:54.586Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Venice", "accuracy": 0.696, "accuracy_std_dev": 0.036049398208569304, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.11530058400000001, "avg_duration_per_task_ms": 126878.5, "avg_output_tokens_per_task": 9663.674, "last_run_timestamp": "2026-10-31T21:47:15.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Reka", "accuracy": 0.6958335, "accuracy_std_dev": 0.03781341986980283, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.132227115, "avg_duration_per_task_ms": 205681.995, "avg_output_tokens_per_task": 9407.695, "last_run_timestamp": "2026-10-01T22:50:26.018Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "InferenceNet", "accuracy": 0.6957762602739725, "accuracy_std_dev": 0.0313433490574777, "run_count": 6, "total_questions": 292, "avg_cost_per_task_usd": 0.011891609589041095, "avg_duration_per_task_ms": 271916.02397260274, "avg_output_tokens_per_task": 12921.842465753425, "last_run_timestamp": "2026-11-04T21:26:44.978Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Inceptron", "accuracy": 0.6957576363636363, "accuracy_std_dev": 0.027600759894916997, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.07744747818181819, "avg_duration_per_task_ms": 159605.20727272728, "avg_output_tokens_per_task": 9554.076363636364, "last_run_timestamp": "2026-10-31T19:41:05.183Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DeepInfra", "accuracy": 0.69499975, "accuracy_std_dev": 0.014813722993815583, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.02174445, "avg_duration_per_task_ms": 448918.0325, "avg_output_tokens_per_task": 18617.01, "last_run_timestamp": "2026-10-31T19:50:01.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Friendli", "accuracy": 0.6945454545454546, "accuracy_std_dev": 0.05278641023693957, "run_count": 11, "total_questions": 550, "avg_cost_per_task_usd": 0.11215429818181818, "avg_duration_per_task_ms": 85066.3509090909, "avg_output_tokens_per_task": 9316.025454545455, "last_run_timestamp": "2026-10-31T19:47:27.755Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Ambient", "accuracy": 0.6942857142857143, "accuracy_std_dev": 0.025305290602753317, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.049070077142857146, "avg_duration_per_task_ms": 249231.63428571427, "avg_output_tokens_per_task": 8147.505714285714, "last_run_timestamp": "2026-10-31T20:09:59.492Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "model_slug": "z-ai/glm-5.1", "model_name": "Z.ai: GLM 5.1", "provider_name": "Friendli", "accuracy": 0.6941667499999999, "accuracy_std_dev": 0.05957600071494812, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0977737325, "avg_duration_per_task_ms": 138301.6425, "avg_output_tokens_per_task": 9130.415, "last_run_timestamp": "2026-10-31T19:19:57.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "StreamLake", "accuracy": 0.6933334999999999, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04279356, "avg_duration_per_task_ms": 161286, "avg_output_tokens_per_task": 8223.72, "last_run_timestamp": "2026-08-29T17:03:30.001Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-pro-20260709", "model_slug": "openai/gpt-5.6-luna-pro", "model_name": "OpenAI: GPT-5.6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6933334999999999, "accuracy_std_dev": 0.018257236010689022, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.08853248000000001, "avg_duration_per_task_ms": 338024.125, "avg_output_tokens_per_task": 32557.16, "last_run_timestamp": "2026-10-01T16:01:04.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "OpenAI", "accuracy": 0.6933333333333334, "accuracy_std_dev": 0.014401903168987411, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03953364, "avg_duration_per_task_ms": 309829.13333333336, "avg_output_tokens_per_task": 32261.04, "last_run_timestamp": "2026-10-02T23:10:33.891Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "model_slug": "minimax/minimax-m2.7", "model_name": "MiniMax: MiniMax M2.7", "provider_name": "DeepInfra", "accuracy": 0.693333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02093496, "avg_duration_per_task_ms": 185229.42, "avg_output_tokens_per_task": 9197.38, "last_run_timestamp": "2026-08-29T16:45:30.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Parasail", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.054513559999999996, "avg_duration_per_task_ms": 221323.18, "avg_output_tokens_per_task": 6973.32, "last_run_timestamp": "2026-08-22T08:43:02.347Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12999802000000002, "avg_duration_per_task_ms": 96250.54, "avg_output_tokens_per_task": 4982.06, "last_run_timestamp": "2026-10-01T08:05:35.867Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "reflection/beam-1.0-501b-a23b-20261016", "model_slug": "reflection/beam-1.0-501b-a23b", "model_name": "Reflection: Beam 1.0 501B A23B", "provider_name": "auto-routing", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02754302, "avg_duration_per_task_ms": 209408.8, "avg_output_tokens_per_task": 14914.54, "last_run_timestamp": "2026-10-24T12:03:24.771Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Venice", "accuracy": 0.693333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00621348, "avg_duration_per_task_ms": 180391.96, "avg_output_tokens_per_task": 9984.56, "last_run_timestamp": "2026-08-23T00:00:19.944Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Cloudflare", "accuracy": 0.6930555833333333, "accuracy_std_dev": 0.0348665084368422, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.12472750499999999, "avg_duration_per_task_ms": 130389.57833333334, "avg_output_tokens_per_task": 9036.746666666666, "last_run_timestamp": "2026-10-31T20:01:39.270Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "DeepInfra", "accuracy": 0.6926297185929648, "accuracy_std_dev": 0.014566408256670558, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.04924894974874372, "avg_duration_per_task_ms": 416469.09547738696, "avg_output_tokens_per_task": 8270.989949748744, "last_run_timestamp": "2026-10-31T21:01:44.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "model_slug": "deepseek/deepseek-v4-pro-0813", "model_name": "DeepSeek: DeepSeek V4 Pro 0813", "provider_name": "DigitalOcean", "accuracy": 0.6918933673469387, "accuracy_std_dev": 0.11325810487735402, "run_count": 12, "total_questions": 588, "avg_cost_per_task_usd": 0.3106834897959183, "avg_duration_per_task_ms": 1590211.7602040817, "avg_output_tokens_per_task": 51719.284013605444, "last_run_timestamp": "2026-10-31T12:54:39.225Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "BaseTen", "accuracy": 0.6909721458333333, "accuracy_std_dev": 0.0024635000000000074, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.0246803125, "avg_duration_per_task_ms": 48638.177083333336, "avg_output_tokens_per_task": 11672.458333333334, "last_run_timestamp": "2026-10-31T11:12:51.723Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-flash-lite-20260507", "model_slug": "google/gemini-3.1-flash-lite", "model_name": "Google: Gemini 3.1 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.69, "accuracy_std_dev": 0.003332999999999975, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10882898999999999, "avg_duration_per_task_ms": 135482.83, "avg_output_tokens_per_task": 51335.48, "last_run_timestamp": "2026-10-01T19:05:16.548Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Minimax", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01750064, "avg_duration_per_task_ms": 190406.46, "avg_output_tokens_per_task": 15714.76, "last_run_timestamp": "2026-10-31T12:31:30.661Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "PrimeIntellect", "accuracy": 0.69, "accuracy_std_dev": 0.016667000000000043, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01877389, "avg_duration_per_task_ms": 125629.58, "avg_output_tokens_per_task": 15732.46, "last_run_timestamp": "2026-10-31T11:20:36.173Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-pro-20260903", "model_slug": "openai/gpt-6-astra-pro", "model_name": "OpenAI: GPT-6 Astra Pro", "provider_name": "Azure", "accuracy": 0.69, "accuracy_std_dev": 0.009999999999999953, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 2.99530482, "avg_duration_per_task_ms": 320001.13, "avg_output_tokens_per_task": 12475.35, "last_run_timestamp": "2026-10-01T08:05:20.373Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "BaseTen", "accuracy": 0.69, "accuracy_std_dev": 0.04333300000000001, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07366, "avg_duration_per_task_ms": 63569.01, "avg_output_tokens_per_task": 3649.5, "last_run_timestamp": "2026-09-12T08:11:26.919Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "Mistral", "accuracy": 0.6899999117647057, "accuracy_std_dev": 0.026333763959163144, "run_count": 34, "total_questions": 1700, "avg_cost_per_task_usd": 0.10328957000000001, "avg_duration_per_task_ms": 78259.03647058824, "avg_output_tokens_per_task": 9525.96294117647, "last_run_timestamp": "2026-11-11T03:23:11.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "CoreWeave", "accuracy": 0.6895238571428572, "accuracy_std_dev": 0.0390126672850483, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06955698857142856, "avg_duration_per_task_ms": 75132.57428571429, "avg_output_tokens_per_task": 10137.50857142857, "last_run_timestamp": "2026-10-31T19:23:52.545Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "GMICloud", "accuracy": 0.687499875, "accuracy_std_dev": 0.10511576786029475, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.01592138, "avg_duration_per_task_ms": 247628.705, "avg_output_tokens_per_task": 11536.2, "last_run_timestamp": "2026-11-10T07:17:44.193Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "model_slug": "deepseek/deepseek-v4-flash", "model_name": "DeepSeek: DeepSeek V4 Flash 0423", "provider_name": "Mancer 2", "accuracy": 0.6872223333333333, "accuracy_std_dev": 0.08398462234374948, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.044577226666666664, "avg_duration_per_task_ms": 505556.9866666667, "avg_output_tokens_per_task": 15808.26, "last_run_timestamp": "2026-10-31T19:37:26.520Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-20260723", "model_slug": "inclusionai/ling-3.0-flash", "model_name": "inclusionAI: Ling 3.0 Flash", "provider_name": "auto-routing", "accuracy": 0.686667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00219582, "avg_duration_per_task_ms": 78121.92, "avg_output_tokens_per_task": 11659.46, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Wafer", "accuracy": 0.6866665000000001, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01561806, "avg_duration_per_task_ms": 117250.14, "avg_output_tokens_per_task": 14519.81, "last_run_timestamp": "2026-10-24T12:06:50.408Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "Azure", "accuracy": 0.6850001666666666, "accuracy_std_dev": 0.0143694957278334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04138883, "avg_duration_per_task_ms": 242662.93333333332, "avg_output_tokens_per_task": 30866.096666666668, "last_run_timestamp": "2026-10-01T08:05:57.699Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "auto-routing", "accuracy": 0.6847617142857143, "accuracy_std_dev": 0.047771144717629, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01437896, "avg_duration_per_task_ms": 347803.89714285714, "avg_output_tokens_per_task": 20220.391428571427, "last_run_timestamp": "2026-10-31T20:18:22.353Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "model_slug": "z-ai/glm-5.2", "model_name": "Z.ai: GLM 5.2", "provider_name": "GMICloud", "accuracy": 0.6844444166666667, "accuracy_std_dev": 0.04779057451432995, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.10127122666666667, "avg_duration_per_task_ms": 175036.245, "avg_output_tokens_per_task": 7013.575, "last_run_timestamp": "2026-11-10T06:53:22.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Amazon Bedrock", "accuracy": 0.6844443333333333, "accuracy_std_dev": 0.016178098288186456, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.06359924333333333, "avg_duration_per_task_ms": 60968.18666666667, "avg_output_tokens_per_task": 4743.64, "last_run_timestamp": "2026-11-02T17:36:19.479Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6841667499999999, "accuracy_std_dev": 0.019982613972838976, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.00873134, "avg_duration_per_task_ms": 302344.745, "avg_output_tokens_per_task": 8518.3375, "last_run_timestamp": "2026-11-03T03:04:12.747Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "PrimeIntellect", "accuracy": 0.6833334999999999, "accuracy_std_dev": 0.023333500000000007, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.018626550000000002, "avg_duration_per_task_ms": 239747.89, "avg_output_tokens_per_task": 15949.04, "last_run_timestamp": "2026-10-31T19:36:36.917Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Venice", "accuracy": 0.6833334999999999, "accuracy_std_dev": 0.03666649999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0168126, "avg_duration_per_task_ms": 195045.15, "avg_output_tokens_per_task": 9021.87, "last_run_timestamp": "2026-08-29T17:28:21.827Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "Azure", "accuracy": 0.6833333333333333, "accuracy_std_dev": 0.012619766977070334, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.10710260999999997, "avg_duration_per_task_ms": 135398.91, "avg_output_tokens_per_task": 4027.3166666666666, "last_run_timestamp": "2026-10-01T08:06:09.665Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "model_slug": "openai/gpt-5.6-luna", "model_name": "OpenAI: GPT-5.6 Luna", "provider_name": "Amazon Bedrock", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.021998563367234276, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.014046139999999999, "avg_duration_per_task_ms": 71689.16, "avg_output_tokens_per_task": 5022.28, "last_run_timestamp": "2026-10-30T00:15:40.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "model_slug": "openai/gpt-6.1-sol", "model_name": "OpenAI: GPT-6.1 Sol", "provider_name": "OpenAI", "accuracy": 0.6822223333333333, "accuracy_std_dev": 0.011330936570096727, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.09746202, "avg_duration_per_task_ms": 132162.47333333333, "avg_output_tokens_per_task": 3507.6066666666666, "last_run_timestamp": "2026-10-02T23:10:34.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Cloudflare", "accuracy": 0.6817421155778893, "accuracy_std_dev": 0.023817302869644456, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.02058146231155779, "avg_duration_per_task_ms": 307187.45477386937, "avg_output_tokens_per_task": 16771.46984924623, "last_run_timestamp": "2026-10-31T20:14:20.109Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "Together", "accuracy": 0.6813334, "accuracy_std_dev": 0.0026667999999999913, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.08704178, "avg_duration_per_task_ms": 82752.556, "avg_output_tokens_per_task": 3576.016, "last_run_timestamp": "2026-10-01T16:01:04.859Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "CoreWeave", "accuracy": 0.68125, "accuracy_std_dev": 0.0202715728731147, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0174918175, "avg_duration_per_task_ms": 123091.5525, "avg_output_tokens_per_task": 21340.095, "last_run_timestamp": "2026-10-31T11:46:38.147Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "Azure", "accuracy": 0.6811111666666666, "accuracy_std_dev": 0.008088865998326882, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00903582, "avg_duration_per_task_ms": 105469.92666666667, "avg_output_tokens_per_task": 8459.866666666667, "last_run_timestamp": "2026-10-01T08:05:22.516Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "model_slug": "qwen/qwen3.5-35b-a3b", "model_name": "Qwen: Qwen3.5-35B-A3B", "provider_name": "AtlasCloud", "accuracy": 0.6809524285714286, "accuracy_std_dev": 0.09813702854661528, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09356050285714285, "avg_duration_per_task_ms": 244745.57714285713, "avg_output_tokens_per_task": 21782.17142857143, "last_run_timestamp": "2026-10-31T19:46:05.991Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Novita", "accuracy": 0.6808944146341463, "accuracy_std_dev": 0.01806743167802219, "run_count": 5, "total_questions": 246, "avg_cost_per_task_usd": 0.033741605691056914, "avg_duration_per_task_ms": 453897.35772357724, "avg_output_tokens_per_task": 18491.044715447155, "last_run_timestamp": "2026-10-17T19:21:01.980Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.2-20261109", "model_slug": "anthropic/claude-fable-5.2", "model_name": "Anthropic: Claude Fable 5.2", "provider_name": "Google", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.79080038, "avg_duration_per_task_ms": 125754.98, "avg_output_tokens_per_task": 9356.94, "last_run_timestamp": "2026-11-10T18:07:28.806Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.2-20261109", "model_slug": "anthropic/claude-fable-5.2", "model_name": "Anthropic: Claude Fable 5.2", "provider_name": "Azure", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.77161558, "avg_duration_per_task_ms": 159103.76, "avg_output_tokens_per_task": 9152.76, "last_run_timestamp": "2026-11-10T18:07:20.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "auto-routing", "accuracy": 0.68, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.58024224, "avg_duration_per_task_ms": 141487.98, "avg_output_tokens_per_task": 3731.58, "last_run_timestamp": "2026-10-01T08:05:19.702Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "OpenAI", "accuracy": 0.68, "accuracy_std_dev": 0.03810292105162893, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.00883898, "avg_duration_per_task_ms": 114396.12, "avg_output_tokens_per_task": 8841.28, "last_run_timestamp": "2026-10-02T23:08:44.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "DekaLLM", "accuracy": 0.6799998571428572, "accuracy_std_dev": 0.0198405229080613, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.015568199999999999, "avg_duration_per_task_ms": 259045.84285714285, "avg_output_tokens_per_task": 18504.7, "last_run_timestamp": "2026-10-31T20:09:24.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "NextBit", "accuracy": 0.6793334, "accuracy_std_dev": 0.03508080956933576, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.016317446, "avg_duration_per_task_ms": 294258.966, "avg_output_tokens_per_task": 19934.788, "last_run_timestamp": "2026-11-01T13:16:57.769Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "Tenstorrent", "accuracy": 0.67833325, "accuracy_std_dev": 0.019075602511781895, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01068972, "avg_duration_per_task_ms": 147233.725, "avg_output_tokens_per_task": 15363.5, "last_run_timestamp": "2026-11-04T18:22:16.954Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-vision-exp-20260821", "model_slug": "deepseek/deepseek-v4-flash-vision-exp", "model_name": "DeepSeek: DeepSeek V4 Flash Vision Exp", "provider_name": "Fireworks", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.013698571612479227, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0340762, "avg_duration_per_task_ms": 244093.44, "avg_output_tokens_per_task": 27086.806666666667, "last_run_timestamp": "2026-09-19T08:01:53.393Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-terra-20261026", "model_slug": "openai/gpt-6.1-terra", "model_name": "OpenAI: GPT-6.1 Terra", "provider_name": "Azure", "accuracy": 0.6777776666666667, "accuracy_std_dev": 0.017497908167041606, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06315626, "avg_duration_per_task_ms": 88978.8, "avg_output_tokens_per_task": 4762.613333333334, "last_run_timestamp": "2026-11-03T15:08:23.560Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Io Net", "accuracy": 0.6767563901345292, "accuracy_std_dev": 0.029269691852423077, "run_count": 9, "total_questions": 446, "avg_cost_per_task_usd": 0.019482901345291483, "avg_duration_per_task_ms": 358550.09865470853, "avg_output_tokens_per_task": 16851.491031390135, "last_run_timestamp": "2026-11-11T08:13:05.865Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "auto-routing", "accuracy": 0.676667, "accuracy_std_dev": 0.030000000000000027, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02188514, "avg_duration_per_task_ms": 255892.84, "avg_output_tokens_per_task": 10205.01, "last_run_timestamp": "2026-08-29T16:11:34.887Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "OpenAI", "accuracy": 0.6766665000000001, "accuracy_std_dev": 0.0033335000000000448, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.5967202500000001, "avg_duration_per_task_ms": 137724.15, "avg_output_tokens_per_task": 4015.19, "last_run_timestamp": "2026-10-02T23:06:01.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "DigitalOcean", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.00831485358994507, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.014157986666666667, "avg_duration_per_task_ms": 117427.1, "avg_output_tokens_per_task": 13587.66, "last_run_timestamp": "2026-10-31T11:44:53.544Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "model_slug": "openai/gpt-6-astra", "model_name": "OpenAI: GPT-6 Astra", "provider_name": "Amazon Bedrock", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.63842362, "avg_duration_per_task_ms": 88201.35333333333, "avg_output_tokens_per_task": 3898.48, "last_run_timestamp": "2026-10-30T00:17:00.667Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "DeepInfra", "accuracy": 0.6755556666666667, "accuracy_std_dev": 0.006285236475706844, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07120372000000001, "avg_duration_per_task_ms": 144319.38666666666, "avg_output_tokens_per_task": 2561.846666666667, "last_run_timestamp": "2026-10-08T22:25:52.178Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "Tenstorrent", "accuracy": 0.675, "accuracy_std_dev": 0.011667000000000038, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.1079227, "avg_duration_per_task_ms": 145751.42, "avg_output_tokens_per_task": 18113.26, "last_run_timestamp": "2026-10-04T05:20:00.959Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "GMICloud", "accuracy": 0.6738889166666666, "accuracy_std_dev": 0.14166895499276846, "run_count": 12, "total_questions": 600, "avg_cost_per_task_usd": 0.11663828999999999, "avg_duration_per_task_ms": 354430.585, "avg_output_tokens_per_task": 8766.405, "last_run_timestamp": "2026-11-10T06:25:16.940Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "DeepInfra", "accuracy": 0.6734008686868687, "accuracy_std_dev": 0.01340150000000001, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.006025121212121212, "avg_duration_per_task_ms": 197802.63636363635, "avg_output_tokens_per_task": 8819.454545454546, "last_run_timestamp": "2026-08-30T00:01:09.439Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "model_slug": "openai/gpt-5", "model_name": "OpenAI: GPT-5", "provider_name": "auto-routing", "accuracy": 0.6733334999999999, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.6292479799999999, "avg_duration_per_task_ms": 729264.79, "avg_output_tokens_per_task": 54931.75, "last_run_timestamp": "2026-10-01T16:01:02.610Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "model_slug": "moonshotai/kimi-k2.6", "model_name": "MoonshotAI: Kimi K2.6", "provider_name": "Decart", "accuracy": 0.6733333333333333, "accuracy_std_dev": 0.14616051044960432, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07790980666666668, "avg_duration_per_task_ms": 178517.31333333332, "avg_output_tokens_per_task": 18244.56, "last_run_timestamp": "2026-10-03T16:16:30.145Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-pro-20260709", "model_slug": "openai/gpt-5.6-sol-pro", "model_name": "OpenAI: GPT-5.6 Sol Pro", "provider_name": "Azure", "accuracy": 0.6733330000000001, "accuracy_std_dev": 0.019999999999999962, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 1.5819717300000002, "avg_duration_per_task_ms": 314318.37, "avg_output_tokens_per_task": 23725.55, "last_run_timestamp": "2026-10-01T16:01:13.647Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "model_slug": "openai/gpt-6-luna", "model_name": "OpenAI: GPT-6 Luna", "provider_name": "auto-routing", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00928508, "avg_duration_per_task_ms": 121838.98, "avg_output_tokens_per_task": 8898.44, "last_run_timestamp": "2026-10-01T08:00:22.469Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Io Net", "accuracy": 0.673333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0059145000000000005, "avg_duration_per_task_ms": 456667.98, "avg_output_tokens_per_task": 17189.68, "last_run_timestamp": "2026-11-11T15:53:31.309Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "model_slug": "deepseek/deepseek-v4-flash-0731", "model_name": "DeepSeek: DeepSeek V4 Flash 0731", "provider_name": "DigitalOcean", "accuracy": 0.6725294120603015, "accuracy_std_dev": 0.11099160244860588, "run_count": 8, "total_questions": 398, "avg_cost_per_task_usd": 0.03270493216080402, "avg_duration_per_task_ms": 1961063.7788944724, "avg_output_tokens_per_task": 69160.1582914573, "last_run_timestamp": "2026-10-31T11:55:35.800Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "model_slug": "thinkingmachines/inkling", "model_name": "Thinking Machines: Inkling", "provider_name": "auto-routing", "accuracy": 0.672, "accuracy_std_dev": 0.028409779224766934, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.07962264, "avg_duration_per_task_ms": 80557.024, "avg_output_tokens_per_task": 3531.236, "last_run_timestamp": "2026-10-01T16:00:45.064Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Minimax", "accuracy": 0.67166675, "accuracy_std_dev": 0.012802094015726486, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.035549775, "avg_duration_per_task_ms": 154518.065, "avg_output_tokens_per_task": 9811.9, "last_run_timestamp": "2026-08-29T17:02:26.936Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Cloudflare", "accuracy": 0.671111, "accuracy_std_dev": 0.016629528997138363, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013801693333333332, "avg_duration_per_task_ms": 193265.92, "avg_output_tokens_per_task": 8641.586666666666, "last_run_timestamp": "2026-10-13T22:01:15.953Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "model_slug": "openai/gpt-6-sol", "model_name": "OpenAI: GPT-6 Sol", "provider_name": "Azure", "accuracy": 0.6706668, "accuracy_std_dev": 0.015548970672041286, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.142799016, "avg_duration_per_task_ms": 115124.684, "avg_output_tokens_per_task": 5068.812, "last_run_timestamp": "2026-10-01T08:04:38.837Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "Google", "accuracy": 0.6702638057553957, "accuracy_std_dev": 0.01542992310055007, "run_count": 3, "total_questions": 139, "avg_cost_per_task_usd": 0.11313071942446042, "avg_duration_per_task_ms": 363635, "avg_output_tokens_per_task": 8911.287769784172, "last_run_timestamp": "2026-10-17T19:51:53.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "Azure", "accuracy": 0.6683335000000001, "accuracy_std_dev": 0.01190259075789804, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.097154565, "avg_duration_per_task_ms": 138323.315, "avg_output_tokens_per_task": 13048.88, "last_run_timestamp": "2026-10-01T19:04:42.436Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "GMICloud", "accuracy": 0.668, "accuracy_std_dev": 0.08103784026983935, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.021783728, "avg_duration_per_task_ms": 280094.124, "avg_output_tokens_per_task": 19729.836, "last_run_timestamp": "2026-11-10T06:31:54.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-pro-20260922", "model_slug": "openai/gpt-6-luna-pro", "model_name": "OpenAI: GPT-6 Luna Pro", "provider_name": "auto-routing", "accuracy": 0.666667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.038762819999999996, "avg_duration_per_task_ms": 314594.08, "avg_output_tokens_per_task": 31952.48, "last_run_timestamp": "2026-10-01T08:00:24.890Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "PrimeIntellect", "accuracy": 0.6666666666666666, "accuracy_std_dev": 0.0524933858267454, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03491572666666667, "avg_duration_per_task_ms": 274075.64666666667, "avg_output_tokens_per_task": 23860.97333333333, "last_run_timestamp": "2026-10-31T19:43:40.322Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "DigitalOcean", "accuracy": 0.6666665, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02906171, "avg_duration_per_task_ms": 246294.85, "avg_output_tokens_per_task": 9968.3, "last_run_timestamp": "2026-08-29T16:36:42.164Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3.1-flash-20261019", "model_slug": "minimax/minimax-m3.1-flash", "model_name": "MiniMax: MiniMax M3.1 Flash", "provider_name": "auto-routing", "accuracy": 0.6666665, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.013174700000000001, "avg_duration_per_task_ms": 181950.26, "avg_output_tokens_per_task": 13269.31, "last_run_timestamp": "2026-10-31T12:42:39.997Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "auto-routing", "accuracy": 0.6666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.08889954, "avg_duration_per_task_ms": 108433.06, "avg_output_tokens_per_task": 12861.73, "last_run_timestamp": "2026-10-01T19:04:42.468Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "auto-routing", "accuracy": 0.6666665, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00821814, "avg_duration_per_task_ms": 219118.91, "avg_output_tokens_per_task": 8875.03, "last_run_timestamp": "2026-08-30T00:01:02.161Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Near AI", "accuracy": 0.66583325, "accuracy_std_dev": 0.03639120582211448, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.006095729999999999, "avg_duration_per_task_ms": 325369.79, "avg_output_tokens_per_task": 18643.27, "last_run_timestamp": "2026-10-31T19:46:03.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Darkbloom", "accuracy": 0.6633329999999998, "accuracy_std_dev": 0.021602468994692835, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.007094233333333332, "avg_duration_per_task_ms": 456871.32, "avg_output_tokens_per_task": 20116.046666666665, "last_run_timestamp": "2026-11-13T01:53:22.455Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Amazon Bedrock", "accuracy": 0.6622221666666667, "accuracy_std_dev": 0.030323393570073177, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.12413147533333334, "avg_duration_per_task_ms": 118698.298, "avg_output_tokens_per_task": 12588.784666666666, "last_run_timestamp": "2026-10-31T12:26:00.225Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DekaLLM", "accuracy": 0.6604760000000001, "accuracy_std_dev": 0.025350253033394797, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.008717225714285714, "avg_duration_per_task_ms": 574345.4257142857, "avg_output_tokens_per_task": 17634.762857142858, "last_run_timestamp": "2026-10-31T20:07:05.983Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "SiliconFlow", "accuracy": 0.66, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.022123540000000004, "avg_duration_per_task_ms": 232117.3, "avg_output_tokens_per_task": 9442.58, "last_run_timestamp": "2026-08-29T16:50:17.446Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "model_slug": "minimax/minimax-m3", "model_name": "MiniMax: MiniMax M3", "provider_name": "Morph", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06369964, "avg_duration_per_task_ms": 706011.28, "avg_output_tokens_per_task": 8805.34, "last_run_timestamp": "2026-08-22T08:53:18.310Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "upstage/solar-pro4-20260810", "model_slug": "upstage/solar-pro4", "model_name": "Upstage: Solar Pro 4", "provider_name": "KT", "accuracy": 0.66, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0747839, "avg_duration_per_task_ms": 223467.62, "avg_output_tokens_per_task": 14969.76, "last_run_timestamp": "2026-11-11T02:34:15.627Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "DigitalOcean", "accuracy": 0.6580952857142857, "accuracy_std_dev": 0.07221803406691293, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.056178177142857146, "avg_duration_per_task_ms": 795605.24, "avg_output_tokens_per_task": 17815.014285714286, "last_run_timestamp": "2026-10-05T20:43:11.175Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Anthropic", "accuracy": 0.6580001, "accuracy_std_dev": 0.027656551565406706, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.11552560199999999, "avg_duration_per_task_ms": 129975.578, "avg_output_tokens_per_task": 12464.086, "last_run_timestamp": "2026-10-31T11:44:30.576Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "Together", "accuracy": 0.657778, "accuracy_std_dev": 0.0505769576190581, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.046507033333333336, "avg_duration_per_task_ms": 270981.30666666664, "avg_output_tokens_per_task": 3230.153333333333, "last_run_timestamp": "2026-09-12T08:12:16.067Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Parasail", "accuracy": 0.6574998750000001, "accuracy_std_dev": 0.012555363509646976, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0252459125, "avg_duration_per_task_ms": 148223.2925, "avg_output_tokens_per_task": 17051.0175, "last_run_timestamp": "2026-11-05T00:31:57.460Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "SiliconFlow", "accuracy": 0.6573334, "accuracy_std_dev": 0.1373785957776538, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.06366923599999999, "avg_duration_per_task_ms": 573529.892, "avg_output_tokens_per_task": 10164.08, "last_run_timestamp": "2026-10-31T19:32:40.676Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "AkashML", "accuracy": 0.6573334, "accuracy_std_dev": 0.02284235636939995, "run_count": 15, "total_questions": 750, "avg_cost_per_task_usd": 0.009446806666666667, "avg_duration_per_task_ms": 308475.5133333333, "avg_output_tokens_per_task": 17229.645333333334, "last_run_timestamp": "2026-10-31T19:05:31.409Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "auto-routing", "accuracy": 0.6566666, "accuracy_std_dev": 0.02720294101820609, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.11512897400000001, "avg_duration_per_task_ms": 122868.958, "avg_output_tokens_per_task": 12575.444, "last_run_timestamp": "2026-10-31T12:42:18.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Friendli", "accuracy": 0.6566665, "accuracy_std_dev": 0.0033334999999999892, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.029754, "avg_duration_per_task_ms": 250765.5, "avg_output_tokens_per_task": 11169.75, "last_run_timestamp": "2026-08-29T16:14:58.314Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "SiliconFlow", "accuracy": 0.656492308139535, "accuracy_std_dev": 0.13396399619194066, "run_count": 7, "total_questions": 344, "avg_cost_per_task_usd": 0.02718404941860465, "avg_duration_per_task_ms": 1177043.0639534884, "avg_output_tokens_per_task": 35003.26453488372, "last_run_timestamp": "2026-10-17T21:28:32.796Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Crusoe", "accuracy": 0.6563219310344829, "accuracy_std_dev": 0.17320901072467976, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.02278490344827586, "avg_duration_per_task_ms": 272417.2068965517, "avg_output_tokens_per_task": 8725.041379310345, "last_run_timestamp": "2026-10-31T19:23:32.973Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "auto-routing", "accuracy": 0.6562239298245615, "accuracy_std_dev": 0.018946289616967236, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.01692722807017544, "avg_duration_per_task_ms": 312804.7944862155, "avg_output_tokens_per_task": 22811.754385964912, "last_run_timestamp": "2026-10-31T12:28:49.050Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Azure", "accuracy": 0.6560001, "accuracy_std_dev": 0.022745126952602394, "run_count": 10, "total_questions": 500, "avg_cost_per_task_usd": 0.113423644, "avg_duration_per_task_ms": 134974.778, "avg_output_tokens_per_task": 12396.082, "last_run_timestamp": "2026-10-31T11:42:13.882Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Phala", "accuracy": 0.6557823102040816, "accuracy_std_dev": 0.009318408855593308, "run_count": 5, "total_questions": 245, "avg_cost_per_task_usd": 0.014797057142857142, "avg_duration_per_task_ms": 277907.2775510204, "avg_output_tokens_per_task": 20245.579591836733, "last_run_timestamp": "2026-10-10T11:18:54.565Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "DeepInfra", "accuracy": 0.6555556666666666, "accuracy_std_dev": 0.03807065880409451, "run_count": 9, "total_questions": 450, "avg_cost_per_task_usd": 0.021039175555555557, "avg_duration_per_task_ms": 248791.5977777778, "avg_output_tokens_per_task": 26269.908888888887, "last_run_timestamp": "2026-10-31T12:25:21.635Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "OpenAI", "accuracy": 0.6555556666666666, "accuracy_std_dev": 0.012570472951413636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.030086579999999998, "avg_duration_per_task_ms": 125366.48, "avg_output_tokens_per_task": 16133.046666666667, "last_run_timestamp": "2026-10-02T23:09:54.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "model_slug": "anthropic/claude-haiku-4.5", "model_name": "Anthropic: Claude Haiku 4.5", "provider_name": "Google", "accuracy": 0.6546666666666666, "accuracy_std_dev": 0.025785448094268634, "run_count": 30, "total_questions": 1500, "avg_cost_per_task_usd": 0.11997013933333336, "avg_duration_per_task_ms": 121089.03933333333, "avg_output_tokens_per_task": 12528.349333333334, "last_run_timestamp": "2026-10-31T12:17:41.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Crusoe", "accuracy": 0.6546664, "accuracy_std_dev": 0.03512514515614133, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.01491196, "avg_duration_per_task_ms": 86567.944, "avg_output_tokens_per_task": 19737.508, "last_run_timestamp": "2026-10-31T19:08:06.670Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-lightning-20260807", "model_slug": "nvidia/nemotron-3.5-lightning", "model_name": "NVIDIA: Nemotron 3.5 Lightning", "provider_name": "Io Net", "accuracy": 0.6535409120879121, "accuracy_std_dev": 0.03689102528724027, "run_count": 11, "total_questions": 546, "avg_cost_per_task_usd": 0.012232007326007327, "avg_duration_per_task_ms": 1066692.706959707, "avg_output_tokens_per_task": 22783.287545787545, "last_run_timestamp": "2026-11-04T19:03:50.422Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "Novita", "accuracy": 0.6533335, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00488981, "avg_duration_per_task_ms": 111736.2, "avg_output_tokens_per_task": 17377.29, "last_run_timestamp": "2026-10-01T08:04:38.584Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Sail Research", "accuracy": 0.6533335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01241216, "avg_duration_per_task_ms": 170776.85, "avg_output_tokens_per_task": 17557.16, "last_run_timestamp": "2026-11-06T22:04:56.505Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "model_slug": "z-ai/glm-4.7-flash", "model_name": "Z.ai: GLM 4.7 Flash", "provider_name": "Novita", "accuracy": 0.6533335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00878474, "avg_duration_per_task_ms": 423822.14, "avg_output_tokens_per_task": 14588.95, "last_run_timestamp": "2026-08-30T00:01:09.470Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "model_slug": "qwen/qwen3.5-9b", "model_name": "Qwen: Qwen3.5-9B", "provider_name": "Venice", "accuracy": 0.653333375, "accuracy_std_dev": 0.07817377366472963, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.0266300325, "avg_duration_per_task_ms": 292622.5475, "avg_output_tokens_per_task": 25080.435, "last_run_timestamp": "2026-10-17T19:25:29.604Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Google", "accuracy": 0.6533333333333333, "accuracy_std_dev": 0.014401440259764123, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.2856746933333333, "avg_duration_per_task_ms": 183725.1, "avg_output_tokens_per_task": 8628.813333333334, "last_run_timestamp": "2026-09-01T13:04:20.137Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "Nebius", "accuracy": 0.653333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.032076099999999996, "avg_duration_per_task_ms": 129950.52, "avg_output_tokens_per_task": 8232.88, "last_run_timestamp": "2026-11-13T00:42:42.289Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "auto-routing", "accuracy": 0.652381, "accuracy_std_dev": 0.035711227806231656, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.01053958857142857, "avg_duration_per_task_ms": 546054.9028571428, "avg_output_tokens_per_task": 18131.937142857143, "last_run_timestamp": "2026-10-31T19:41:05.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Darkbloom", "accuracy": 0.6519705410821643, "accuracy_std_dev": 0.024087275055721835, "run_count": 10, "total_questions": 499, "avg_cost_per_task_usd": 0.012394563126252504, "avg_duration_per_task_ms": 729028.881763527, "avg_output_tokens_per_task": 25595.735470941883, "last_run_timestamp": "2026-11-13T02:07:14.326Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "AkashML", "accuracy": 0.65166675, "accuracy_std_dev": 0.028819327042585528, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.01676374, "avg_duration_per_task_ms": 406825.29, "avg_output_tokens_per_task": 27732.185, "last_run_timestamp": "2026-10-31T19:20:36.815Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Groq", "accuracy": 0.6490477142857143, "accuracy_std_dev": 0.03593273004504106, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.033029654285714284, "avg_duration_per_task_ms": 56698.462857142855, "avg_output_tokens_per_task": 14694.748571428572, "last_run_timestamp": "2026-10-31T18:54:40.763Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "model_slug": "openai/gpt-5.4-mini", "model_name": "OpenAI: GPT-5.4 Mini", "provider_name": "OpenAI", "accuracy": 0.6488889999999999, "accuracy_std_dev": 0.019115994210782406, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08501352666666667, "avg_duration_per_task_ms": 97755.09333333334, "avg_output_tokens_per_task": 12128.533333333333, "last_run_timestamp": "2026-10-02T23:10:27.235Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "auto-routing", "accuracy": 0.6466665, "accuracy_std_dev": 0.026666499999999982, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.0295618, "avg_duration_per_task_ms": 121397.75, "avg_output_tokens_per_task": 15953.93, "last_run_timestamp": "2026-10-01T19:04:28.503Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "DeepInfra", "accuracy": 0.6464647229437229, "accuracy_std_dev": 0.03922157828554563, "run_count": 14, "total_questions": 693, "avg_cost_per_task_usd": 0.02244592207792208, "avg_duration_per_task_ms": 314148.4891774892, "avg_output_tokens_per_task": 17992.321789321788, "last_run_timestamp": "2026-10-31T20:19:17.569Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "model_slug": "deepseek/deepseek-v4.1-flash", "model_name": "DeepSeek: DeepSeek V4.1 Flash", "provider_name": "Krea", "accuracy": 0.6455556666666666, "accuracy_std_dev": 0.09964144484545016, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.11177372666666668, "avg_duration_per_task_ms": 710817.72, "avg_output_tokens_per_task": 87845.5, "last_run_timestamp": "2026-09-29T19:45:03.655Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "model_slug": "openai/gpt-5.4-nano", "model_name": "OpenAI: GPT-5.4 Nano", "provider_name": "Azure", "accuracy": 0.6416670000000001, "accuracy_std_dev": 0.016583123951777013, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.029366035, "avg_duration_per_task_ms": 220039.595, "avg_output_tokens_per_task": 15350.045, "last_run_timestamp": "2026-10-01T19:05:16.543Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "model_slug": "anthropic/claude-sonnet-4", "model_name": "Anthropic: Claude Sonnet 4", "provider_name": "Amazon Bedrock", "accuracy": 0.64, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.29234208, "avg_duration_per_task_ms": 227202.07, "avg_output_tokens_per_task": 8702.01, "last_run_timestamp": "2026-09-01T13:00:51.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "DeepInfra", "accuracy": 0.64, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01684376, "avg_duration_per_task_ms": 1178898.4, "avg_output_tokens_per_task": 43776.78, "last_run_timestamp": "2026-10-09T00:39:20.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "Together", "accuracy": 0.64, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.026583100000000002, "avg_duration_per_task_ms": 49494.66, "avg_output_tokens_per_task": 8435.96, "last_run_timestamp": "2026-11-13T00:16:41.107Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "AtlasCloud", "accuracy": 0.6388886666666667, "accuracy_std_dev": 0.09896984825805394, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04814041333333333, "avg_duration_per_task_ms": 256985.87333333332, "avg_output_tokens_per_task": 7413.113333333334, "last_run_timestamp": "2026-10-03T16:13:53.555Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "inclusionai/ling-3.0-flash-fin-20260827", "model_slug": "inclusionai/ling-3.0-flash-fin", "model_name": "inclusionAI: Ling 3.0 Flash Fin", "provider_name": "DeepInfra", "accuracy": 0.638298, "accuracy_std_dev": null, "run_count": 1, "total_questions": 47, "avg_cost_per_task_usd": 0.020538510638297872, "avg_duration_per_task_ms": 686659.4680851063, "avg_output_tokens_per_task": 47094.31914893617, "last_run_timestamp": "2026-10-09T00:23:16.829Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Venice", "accuracy": 0.6355553333333334, "accuracy_std_dev": 0.003142853940113798, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.008414553333333331, "avg_duration_per_task_ms": 1275373.5733333332, "avg_output_tokens_per_task": 17754.266666666666, "last_run_timestamp": "2026-10-31T21:55:54.854Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Venice", "accuracy": 0.633333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.21351704000000002, "avg_duration_per_task_ms": 877395.36, "avg_output_tokens_per_task": 27870.24, "last_run_timestamp": "2026-08-22T16:17:16.779Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "Modal", "accuracy": 0.633333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.042546600000000004, "avg_duration_per_task_ms": 200101.72, "avg_output_tokens_per_task": 26828.8, "last_run_timestamp": "2026-11-13T06:54:36.403Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "PrimeIntellect", "accuracy": 0.633333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01754236, "avg_duration_per_task_ms": 112671.48, "avg_output_tokens_per_task": 8415.52, "last_run_timestamp": "2026-11-13T03:43:05.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Phala", "accuracy": 0.6331658693467337, "accuracy_std_dev": 0.03188449882289982, "run_count": 4, "total_questions": 199, "avg_cost_per_task_usd": 0.03419022110552764, "avg_duration_per_task_ms": 256833.6432160804, "avg_output_tokens_per_task": 16666.94974874372, "last_run_timestamp": "2026-10-31T19:04:42.700Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mara", "accuracy": 0.6326291619718309, "accuracy_std_dev": 0.04710741386094833, "run_count": 3, "total_questions": 142, "avg_cost_per_task_usd": 0.022965091549295776, "avg_duration_per_task_ms": 52073.43661971831, "avg_output_tokens_per_task": 11681.894366197183, "last_run_timestamp": "2026-10-24T18:40:18.531Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-nano-banana-2.1-20261006", "model_slug": "google/gemini-nano-banana-2.1", "model_name": "Google: Nano Banana 2.1", "provider_name": "Google", "accuracy": 0.631944, "accuracy_std_dev": null, "run_count": 1, "total_questions": 48, "avg_cost_per_task_usd": 0.9708985208333334, "avg_duration_per_task_ms": 253667.60416666666, "avg_output_tokens_per_task": 72982.125, "last_run_timestamp": "2026-11-12T21:07:31.630Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "SambaNova", "accuracy": 0.6299079172932331, "accuracy_std_dev": 0.02774706876297926, "run_count": 8, "total_questions": 399, "avg_cost_per_task_usd": 0.04092825313283208, "avg_duration_per_task_ms": 87144.26065162907, "avg_output_tokens_per_task": 19555.696741854637, "last_run_timestamp": "2026-11-11T19:01:52.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Cerebras", "accuracy": 0.627619, "accuracy_std_dev": 0.022090270379254585, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06924706571428572, "avg_duration_per_task_ms": 33379.60857142857, "avg_output_tokens_per_task": 21963.825714285715, "last_run_timestamp": "2026-10-31T19:45:42.530Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "auto-routing", "accuracy": 0.626667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12019484, "avg_duration_per_task_ms": 635855.02, "avg_output_tokens_per_task": 26775.66, "last_run_timestamp": "2026-08-22T16:01:40.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-2-small-20261107", "model_slug": "thinkingmachines/inkling-2-small", "model_name": "Thinking Machines: Inkling 2 Small", "provider_name": "Cloudflare", "accuracy": 0.626667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0507923, "avg_duration_per_task_ms": 58773.58, "avg_output_tokens_per_task": 11296.32, "last_run_timestamp": "2026-11-13T00:15:45.631Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3.5-super-vl-20261015", "model_slug": "nvidia/nemotron-3.5-super-vl", "model_name": "NVIDIA: Nemotron 3.5 Super VL", "provider_name": "Crusoe", "accuracy": 0.6266666666666667, "accuracy_std_dev": 0.14676766724392065, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03268331333333334, "avg_duration_per_task_ms": 135271.32666666666, "avg_output_tokens_per_task": 12787.166666666666, "last_run_timestamp": "2026-10-31T11:51:01.680Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Venice", "accuracy": 0.6238095714285714, "accuracy_std_dev": 0.058538454520712026, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.023625271428571425, "avg_duration_per_task_ms": 375289.0742857143, "avg_output_tokens_per_task": 15378.111428571428, "last_run_timestamp": "2026-10-31T19:07:04.114Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "Novita", "accuracy": 0.6233335, "accuracy_std_dev": 0.01666650000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02295258, "avg_duration_per_task_ms": 131170.99, "avg_output_tokens_per_task": 9602.23, "last_run_timestamp": "2026-08-29T16:49:08.574Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "AtlasCloud", "accuracy": 0.62, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02806645, "avg_duration_per_task_ms": 246864.07, "avg_output_tokens_per_task": 9796.43, "last_run_timestamp": "2026-08-29T16:13:54.380Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "PrimeIntellect", "accuracy": 0.62, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.006431559999999999, "avg_duration_per_task_ms": 321502.32, "avg_output_tokens_per_task": 15491.98, "last_run_timestamp": "2026-11-01T06:36:42.081Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Mancer 2", "accuracy": 0.6173370344827587, "accuracy_std_dev": 0.07436853642639527, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.027499977011494252, "avg_duration_per_task_ms": 1452543.2729885057, "avg_output_tokens_per_task": 49684.43103448276, "last_run_timestamp": "2026-10-31T19:40:44.417Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "auto-routing", "accuracy": 0.6133333333333333, "accuracy_std_dev": 0.024944382578492966, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04715752, "avg_duration_per_task_ms": 100008.15333333334, "avg_output_tokens_per_task": 4739.7266666666665, "last_run_timestamp": "2026-10-03T16:07:50.329Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "model_slug": "deepseek/deepseek-v3.2", "model_name": "DeepSeek: DeepSeek V3.2", "provider_name": "AtlasCloud", "accuracy": 0.6133332857142857, "accuracy_std_dev": 0.12608878454226982, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.051117977142857145, "avg_duration_per_task_ms": 942298.3057142857, "avg_output_tokens_per_task": 28768.634285714284, "last_run_timestamp": "2026-10-31T19:22:14.979Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "DeepInfra", "accuracy": 0.6133331666666667, "accuracy_std_dev": 0.031972140650972335, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.04608945666666667, "avg_duration_per_task_ms": 205329.73, "avg_output_tokens_per_task": 3416.34, "last_run_timestamp": "2026-10-08T22:20:40.622Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Alibaba", "accuracy": 0.613333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.12290606, "avg_duration_per_task_ms": 597000.58, "avg_output_tokens_per_task": 27061.54, "last_run_timestamp": "2026-08-22T16:05:00.942Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "StreamLake", "accuracy": 0.6091664999999999, "accuracy_std_dev": 0.023018682026779924, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.048386775, "avg_duration_per_task_ms": 95869.55, "avg_output_tokens_per_task": 4703.465, "last_run_timestamp": "2026-10-03T16:14:40.781Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "auto-routing", "accuracy": 0.606667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03212956, "avg_duration_per_task_ms": 335001.52, "avg_output_tokens_per_task": 8851.18, "last_run_timestamp": "2026-10-01T08:03:01.921Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "model_slug": "thinkingmachines/inkling-small", "model_name": "Thinking Machines: Inkling Small", "provider_name": "auto-routing", "accuracy": 0.6066668000000001, "accuracy_std_dev": 0.026666399999999996, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.046513716, "avg_duration_per_task_ms": 209556.196, "avg_output_tokens_per_task": 3673.824, "last_run_timestamp": "2026-10-01T16:00:32.452Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "model_slug": "minimax/minimax-m2.5", "model_name": "MiniMax: MiniMax M2.5", "provider_name": "StreamLake", "accuracy": 0.6066665, "accuracy_std_dev": 0.026666500000000037, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02093522, "avg_duration_per_task_ms": 280645.76, "avg_output_tokens_per_task": 9568.11, "last_run_timestamp": "2026-08-29T16:17:15.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google", "accuracy": 0.6000001666666667, "accuracy_std_dev": 0.03771239112642186, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.19008634000000002, "avg_duration_per_task_ms": 186975.22, "avg_output_tokens_per_task": 11452.75, "last_run_timestamp": "2026-10-01T19:04:55.725Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "model_slug": "moonshotai/kimi-k2.7-code", "model_name": "MoonshotAI: Kimi K2.7 Code", "provider_name": "BaseTen", "accuracy": 0.593333, "accuracy_std_dev": 0.020000000000000018, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04552234, "avg_duration_per_task_ms": 32685.39, "avg_output_tokens_per_task": 3125.98, "last_run_timestamp": "2026-09-26T08:22:32.737Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Z.AI", "accuracy": 0.593333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01896666, "avg_duration_per_task_ms": 306972.98, "avg_output_tokens_per_task": 7436.64, "last_run_timestamp": "2026-10-01T08:02:59.889Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "model_slug": "moonshotai/kimi-k2.5", "model_name": "MoonshotAI: Kimi K2.5", "provider_name": "Amazon Bedrock", "accuracy": 0.5914285714285714, "accuracy_std_dev": 0.03558384433120956, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.0944001457142857, "avg_duration_per_task_ms": 57605.94, "avg_output_tokens_per_task": 3609.52, "last_run_timestamp": "2026-10-31T19:06:25.403Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google", "accuracy": 0.591111, "accuracy_std_dev": 0.006285295405415193, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029060479999999996, "avg_duration_per_task_ms": 77755.87, "avg_output_tokens_per_task": 7763.98, "last_run_timestamp": "2026-10-01T19:05:46.198Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Parasail", "accuracy": 0.59, "accuracy_std_dev": 0.0196263239723252, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.020891519999999997, "avg_duration_per_task_ms": 97200.98666666666, "avg_output_tokens_per_task": 4540.886666666666, "last_run_timestamp": "2026-10-03T16:14:27.398Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "model_slug": "xiaomi/mimo-v2.5", "model_name": "Xiaomi: MiMo-V2.5", "provider_name": "DeepInfra", "accuracy": 0.5850695, "accuracy_std_dev": 0.12673650000000003, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.052565635416666666, "avg_duration_per_task_ms": 523237.0729166667, "avg_output_tokens_per_task": 32134.21875, "last_run_timestamp": "2026-08-29T08:10:31.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "Novita", "accuracy": 0.5839998000000001, "accuracy_std_dev": 0.06031032194707637, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.21038078000000002, "avg_duration_per_task_ms": 2412162.416, "avg_output_tokens_per_task": 47038.516, "last_run_timestamp": "2026-10-17T19:42:10.129Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "ibm-granite/granite-4.2-8b-20260831", "model_slug": "ibm-granite/granite-4.2-8b", "model_name": "IBM: Granite 4.2 8B", "provider_name": "auto-routing", "accuracy": 0.583333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.015297799999999999, "avg_duration_per_task_ms": 833284.24, "avg_output_tokens_per_task": 38596.74, "last_run_timestamp": "2026-10-01T08:04:13.086Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "auto-routing", "accuracy": 0.58, "accuracy_std_dev": 0.026666999999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.20638665, "avg_duration_per_task_ms": 179102.02, "avg_output_tokens_per_task": 12109.41, "last_run_timestamp": "2026-10-01T19:04:42.461Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "model_slug": "qwen/qwen3-235b-a22b-thinking-2507", "model_name": "Qwen: Qwen3 235B A22B Thinking 2507", "provider_name": "Novita", "accuracy": 0.58, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.14568854, "avg_duration_per_task_ms": 545436.22, "avg_output_tokens_per_task": 23072.82, "last_run_timestamp": "2026-08-22T16:20:42.106Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "auto-routing", "accuracy": 0.5666665000000001, "accuracy_std_dev": 0.006666499999999964, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.028862480000000003, "avg_duration_per_task_ms": 75034.33, "avg_output_tokens_per_task": 7701.06, "last_run_timestamp": "2026-10-01T19:04:40.340Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Alibaba", "accuracy": 0.5567129305555556, "accuracy_std_dev": 0.0359478068964807, "run_count": 3, "total_questions": 144, "avg_cost_per_task_usd": 0.06927373611111111, "avg_duration_per_task_ms": 83137.22916666667, "avg_output_tokens_per_task": 4080.9305555555557, "last_run_timestamp": "2026-10-03T16:05:18.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Cerebras", "accuracy": 0.556667, "accuracy_std_dev": 0.04999999999999999, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.16220164, "avg_duration_per_task_ms": 20560.07, "avg_output_tokens_per_task": 2975.77, "last_run_timestamp": "2026-08-29T17:07:32.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "model_slug": "google/gemini-2.5-pro", "model_name": "Google: Gemini 2.5 Pro", "provider_name": "Google AI Studio", "accuracy": 0.55, "accuracy_std_dev": 0.036667000000000005, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.2373302, "avg_duration_per_task_ms": 164429.2, "avg_output_tokens_per_task": 12508.34, "last_run_timestamp": "2026-10-01T19:04:52.649Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "model_slug": "google/gemini-2.5-flash", "model_name": "Google: Gemini 2.5 Flash", "provider_name": "Google AI Studio", "accuracy": 0.546667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03543453, "avg_duration_per_task_ms": 61133.82, "avg_output_tokens_per_task": 6656.99, "last_run_timestamp": "2026-10-01T19:04:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Novita", "accuracy": 0.5401338361204013, "accuracy_std_dev": 0.0534017755585482, "run_count": 6, "total_questions": 299, "avg_cost_per_task_usd": 0.010190789297658862, "avg_duration_per_task_ms": 167765.66220735785, "avg_output_tokens_per_task": 16034.110367892976, "last_run_timestamp": "2026-10-31T19:08:29.434Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google AI Studio", "accuracy": 0.5370371414141414, "accuracy_std_dev": 0.020577999999999985, "run_count": 2, "total_questions": 99, "avg_cost_per_task_usd": 0.02047550505050505, "avg_duration_per_task_ms": 144559.1616161616, "avg_output_tokens_per_task": 38541.121212121216, "last_run_timestamp": "2026-10-01T19:05:00.102Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "auto-routing", "accuracy": 0.5366089075144508, "accuracy_std_dev": 0.018572598267559737, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.015467890173410406, "avg_duration_per_task_ms": 1042194.598265896, "avg_output_tokens_per_task": 62203.8063583815, "last_run_timestamp": "2026-10-31T19:53:39.071Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "auto-routing", "accuracy": 0.534014, "accuracy_std_dev": null, "run_count": 1, "total_questions": 49, "avg_cost_per_task_usd": 0.01161230612244898, "avg_duration_per_task_ms": 94666.5306122449, "avg_output_tokens_per_task": 16582.326530612245, "last_run_timestamp": "2026-10-01T19:04:42.466Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "SiliconFlow", "accuracy": 0.5333334285714285, "accuracy_std_dev": 0.019518099050128128, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.06718807142857143, "avg_duration_per_task_ms": 288835.63714285713, "avg_output_tokens_per_task": 4054.082857142857, "last_run_timestamp": "2026-10-31T19:28:47.220Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Darkbloom", "accuracy": 0.5246082751677852, "accuracy_std_dev": 0.030890869335792208, "run_count": 9, "total_questions": 447, "avg_cost_per_task_usd": 0.004759183445190157, "avg_duration_per_task_ms": 649707.9328859061, "avg_output_tokens_per_task": 27601.37360178971, "last_run_timestamp": "2026-11-13T03:45:48.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "Azure", "accuracy": 0.522831205479452, "accuracy_std_dev": 0.027543964303555765, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.11947870547945207, "avg_duration_per_task_ms": 69921.19178082192, "avg_output_tokens_per_task": 2362.404109589041, "last_run_timestamp": "2026-10-01T19:05:18.003Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "model_slug": "google/gemma-4-26b-a4b-it", "model_name": "Google: Gemma 4 26B A4B ", "provider_name": "Google", "accuracy": 0.5211111666666667, "accuracy_std_dev": 0.05297332778677922, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029030346666666665, "avg_duration_per_task_ms": 408571.6, "avg_output_tokens_per_task": 15151.166666666666, "last_run_timestamp": "2026-10-24T19:54:36.551Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "auto-routing", "accuracy": 0.517778, "accuracy_std_dev": 0.027125670043951618, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.02834386, "avg_duration_per_task_ms": 441858.92333333334, "avg_output_tokens_per_task": 58278.28333333333, "last_run_timestamp": "2026-10-03T16:09:19.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "OpenAI", "accuracy": 0.5133333333333333, "accuracy_std_dev": 0.028803034820811636, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.12443134666666665, "avg_duration_per_task_ms": 52357.68, "avg_output_tokens_per_task": 2755.9866666666667, "last_run_timestamp": "2026-10-02T23:07:04.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6-20251208", "model_slug": "z-ai/glm-4.6v", "model_name": "Z.ai: GLM 4.6V", "provider_name": "Novita", "accuracy": 0.513333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03238348, "avg_duration_per_task_ms": 271942.88, "avg_output_tokens_per_task": 7084.96, "last_run_timestamp": "2026-10-01T08:05:50.069Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "model_slug": "xiaomi/mimo-v2.5-pro", "model_name": "Xiaomi: MiMo-V2.5-Pro", "provider_name": "AtlasCloud", "accuracy": 0.5019102607449856, "accuracy_std_dev": 0.2614901575142291, "run_count": 7, "total_questions": 349, "avg_cost_per_task_usd": 0.013760968481375356, "avg_duration_per_task_ms": 262073.68481375359, "avg_output_tokens_per_task": 6586.329512893983, "last_run_timestamp": "2026-10-17T11:22:12.284Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "auto-routing", "accuracy": 0.4953334, "accuracy_std_dev": 0.03792387345512057, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.17013926399999998, "avg_duration_per_task_ms": 2120647.312, "avg_output_tokens_per_task": 40036.992, "last_run_timestamp": "2026-10-17T19:35:49.791Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.492, "accuracy_std_dev": 0.02578540500360622, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.026663912, "avg_duration_per_task_ms": 680695.888, "avg_output_tokens_per_task": 52711.976, "last_run_timestamp": "2026-09-26T16:00:25.689Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "DeepInfra", "accuracy": 0.4885847808219178, "accuracy_std_dev": 0.019611823395311527, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.023597294520547946, "avg_duration_per_task_ms": 1384055.6575342466, "avg_output_tokens_per_task": 116134.45205479451, "last_run_timestamp": "2026-10-31T19:48:14.501Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Nebius", "accuracy": 0.48850570114942526, "accuracy_std_dev": 0.017439968302957548, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.031982068965517245, "avg_duration_per_task_ms": 642915.867816092, "avg_output_tokens_per_task": 59401.63505747126, "last_run_timestamp": "2026-10-23T09:26:25.528Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "auto-routing", "accuracy": 0.48333349999999997, "accuracy_std_dev": 0.03666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.04059527, "avg_duration_per_task_ms": 669013.45, "avg_output_tokens_per_task": 93900.79, "last_run_timestamp": "2026-10-01T19:04:24.509Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-3-12b-it", "model_slug": "google/gemma-3-12b-it", "model_name": "Google: Gemma 3 12B", "provider_name": "DeepInfra", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011001320000000002, "avg_duration_per_task_ms": 80205.64, "avg_output_tokens_per_task": 3762.88, "last_run_timestamp": "2026-10-08T22:08:03.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Google", "accuracy": 0.48, "accuracy_std_dev": 0.0282842712474619, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.0346484, "avg_duration_per_task_ms": 39535.24, "avg_output_tokens_per_task": 3626.5266666666666, "last_run_timestamp": "2026-10-03T16:17:11.217Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.48, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.07899766, "avg_duration_per_task_ms": 132704.5, "avg_output_tokens_per_task": 3161.38, "last_run_timestamp": "2026-10-04T03:00:16.401Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Nebius", "accuracy": 0.476667, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.023033520000000002, "avg_duration_per_task_ms": 894803.72, "avg_output_tokens_per_task": 22315.68, "last_run_timestamp": "2026-08-29T16:13:54.382Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "auto-routing", "accuracy": 0.47666650000000005, "accuracy_std_dev": 0.0166665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.12559784000000002, "avg_duration_per_task_ms": 52928.96, "avg_output_tokens_per_task": 2668.55, "last_run_timestamp": "2026-10-01T19:04:26.703Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash-lite", "model_slug": "google/gemini-2.5-flash-lite", "model_name": "Google: Gemini 2.5 Flash Lite", "provider_name": "Google", "accuracy": 0.4754382631578947, "accuracy_std_dev": 0.054999999999999966, "run_count": 2, "total_questions": 95, "avg_cost_per_task_usd": 0.01757382105263158, "avg_duration_per_task_ms": 167109.45263157896, "avg_output_tokens_per_task": 28512.663157894738, "last_run_timestamp": "2026-10-01T19:06:10.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-nano-30b-a3b", "model_slug": "nvidia/nemotron-3-nano-30b-a3b", "model_name": "NVIDIA: Nemotron 3 Nano 30B A3B", "provider_name": "Crusoe", "accuracy": 0.475, "accuracy_std_dev": 0.01960262643116988, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.029653983333333335, "avg_duration_per_task_ms": 359829.5333333333, "avg_output_tokens_per_task": 63168.53, "last_run_timestamp": "2026-10-03T16:14:03.053Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "model_slug": "deepseek/deepseek-r1-0528", "model_name": "DeepSeek: R1 0528", "provider_name": "SiliconFlow", "accuracy": 0.47499975000000005, "accuracy_std_dev": 0.002886895683515417, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.171962235, "avg_duration_per_task_ms": 2391474.385, "avg_output_tokens_per_task": 45557.86, "last_run_timestamp": "2026-10-17T19:38:30.603Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "DeepInfra", "accuracy": 0.47166650000000004, "accuracy_std_dev": 0.011666499999999996, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02012662, "avg_duration_per_task_ms": 866874.6, "avg_output_tokens_per_task": 18771.04, "last_run_timestamp": "2026-10-09T00:21:42.483Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "model_slug": "qwen/qwen3-30b-a3b", "model_name": "Qwen: Qwen3 30B A3B", "provider_name": "DeepInfra", "accuracy": 0.47, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03551402, "avg_duration_per_task_ms": 437816.58, "avg_output_tokens_per_task": 20922.04, "last_run_timestamp": "2026-10-08T23:09:09.727Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "auto-routing", "accuracy": 0.47, "accuracy_std_dev": 0.009999999999999981, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02032128, "avg_duration_per_task_ms": 826135.39, "avg_output_tokens_per_task": 18360.23, "last_run_timestamp": "2026-08-29T16:16:50.456Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "SiliconFlow", "accuracy": 0.466667, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06821334999999999, "avg_duration_per_task_ms": 285666.19, "avg_output_tokens_per_task": 4113.59, "last_run_timestamp": "2026-08-29T16:26:59.974Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "OpenAI", "accuracy": 0.4644446666666666, "accuracy_std_dev": 0.029979285029648195, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04325750666666667, "avg_duration_per_task_ms": 738509.7133333334, "avg_output_tokens_per_task": 100007.46, "last_run_timestamp": "2026-10-02T23:22:06.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "auto-routing", "accuracy": 0.4622223333333333, "accuracy_std_dev": 0.01911642045177101, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.03776963333333334, "avg_duration_per_task_ms": 60587.16, "avg_output_tokens_per_task": 2732.9733333333334, "last_run_timestamp": "2026-10-03T16:07:50.332Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.4622220000000001, "accuracy_std_dev": 0.02266219999617573, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023695320000000002, "avg_duration_per_task_ms": 107307.77333333333, "avg_output_tokens_per_task": 3751.38, "last_run_timestamp": "2026-10-03T16:13:56.619Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "model_slug": "openai/gpt-5-nano", "model_name": "OpenAI: GPT-5 Nano", "provider_name": "Azure", "accuracy": 0.46166674999999996, "accuracy_std_dev": 0.04010416645047619, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.04448468, "avg_duration_per_task_ms": 738816.635, "avg_output_tokens_per_task": 95390.315, "last_run_timestamp": "2026-10-01T19:06:01.717Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "auto-routing", "accuracy": 0.46, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.011152899999999999, "avg_duration_per_task_ms": 87357.46, "avg_output_tokens_per_task": 9087.9, "last_run_timestamp": "2026-09-01T13:02:23.335Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "model_slug": "openai/gpt-4.1", "model_name": "OpenAI: GPT-4.1", "provider_name": "FakeProvider", "accuracy": 0.46, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": null, "avg_duration_per_task_ms": 158124.84, "avg_output_tokens_per_task": 4515, "last_run_timestamp": "2026-10-01T19:06:51.244Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Parasail", "accuracy": 0.45720697972972973, "accuracy_std_dev": 0.0338422225896225, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.027552510135135135, "avg_duration_per_task_ms": 2287617.1587837837, "avg_output_tokens_per_task": 135257.9527027027, "last_run_timestamp": "2026-10-24T19:05:04.624Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Together", "accuracy": 0.45666671428571426, "accuracy_std_dev": 0.154292962939907, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.027580119999999996, "avg_duration_per_task_ms": 194202.16571428571, "avg_output_tokens_per_task": 18360.19714285714, "last_run_timestamp": "2026-10-31T19:46:58.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.45333349999999994, "accuracy_std_dev": 0.031001890623799048, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.021677975, "avg_duration_per_task_ms": 234042.77, "avg_output_tokens_per_task": 3500.315, "last_run_timestamp": "2026-10-08T22:47:27.753Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "OpenAI", "accuracy": 0.45111100000000004, "accuracy_std_dev": 0.01571332689152746, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.023389446666666668, "avg_duration_per_task_ms": 54689.44666666666, "avg_output_tokens_per_task": 2398.8933333333334, "last_run_timestamp": "2026-10-02T23:07:02.872Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "auto-routing", "accuracy": 0.45, "accuracy_std_dev": 0.0033330000000000026, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07979226, "avg_duration_per_task_ms": 840715.05, "avg_output_tokens_per_task": 21258.48, "last_run_timestamp": "2026-08-29T16:11:46.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "SiliconFlow", "accuracy": 0.45, "accuracy_std_dev": 0.010000000000000009, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.09892811, "avg_duration_per_task_ms": 969037.56, "avg_output_tokens_per_task": 23498.89, "last_run_timestamp": "2026-08-29T16:10:09.206Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "model_slug": "qwen/qwen3-32b", "model_name": "Qwen: Qwen3 32B", "provider_name": "Tenstorrent", "accuracy": 0.4491300441767068, "accuracy_std_dev": 0.023957999896485522, "run_count": 5, "total_questions": 249, "avg_cost_per_task_usd": 0.02306757831325301, "avg_duration_per_task_ms": 1875501.032128514, "avg_output_tokens_per_task": 14213.674698795181, "last_run_timestamp": "2026-11-07T04:00:08.022Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Groq", "accuracy": 0.4487706885245901, "accuracy_std_dev": 0.024486006562116254, "run_count": 5, "total_questions": 244, "avg_cost_per_task_usd": 0.05463102459016393, "avg_duration_per_task_ms": 111409.63114754099, "avg_output_tokens_per_task": 62778.315573770495, "last_run_timestamp": "2026-10-31T19:03:34.962Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Parasail", "accuracy": 0.4458811867816092, "accuracy_std_dev": 0.045214591965110434, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.015823612068965518, "avg_duration_per_task_ms": 133666.3620689655, "avg_output_tokens_per_task": 3370.8706896551726, "last_run_timestamp": "2026-10-31T18:58:15.016Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Parasail", "accuracy": 0.4455556666666666, "accuracy_std_dev": 0.03359162888114966, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.019044513333333336, "avg_duration_per_task_ms": 90751.12, "avg_output_tokens_per_task": 3574.44, "last_run_timestamp": "2026-10-03T16:12:20.062Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "auto-routing", "accuracy": 0.44333300000000003, "accuracy_std_dev": 0.009999999999999981, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.023192820000000003, "avg_duration_per_task_ms": 58714.73, "avg_output_tokens_per_task": 2330.08, "last_run_timestamp": "2026-10-01T19:04:38.295Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.44222233333333333, "accuracy_std_dev": 0.03281073296279061, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.026376873333333332, "avg_duration_per_task_ms": 59340.306666666664, "avg_output_tokens_per_task": 3752.213333333333, "last_run_timestamp": "2026-10-03T16:10:49.378Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "auto-routing", "accuracy": 0.44, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.03130282, "avg_duration_per_task_ms": 680376.23, "avg_output_tokens_per_task": 16688.33, "last_run_timestamp": "2026-10-01T16:00:49.878Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Google", "accuracy": 0.43949773972602735, "accuracy_std_dev": 0.06335403547437912, "run_count": 3, "total_questions": 146, "avg_cost_per_task_usd": 0.053683828767123284, "avg_duration_per_task_ms": 81785.06849315068, "avg_output_tokens_per_task": 3261.6164383561645, "last_run_timestamp": "2026-10-03T16:07:05.221Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Parasail", "accuracy": 0.43555550000000004, "accuracy_std_dev": 0.03136793204186934, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.00797013, "avg_duration_per_task_ms": 203443.01, "avg_output_tokens_per_task": 4634.386666666666, "last_run_timestamp": "2026-10-31T19:46:05.351Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.434, "accuracy_std_dev": 0.03129784515266187, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.053281564000000003, "avg_duration_per_task_ms": 160868.104, "avg_output_tokens_per_task": 3661.612, "last_run_timestamp": "2026-10-17T19:11:29.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Venice", "accuracy": 0.43277766666666667, "accuracy_std_dev": 0.021638103680518977, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.03916930333333333, "avg_duration_per_task_ms": 287824.24666666664, "avg_output_tokens_per_task": 3498.32, "last_run_timestamp": "2026-10-31T19:56:10.591Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "OpenAI", "accuracy": 0.4322221666666667, "accuracy_std_dev": 0.01651776073823432, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.21270556333333332, "avg_duration_per_task_ms": 46305.06, "avg_output_tokens_per_task": 2266.36, "last_run_timestamp": "2026-10-02T23:10:05.271Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "OpenAI", "accuracy": 0.431111, "accuracy_std_dev": 0.00314238253559302, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.24174756666666666, "avg_duration_per_task_ms": 46576.24, "avg_output_tokens_per_task": 2473.9733333333334, "last_run_timestamp": "2026-10-02T23:09:02.629Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "DeepInfra", "accuracy": 0.4301802162162162, "accuracy_std_dev": 0.02412263108922131, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.06743897297297298, "avg_duration_per_task_ms": 360892.5135135135, "avg_output_tokens_per_task": 3318.9054054054054, "last_run_timestamp": "2026-10-08T22:54:33.845Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "auto-routing", "accuracy": 0.43, "accuracy_std_dev": 0.17237997649475806, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07492743333333333, "avg_duration_per_task_ms": 588774.0866666667, "avg_output_tokens_per_task": 9586.053333333333, "last_run_timestamp": "2026-10-31T19:59:00.038Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Novita", "accuracy": 0.42888899999999996, "accuracy_std_dev": 0.01498961780700229, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.025741633333333337, "avg_duration_per_task_ms": 239678.91333333333, "avg_output_tokens_per_task": 3503.96, "last_run_timestamp": "2026-10-03T16:13:22.758Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-next-80b-a3b-instruct-2509", "model_slug": "qwen/qwen3-next-80b-a3b-instruct", "model_name": "Qwen: Qwen3 Next 80B A3B Instruct", "provider_name": "Novita", "accuracy": 0.4288886666666667, "accuracy_std_dev": 0.011331263428624758, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.04006192, "avg_duration_per_task_ms": 171579.76, "avg_output_tokens_per_task": 3385.233333333333, "last_run_timestamp": "2026-10-03T16:18:20.377Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Chutes", "accuracy": 0.42822731213872833, "accuracy_std_dev": 0.02920419045134392, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.0052024595375722545, "avg_duration_per_task_ms": 337191.0375722543, "avg_output_tokens_per_task": 4868.939306358381, "last_run_timestamp": "2026-10-31T19:55:29.787Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "model_slug": "openai/gpt-4o-2024-08-06", "model_name": "OpenAI: GPT-4o (2024-08-06)", "provider_name": "auto-routing", "accuracy": 0.42666650000000006, "accuracy_std_dev": 0.00666650000000002, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.21223042999999997, "avg_duration_per_task_ms": 51685.1, "avg_output_tokens_per_task": 2189.48, "last_run_timestamp": "2026-10-01T16:00:49.880Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "DeepInfra", "accuracy": 0.42555566666666667, "accuracy_std_dev": 0.02910172028515764, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.027489039999999996, "avg_duration_per_task_ms": 68221.56666666667, "avg_output_tokens_per_task": 2795.5733333333333, "last_run_timestamp": "2026-10-03T16:12:20.034Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "model_slug": "qwen/qwen3-14b", "model_name": "Qwen: Qwen3 14B", "provider_name": "DeepInfra", "accuracy": 0.4255553333333334, "accuracy_std_dev": 0.034462310778910675, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.029985173333333334, "avg_duration_per_task_ms": 822675.42, "avg_output_tokens_per_task": 18074.12, "last_run_timestamp": "2026-10-08T22:41:10.664Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "DeepInfra", "accuracy": 0.423750125, "accuracy_std_dev": 0.03212117519424492, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.023984480000000002, "avg_duration_per_task_ms": 452779.21, "avg_output_tokens_per_task": 3544.9625, "last_run_timestamp": "2026-10-31T19:22:37.207Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "auto-routing", "accuracy": 0.4233335, "accuracy_std_dev": 0.0166665, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07010642, "avg_duration_per_task_ms": 253218.68, "avg_output_tokens_per_task": 2643.24, "last_run_timestamp": "2026-08-29T16:33:29.885Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "model_slug": "google/gemma-4-31b-it", "model_name": "Google: Gemma 4 31B", "provider_name": "Modal", "accuracy": 0.423333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.00707156, "avg_duration_per_task_ms": 19040.86, "avg_output_tokens_per_task": 3450.68, "last_run_timestamp": "2026-11-04T09:13:34.415Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "model_slug": "mistralai/mistral-small-2603", "model_name": "Mistral: Mistral Small 4", "provider_name": "Mistral", "accuracy": 0.41888899999999996, "accuracy_std_dev": 0.014614444065147779, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.01463076, "avg_duration_per_task_ms": 76549.69333333333, "avg_output_tokens_per_task": 8814.213333333333, "last_run_timestamp": "2026-09-17T02:10:06.482Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Google", "accuracy": 0.4188886666666667, "accuracy_std_dev": 0.03424661828885039, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.052051466666666664, "avg_duration_per_task_ms": 38040.333333333336, "avg_output_tokens_per_task": 2546.826666666667, "last_run_timestamp": "2026-10-03T16:12:29.903Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "Parasail", "accuracy": 0.4183672857142857, "accuracy_std_dev": 0.0459725, "run_count": 2, "total_questions": 98, "avg_cost_per_task_usd": 0.06366735714285714, "avg_duration_per_task_ms": 88350.82653061225, "avg_output_tokens_per_task": 2676.5102040816328, "last_run_timestamp": "2026-08-29T16:52:55.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Venice", "accuracy": 0.4177776666666667, "accuracy_std_dev": 0.03325923616414275, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07143351333333332, "avg_duration_per_task_ms": 333242.26, "avg_output_tokens_per_task": 14143.333333333334, "last_run_timestamp": "2026-10-03T16:12:29.905Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o", "model_slug": "openai/gpt-4o", "model_name": "OpenAI: GPT-4o", "provider_name": "auto-routing", "accuracy": 0.416667, "accuracy_std_dev": 0.03, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.19647352999999998, "avg_duration_per_task_ms": 53166.02, "avg_output_tokens_per_task": 2086.91, "last_run_timestamp": "2026-10-01T16:01:13.623Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Venice", "accuracy": 0.41555566666666666, "accuracy_std_dev": 0.015713798296048267, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.01727216, "avg_duration_per_task_ms": 121607.27333333333, "avg_output_tokens_per_task": 2838.9866666666667, "last_run_timestamp": "2026-10-03T16:14:29.926Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.415, "accuracy_std_dev": 0.02763845951749121, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.051504745, "avg_duration_per_task_ms": 126446.71, "avg_output_tokens_per_task": 3278.695, "last_run_timestamp": "2026-10-03T16:19:54.632Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "model_slug": "meta-llama/llama-4-maverick", "model_name": "Meta: Llama 4 Maverick", "provider_name": "DigitalOcean", "accuracy": 0.4144140810810811, "accuracy_std_dev": 0.03472111109333275, "run_count": 3, "total_questions": 148, "avg_cost_per_task_usd": 0.07118006081081081, "avg_duration_per_task_ms": 303449.4054054054, "avg_output_tokens_per_task": 2651.4324324324325, "last_run_timestamp": "2026-10-05T20:09:54.588Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "StreamLake", "accuracy": 0.4136123821989529, "accuracy_std_dev": 0.024976865859981302, "run_count": 4, "total_questions": 191, "avg_cost_per_task_usd": 0.009809837696335078, "avg_duration_per_task_ms": 75939.89528795812, "avg_output_tokens_per_task": 2670.895287958115, "last_run_timestamp": "2026-10-03T16:17:21.424Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Novita", "accuracy": 0.40888899999999995, "accuracy_std_dev": 0.08368091364622321, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.2754822733333333, "avg_duration_per_task_ms": 135119.14, "avg_output_tokens_per_task": 4063.2466666666664, "last_run_timestamp": "2026-10-03T16:19:14.095Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "model_slug": "qwen/qwen3-coder", "model_name": "Qwen: Qwen3 Coder 480B A35B", "provider_name": "Alibaba", "accuracy": 0.40888866666666673, "accuracy_std_dev": 0.0599587430173634, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.6742300533333333, "avg_duration_per_task_ms": 128242.34, "avg_output_tokens_per_task": 3782.766666666667, "last_run_timestamp": "2026-10-03T16:14:53.189Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Novita", "accuracy": 0.40888866666666673, "accuracy_std_dev": 0.03189478845969806, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.07531134, "avg_duration_per_task_ms": 109862.38666666667, "avg_output_tokens_per_task": 3035.5333333333333, "last_run_timestamp": "2026-10-03T16:12:07.495Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Novita", "accuracy": 0.40878389864864867, "accuracy_std_dev": 0.025276434244942244, "run_count": 6, "total_questions": 296, "avg_cost_per_task_usd": 0.06894762837837838, "avg_duration_per_task_ms": 264205.23986486485, "avg_output_tokens_per_task": 5484.4628378378375, "last_run_timestamp": "2026-10-03T08:12:06.170Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Io Net", "accuracy": 0.4069373604060914, "accuracy_std_dev": 0.01904818344849452, "run_count": 4, "total_questions": 197, "avg_cost_per_task_usd": 0.059585842639593906, "avg_duration_per_task_ms": 733094.8121827411, "avg_output_tokens_per_task": 26202.4923857868, "last_run_timestamp": "2026-09-19T16:00:30.864Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.406667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.06754938, "avg_duration_per_task_ms": 114529.78, "avg_output_tokens_per_task": 3789.7, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Alibaba", "accuracy": 0.406667, "accuracy_std_dev": 0.016329931618554536, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06185437333333333, "avg_duration_per_task_ms": 96669.76666666666, "avg_output_tokens_per_task": 2924.06, "last_run_timestamp": "2026-10-03T16:12:33.500Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "SiliconFlow", "accuracy": 0.4066666666666667, "accuracy_std_dev": 0.02494438257849294, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.028486626666666664, "avg_duration_per_task_ms": 245942.52666666667, "avg_output_tokens_per_task": 4212.966666666666, "last_run_timestamp": "2026-10-03T16:13:56.972Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "Parasail", "accuracy": 0.40444433333333335, "accuracy_std_dev": 0.008314853589945066, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.034242013333333335, "avg_duration_per_task_ms": 165678.99333333335, "avg_output_tokens_per_task": 3408.8533333333335, "last_run_timestamp": "2026-10-03T16:08:50.061Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "StreamLake", "accuracy": 0.40380291275167784, "accuracy_std_dev": 0.008840018715415086, "run_count": 3, "total_questions": 149, "avg_cost_per_task_usd": 0.07104265771812081, "avg_duration_per_task_ms": 235544.13422818793, "avg_output_tokens_per_task": 8668.979865771813, "last_run_timestamp": "2026-10-02T15:44:48.231Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "model_slug": "openai/gpt-4.1-mini", "model_name": "OpenAI: GPT-4.1 Mini", "provider_name": "Azure", "accuracy": 0.40333325000000003, "accuracy_std_dev": 0.01914841159698371, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.023807599999999998, "avg_duration_per_task_ms": 77145.375, "avg_output_tokens_per_task": 2775.135, "last_run_timestamp": "2026-10-01T19:05:00.133Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "AkashML", "accuracy": 0.401667, "accuracy_std_dev": 0.025000000000000022, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01796061, "avg_duration_per_task_ms": 1329879.6, "avg_output_tokens_per_task": 125682.59, "last_run_timestamp": "2026-10-31T20:05:39.534Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Nebius", "accuracy": 0.4, "accuracy_std_dev": 0, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.016500189999999998, "avg_duration_per_task_ms": 114969.84, "avg_output_tokens_per_task": 649.31, "last_run_timestamp": "2026-08-29T17:06:47.617Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "auto-routing", "accuracy": 0.4, "accuracy_std_dev": 0.014401594564491808, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.08217413333333333, "avg_duration_per_task_ms": 498824.76, "avg_output_tokens_per_task": 14205.28, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "model_slug": "deepseek/deepseek-chat", "model_name": "DeepSeek: DeepSeek V3", "provider_name": "auto-routing", "accuracy": 0.39833349999999995, "accuracy_std_dev": 0.018333500000000003, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.06903712000000001, "avg_duration_per_task_ms": 315593.59, "avg_output_tokens_per_task": 7308.28, "last_run_timestamp": "2026-10-01T19:04:42.463Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "model_slug": "deepseek/deepseek-chat-v3-0324", "model_name": "DeepSeek: DeepSeek V3 0324", "provider_name": "DeepInfra", "accuracy": 0.393333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.03258216, "avg_duration_per_task_ms": 687273.92, "avg_output_tokens_per_task": 4792.38, "last_run_timestamp": "2026-08-22T16:13:43.431Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-235b-a22b-instruct", "model_slug": "qwen/qwen3-vl-235b-a22b-instruct", "model_name": "Qwen: Qwen3 VL 235B A22B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3872055252525253, "accuracy_std_dev": 0.01879637043919651, "run_count": 4, "total_questions": 198, "avg_cost_per_task_usd": 0.05699022222222223, "avg_duration_per_task_ms": 384913.5606060606, "avg_output_tokens_per_task": 3819.5858585858587, "last_run_timestamp": "2026-10-08T23:21:09.489Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "AtlasCloud", "accuracy": 0.386667, "accuracy_std_dev": 0.06, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.052157499999999996, "avg_duration_per_task_ms": 111017.7, "avg_output_tokens_per_task": 2942.27, "last_run_timestamp": "2026-08-29T16:12:10.467Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Ambient", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.014510460000000001, "avg_duration_per_task_ms": 64332.54, "avg_output_tokens_per_task": 4106.96, "last_run_timestamp": "2026-10-09T15:37:38.473Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.386667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0592903, "avg_duration_per_task_ms": 203536.16, "avg_output_tokens_per_task": 5170.96, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.3844443333333334, "accuracy_std_dev": 0.008314853589945041, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.02693198, "avg_duration_per_task_ms": 219333.54666666666, "avg_output_tokens_per_task": 4471.74, "last_run_timestamp": "2026-10-03T16:03:16.197Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Novita", "accuracy": 0.38148099999999996, "accuracy_std_dev": null, "run_count": 1, "total_questions": 45, "avg_cost_per_task_usd": 0.008997444444444445, "avg_duration_per_task_ms": 232957.2888888889, "avg_output_tokens_per_task": 3561.511111111111, "last_run_timestamp": "2026-10-03T16:15:51.302Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "CoreWeave", "accuracy": 0.38, "accuracy_std_dev": 0.0066669999999999785, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.02802411, "avg_duration_per_task_ms": 79737.77, "avg_output_tokens_per_task": 4093.08, "last_run_timestamp": "2026-08-29T16:13:12.633Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "auto-routing", "accuracy": 0.3780952857142857, "accuracy_std_dev": 0.13062556690864188, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.019071360000000002, "avg_duration_per_task_ms": 234782.5542857143, "avg_output_tokens_per_task": 3254.2371428571428, "last_run_timestamp": "2026-10-31T19:18:14.606Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "AkashML", "accuracy": 0.3742767863818424, "accuracy_std_dev": 0.02350411518067989, "run_count": 15, "total_questions": 749, "avg_cost_per_task_usd": 0.013144802403204274, "avg_duration_per_task_ms": 41061.73831775701, "avg_output_tokens_per_task": 781.1562082777036, "last_run_timestamp": "2026-10-31T19:10:53.562Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "DeepInfra", "accuracy": 0.3694728520408163, "accuracy_std_dev": 0.1273174900715113, "run_count": 8, "total_questions": 392, "avg_cost_per_task_usd": 0.05665731377551021, "avg_duration_per_task_ms": 831256.9948979592, "avg_output_tokens_per_task": 9740.272959183674, "last_run_timestamp": "2026-10-31T19:55:08.491Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "GMICloud", "accuracy": 0.36833325, "accuracy_std_dev": 0.17853565236861657, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.015193219999999999, "avg_duration_per_task_ms": 127682.09, "avg_output_tokens_per_task": 2917.505, "last_run_timestamp": "2026-10-03T16:12:29.879Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Nebius", "accuracy": 0.36666650000000006, "accuracy_std_dev": 0.021602468996621663, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.033141745, "avg_duration_per_task_ms": 134753.71, "avg_output_tokens_per_task": 4403.68, "last_run_timestamp": "2026-10-23T08:14:36.871Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "PrimeIntellect", "accuracy": 0.36, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.041000339999999996, "avg_duration_per_task_ms": 322345.3, "avg_output_tokens_per_task": 4212.74, "last_run_timestamp": "2026-10-04T03:00:10.237Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3568007183908046, "accuracy_std_dev": 0.033975072562581136, "run_count": 7, "total_questions": 348, "avg_cost_per_task_usd": 0.06755664942528736, "avg_duration_per_task_ms": 409777.1005747126, "avg_output_tokens_per_task": 7849.508620689655, "last_run_timestamp": "2026-10-08T23:34:52.306Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "auto-routing", "accuracy": 0.35570486577181204, "accuracy_std_dev": 0.024452362985677545, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.07792019798657716, "avg_duration_per_task_ms": 210172.6476510067, "avg_output_tokens_per_task": 7299.98322147651, "last_run_timestamp": "2026-10-03T08:12:20.019Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "model_slug": "qwen/qwen3-235b-a22b-2507", "model_name": "Qwen: Qwen3 235B A22B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.35333333333333333, "accuracy_std_dev": 0.04455559988997518, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.06266134666666666, "avg_duration_per_task_ms": 100248.80666666667, "avg_output_tokens_per_task": 3931.2533333333336, "last_run_timestamp": "2026-10-03T16:05:51.296Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "model_slug": "qwen/qwen3-30b-a3b-instruct-2507", "model_name": "Qwen: Qwen3 30B A3B Instruct 2507", "provider_name": "Alibaba", "accuracy": 0.351111, "accuracy_std_dev": 0.013698950130089059, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.057490099999999995, "avg_duration_per_task_ms": 70349.37333333334, "avg_output_tokens_per_task": 4393.28, "last_run_timestamp": "2026-10-03T16:05:34.166Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "CoreWeave", "accuracy": 0.35047614285714285, "accuracy_std_dev": 0.03596402460566301, "run_count": 7, "total_questions": 350, "avg_cost_per_task_usd": 0.09883995142857144, "avg_duration_per_task_ms": 29505.717142857142, "avg_output_tokens_per_task": 1132.6742857142858, "last_run_timestamp": "2026-10-31T18:48:48.352Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "model_slug": "deepseek/deepseek-chat-v3.1", "model_name": "DeepSeek: DeepSeek V3.1", "provider_name": "CoreWeave", "accuracy": 0.34013619387755106, "accuracy_std_dev": 0.08215978685882466, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.5213178979591837, "avg_duration_per_task_ms": 540508.3367346938, "avg_output_tokens_per_task": 21094.061224489797, "last_run_timestamp": "2026-10-10T18:31:45.008Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "auto-routing", "accuracy": 0.33622356069364157, "accuracy_std_dev": 0.046445988932944285, "run_count": 7, "total_questions": 346, "avg_cost_per_task_usd": 0.057168858381502895, "avg_duration_per_task_ms": 73268.67919075144, "avg_output_tokens_per_task": 1325.586705202312, "last_run_timestamp": "2026-10-31T19:07:35.559Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "PrimeIntellect", "accuracy": 0.333333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.051354699999999996, "avg_duration_per_task_ms": 358714.54, "avg_output_tokens_per_task": 69285.14, "last_run_timestamp": "2026-10-04T03:00:10.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-8b-instruct", "model_slug": "qwen/qwen3-vl-8b-instruct", "model_name": "Qwen: Qwen3 VL 8B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.326667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.02872976, "avg_duration_per_task_ms": 133695.86, "avg_output_tokens_per_task": 3408.56, "last_run_timestamp": "2026-10-04T03:00:16.402Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "Alibaba", "accuracy": 0.3233333333333333, "accuracy_std_dev": 0.027419971095697552, "run_count": 6, "total_questions": 300, "avg_cost_per_task_usd": 0.07416080666666668, "avg_duration_per_task_ms": 130738.53, "avg_output_tokens_per_task": 7429.926666666666, "last_run_timestamp": "2026-10-03T08:23:09.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "PrimeIntellect", "accuracy": 0.313333, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.15186704, "avg_duration_per_task_ms": 32635.52, "avg_output_tokens_per_task": 1294.54, "last_run_timestamp": "2026-10-04T03:00:16.406Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "Amazon Bedrock", "accuracy": 0.31, "accuracy_std_dev": 0.12415428957205972, "run_count": 14, "total_questions": 700, "avg_cost_per_task_usd": 0.037997529999999995, "avg_duration_per_task_ms": 470949.48285714287, "avg_output_tokens_per_task": 124103.05857142857, "last_run_timestamp": "2026-10-31T20:07:06.512Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "auto-routing", "accuracy": 0.30666649999999995, "accuracy_std_dev": 0.006666499999999992, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.01779966, "avg_duration_per_task_ms": 55498.41, "avg_output_tokens_per_task": 3406.81, "last_run_timestamp": "2026-10-01T19:04:58.075Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "DeepInfra", "accuracy": 0.3055554583333333, "accuracy_std_dev": 0.054493000000000014, "run_count": 2, "total_questions": 96, "avg_cost_per_task_usd": 0.024076770833333334, "avg_duration_per_task_ms": 191488.85416666666, "avg_output_tokens_per_task": 2060.5833333333335, "last_run_timestamp": "2026-08-29T16:43:42.449Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "model_slug": "meta-llama/llama-3.3-70b-instruct", "model_name": "Meta: Llama 3.3 70B Instruct", "provider_name": "Together", "accuracy": 0.28533339999999996, "accuracy_std_dev": 0.02872100586400135, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.174218524, "avg_duration_per_task_ms": 145567.084, "avg_output_tokens_per_task": 1889.128, "last_run_timestamp": "2026-10-17T19:19:08.724Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "model_slug": "openai/gpt-4o-mini", "model_name": "OpenAI: GPT-4o-mini", "provider_name": "OpenAI", "accuracy": 0.2844446666666667, "accuracy_std_dev": 0.02997950740459592, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.025635820000000004, "avg_duration_per_task_ms": 67806.75333333333, "avg_output_tokens_per_task": 4107.166666666667, "last_run_timestamp": "2026-10-02T23:13:47.090Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "Novita", "accuracy": 0.2733335, "accuracy_std_dev": 0.013333499999999998, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07945186, "avg_duration_per_task_ms": 632916.72, "avg_output_tokens_per_task": 5086.04, "last_run_timestamp": "2026-10-01T16:00:45.063Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "model_slug": "meta-llama/llama-3.1-8b-instruct", "model_name": "Meta: Llama 3.1 8B Instruct", "provider_name": "auto-routing", "accuracy": 0.25632182758620686, "accuracy_std_dev": 0.05543966898249743, "run_count": 3, "total_questions": 145, "avg_cost_per_task_usd": 0.005220103448275862, "avg_duration_per_task_ms": 18594.572413793103, "avg_output_tokens_per_task": 2050.062068965517, "last_run_timestamp": "2026-10-24T19:30:38.210Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "DeepInfra", "accuracy": 0.25, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.0713485, "avg_duration_per_task_ms": 660516.14, "avg_output_tokens_per_task": 6662.1, "last_run_timestamp": "2026-09-01T12:59:14.290Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "Novita", "accuracy": 0.24666666666666667, "accuracy_std_dev": 0.08018487896660371, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.013351073333333331, "avg_duration_per_task_ms": 21559.86, "avg_output_tokens_per_task": 1099.86, "last_run_timestamp": "2026-10-03T16:13:42.255Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "model_slug": "qwen/qwen-2.5-72b-instruct", "model_name": "Qwen2.5 72B Instruct", "provider_name": "auto-routing", "accuracy": 0.24, "accuracy_std_dev": 0.033333, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.07327475, "avg_duration_per_task_ms": 662655.63, "avg_output_tokens_per_task": 5322.46, "last_run_timestamp": "2026-10-01T16:00:49.856Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "model_slug": "z-ai/glm-4.6", "model_name": "Z.ai: GLM 4.6", "provider_name": "AtlasCloud", "accuracy": 0.2366665, "accuracy_std_dev": 0.1233335, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.10252787000000002, "avg_duration_per_task_ms": 324411.25, "avg_output_tokens_per_task": 8077.93, "last_run_timestamp": "2026-08-29T16:22:37.844Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "CoreWeave", "accuracy": 0.22000033333333335, "accuracy_std_dev": 0.30169889330626026, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.003046386666666667, "avg_duration_per_task_ms": 163087.96666666667, "avg_output_tokens_per_task": 5509.633333333333, "last_run_timestamp": "2026-10-24T19:32:27.020Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "model_slug": "qwen/qwen3-coder-next", "model_name": "Qwen: Qwen3 Coder Next", "provider_name": "PrimeIntellect", "accuracy": 0.186667, "accuracy_std_dev": null, "run_count": 1, "total_questions": 50, "avg_cost_per_task_usd": 0.01693508, "avg_duration_per_task_ms": 26941.4, "avg_output_tokens_per_task": 1041.58, "last_run_timestamp": "2026-10-04T03:00:16.405Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "amazon/nova-micro-v1", "model_slug": "amazon/nova-micro-v1", "model_name": "Amazon: Nova Micro 1.0", "provider_name": "Amazon Bedrock", "accuracy": 0.17942213775510205, "accuracy_std_dev": 0.009739601557558703, "run_count": 4, "total_questions": 196, "avg_cost_per_task_usd": 0.010118459183673469, "avg_duration_per_task_ms": 155344.85714285713, "avg_output_tokens_per_task": 4996.790816326531, "last_run_timestamp": "2026-10-01T19:04:52.650Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-vl-30b-a3b-instruct", "model_slug": "qwen/qwen3-vl-30b-a3b-instruct", "model_name": "Qwen: Qwen3 VL 30B A3B Instruct", "provider_name": "SiliconFlow", "accuracy": 0.1466666, "accuracy_std_dev": 0.1569147341005299, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.26473028, "avg_duration_per_task_ms": 1632124.184, "avg_output_tokens_per_task": 10106.508, "last_run_timestamp": "2026-10-03T08:09:23.857Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "Azure", "accuracy": 0.14, "accuracy_std_dev": 0.014142135623730954, "run_count": 4, "total_questions": 200, "avg_cost_per_task_usd": 0.0072234, "avg_duration_per_task_ms": 76455.195, "avg_output_tokens_per_task": 2322.375, "last_run_timestamp": "2026-10-01T19:04:58.093Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "BaseTen", "accuracy": 0.1366668, "accuracy_std_dev": 0.2566795840906713, "run_count": 5, "total_questions": 250, "avg_cost_per_task_usd": 0.004989168, "avg_duration_per_task_ms": 18430.86, "avg_output_tokens_per_task": 5212.152, "last_run_timestamp": "2026-10-31T19:36:09.009Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "model_slug": "openai/gpt-oss-120b", "model_name": "OpenAI: gpt-oss-120b", "provider_name": "Nebius", "accuracy": 0.132499875, "accuracy_std_dev": 0.20457100455064345, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.04783906, "avg_duration_per_task_ms": 64368.49, "avg_output_tokens_per_task": 12062.3875, "last_run_timestamp": "2026-10-31T19:46:05.356Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "OpenAI", "accuracy": 0.111111, "accuracy_std_dev": 0.01571332689152746, "run_count": 3, "total_questions": 150, "avg_cost_per_task_usd": 0.007342933333333333, "avg_duration_per_task_ms": 57269.24, "avg_output_tokens_per_task": 2382.0866666666666, "last_run_timestamp": "2026-10-02T23:07:59.112Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "model_slug": "openai/gpt-4.1-nano", "model_name": "OpenAI: GPT-4.1 Nano", "provider_name": "auto-routing", "accuracy": 0.11, "accuracy_std_dev": 0.009999999999999995, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.006762440000000001, "avg_duration_per_task_ms": 64306.02, "avg_output_tokens_per_task": 2312.51, "last_run_timestamp": "2026-10-01T19:04:36.194Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "model_slug": "openai/gpt-oss-20b", "model_name": "OpenAI: gpt-oss-20b", "provider_name": "CoreWeave", "accuracy": 0.09172277852348994, "accuracy_std_dev": 0.1783848110580669, "run_count": 6, "total_questions": 298, "avg_cost_per_task_usd": 0.003983040268456375, "avg_duration_per_task_ms": 283449.6208053691, "avg_output_tokens_per_task": 25704.31879194631, "last_run_timestamp": "2026-10-31T19:13:59.240Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "model_slug": "deepseek/deepseek-v3.1-terminus", "model_name": "DeepSeek: DeepSeek V3.1 Terminus", "provider_name": "AtlasCloud", "accuracy": 0.08666650000000001, "accuracy_std_dev": 0.013333500000000005, "run_count": 2, "total_questions": 100, "avg_cost_per_task_usd": 0.00166133, "avg_duration_per_task_ms": 9310, "avg_output_tokens_per_task": 202.06, "last_run_timestamp": "2026-08-29T16:17:08.780Z" }, { "benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "model_slug": "qwen/qwen3.6-35b-a3b", "model_name": "Qwen: Qwen3.6 35B A3B", "provider_name": "Venice", "accuracy": 0.0633335, "accuracy_std_dev": 0.14742210372515377, "run_count": 8, "total_questions": 400, "avg_cost_per_task_usd": 0.001561505, "avg_duration_per_task_ms": 4223.18, "avg_output_tokens_per_task": 528.385, "last_run_timestamp": "2026-10-17T19:48:24.252Z" } ] }