{"data":{"runs":[{"id":"6fec06e9-8277-4196-b55a-799d7566e42c","workflow_id":"rbds1017-qa-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"deepseek/deepseek-v4.1-flash-1017","provider_name":"OpenRouter","timestamp":"2026-10-23T09:28:55.962Z","resumed_from_workflow_id":null,"config_key":"5689e52ebcdf36db8970c4805480ccb8","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.580645,"total_questions":248,"correct_answers":144,"total_cost":0,"generation_time_ms":84922306,"duration_ms":13073962,"input_tokens":1077748369,"output_tokens":7347724,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"3c57ac3a-68ae-4ce3-a6b2-28a9d754b83d","workflow_id":"rbg38f-mmmu-hi","benchmark_type":"mmmu_pro_vision","model_permaslug":"google/gemini-3.8-flash","provider_name":"auto-routing","timestamp":"2026-10-23T08:42:58.312Z","resumed_from_workflow_id":null,"config_key":"dfc67ab7a64132180b624dcd8f13068b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.815029,"total_questions":1730,"correct_answers":1410,"total_cost":40.319382,"generation_time_ms":78276185,"duration_ms":4576312,"input_tokens":2125126,"output_tokens":10326810,"reasoning_tokens":9853919,"success":true,"baseline":true},{"id":"f58b76bd-6865-4cc3-bab9-6fac3dfe46b3","workflow_id":"rbgarg-mmmu-hi","benchmark_type":"mmmu_pro_vision","model_permaslug":"google/gemini-4-argon","provider_name":"auto-routing","timestamp":"2026-10-23T08:34:30.750Z","resumed_from_workflow_id":null,"config_key":"16a8dbe6b5e6041ed3c0cd21bbcd3062","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.869942,"total_questions":1730,"correct_answers":1505,"total_cost":46.46712,"generation_time_ms":62179362,"duration_ms":4072750,"input_tokens":2193245,"output_tokens":4208063,"reasoning_tokens":3655757,"success":true,"baseline":true},{"id":"b4e94a53-7f1f-4e8b-b549-1dead66e0996","workflow_id":"rbgarg-tau3-hi","benchmark_type":"tau3_bench_banking","model_permaslug":"google/gemini-4-argon","provider_name":"auto-routing","timestamp":"2026-10-23T08:17:26.120Z","resumed_from_workflow_id":null,"config_key":"b27cc2fe7ff8748bc377dbf5d394aefa","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.463918,"total_questions":97,"correct_answers":45,"total_cost":91.435361,"generation_time_ms":61377940,"duration_ms":3046120,"input_tokens":325064608,"output_tokens":2543894,"reasoning_tokens":2143769,"success":true,"baseline":true},{"id":"65107e65-8e3a-469a-9499-8879a28811bb","workflow_id":"rbds1017-mmmu-hi","benchmark_type":"mmmu_pro_vision","model_permaslug":"deepseek/deepseek-v4.1-flash-1017","provider_name":"auto-routing","timestamp":"2026-10-23T07:39:30.612Z","resumed_from_workflow_id":null,"config_key":"2ecc3d77d70499c57a86b0f812df8ba2","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.72659,"total_questions":3460,"correct_answers":2514,"total_cost":22.045348,"generation_time_ms":170924779,"duration_ms":6509612,"input_tokens":3415310,"output_tokens":34575216,"reasoning_tokens":34198954,"success":true,"baseline":true},{"id":"0ce76da5-695f-4172-b117-08f70c02a666","workflow_id":"rbgarg-gpqa-hi","benchmark_type":"gpqa_diamond","model_permaslug":"google/gemini-4-argon","provider_name":"auto-routing","timestamp":"2026-10-23T07:38:37.725Z","resumed_from_workflow_id":null,"config_key":"c73bac0b4caf9e39e5a270613948d658","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.959596,"total_questions":198,"correct_answers":190,"total_cost":5.793512,"generation_time_ms":7970059,"duration_ms":720725,"input_tokens":55431,"output_tokens":568265,"reasoning_tokens":445225,"success":true,"baseline":true},{"id":"87fad882-3f09-4f6a-84ce-8b66f7880733","workflow_id":"rbg38f-gpqa-hi","benchmark_type":"gpqa_diamond","model_permaslug":"google/gemini-3.8-flash","provider_name":"auto-routing","timestamp":"2026-10-23T07:37:43.257Z","resumed_from_workflow_id":null,"config_key":"ed803d5d60250345aeaff3d562d1bdec","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.934343,"total_questions":198,"correct_answers":185,"total_cost":4.588658,"generation_time_ms":8787443,"duration_ms":662257,"input_tokens":48396,"output_tokens":1213963,"reasoning_tokens":1119611,"success":true,"baseline":true},{"id":"a4a0bc72-a58d-420e-ae68-6cb29b323902","workflow_id":"rbds1017-tau3-hi","benchmark_type":"tau3_bench_banking","model_permaslug":"deepseek/deepseek-v4.1-flash-1017","provider_name":"auto-routing","timestamp":"2026-10-23T06:39:14.234Z","resumed_from_workflow_id":null,"config_key":"0e54fa8a7c92dd37bf0245dff8cff26d","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.247423,"total_questions":194,"correct_answers":48,"total_cost":6.187869,"generation_time_ms":25576221,"duration_ms":2892234,"input_tokens":155929592,"output_tokens":4158384,"reasoning_tokens":3706583,"success":true,"baseline":true},{"id":"11073fba-eeb3-43c5-9ded-adc817ea3cc5","workflow_id":"rbds1017-gpqa-hi","benchmark_type":"gpqa_diamond","model_permaslug":"deepseek/deepseek-v4.1-flash-1017","provider_name":"auto-routing","timestamp":"2026-10-23T06:09:54.243Z","resumed_from_workflow_id":null,"config_key":"32a2040bcdffe9a136a0c7454436fc7c","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.901515,"total_questions":396,"correct_answers":357,"total_cost":1.306434,"generation_time_ms":14452108,"duration_ms":1134243,"input_tokens":104849,"output_tokens":2301407,"reasoning_tokens":2253829,"success":true,"baseline":true},{"id":"f1ae6c02-4c17-45ef-8487-d012a0463f31","workflow_id":"rbp2610-dswe-10160209","benchmark_type":"deep_swe","model_permaslug":"unbiased/pareto-26.10","provider_name":"OpenRouter","timestamp":"2026-10-16T04:27:58.845Z","resumed_from_workflow_id":null,"config_key":"1b2f717aa7802ea5bce7571dbc5420f1","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.743363,"total_questions":113,"correct_answers":84,"total_cost":21.868471,"generation_time_ms":50632327,"duration_ms":8287845,"input_tokens":111770138,"output_tokens":1995742,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"3c0026a7-2326-4162-9420-0bd88496e09a","workflow_id":"rbp2610-dswe-10152339","benchmark_type":"deep_swe","model_permaslug":"unbiased/pareto-26.10","provider_name":"OpenRouter","timestamp":"2026-10-16T02:00:21.730Z","resumed_from_workflow_id":null,"config_key":"d3984cce521023be918d320d8967c526","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.725664,"total_questions":113,"correct_answers":82,"total_cost":21.716664,"generation_time_ms":50931529,"duration_ms":8445730,"input_tokens":106374919,"output_tokens":1920461,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"c11ccf71-16d1-405f-b6a3-aaa14dc14c57","workflow_id":"rbp2610p-qa-10151409","benchmark_type":"swe_atlas_qa","model_permaslug":"unbiased/pareto-26.10-preview","provider_name":"OpenRouter","timestamp":"2026-10-15T15:57:30.964Z","resumed_from_workflow_id":null,"config_key":"19ae85d36afb99344808d02eca14ecc7","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.604839,"total_questions":124,"correct_answers":75,"total_cost":17.531838,"generation_time_ms":35978373,"duration_ms":6481964,"input_tokens":78202785,"output_tokens":1425715,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"be84bfe9-b65d-4ef6-bb92-3401d9c01fe1","workflow_id":"rbp2610p-mmmu-10151409","benchmark_type":"mmmu_pro_vision","model_permaslug":"unbiased/pareto-26.10-preview","provider_name":"auto-routing","timestamp":"2026-10-15T14:59:05.363Z","resumed_from_workflow_id":null,"config_key":"d793697202561c1533aa216f0c8c0f1d","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.754913,"total_questions":1730,"correct_answers":1306,"total_cost":25.331648,"generation_time_ms":28917626,"duration_ms":2978363,"input_tokens":7333192,"output_tokens":6177446,"reasoning_tokens":0,"success":true,"baseline":false},{"id":"f3622ea7-1aeb-4c58-b3bf-d595e01e7dfd","workflow_id":"rbp2610p-tau3-10151409","benchmark_type":"tau3_bench_banking","model_permaslug":"unbiased/pareto-26.10-preview","provider_name":"auto-routing","timestamp":"2026-10-15T14:26:21.572Z","resumed_from_workflow_id":null,"config_key":"643e797d5479734f71407719df8b9267","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.350515,"total_questions":97,"correct_answers":34,"total_cost":14.861935,"generation_time_ms":10003783,"duration_ms":1013572,"input_tokens":93877806,"output_tokens":332590,"reasoning_tokens":0,"success":true,"baseline":false},{"id":"34d10361-0032-4562-a6ca-42d0fbeb3e50","workflow_id":"rbp2610p-gpqa-10151409","benchmark_type":"gpqa_diamond","model_permaslug":"unbiased/pareto-26.10-preview","provider_name":"auto-routing","timestamp":"2026-10-15T14:24:31.863Z","resumed_from_workflow_id":null,"config_key":"e3f88df471b07d2d95a9d6f61abfe947","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.924242,"total_questions":198,"correct_answers":183,"total_cost":0.608324,"generation_time_ms":7485822,"duration_ms":905863,"input_tokens":48936,"output_tokens":178974,"reasoning_tokens":0,"success":true,"baseline":false},{"id":"e95bfc00-f9da-4a46-b492-d0c7e6448c08","workflow_id":"rb1013-dswe-jevhi","benchmark_type":"deep_swe","model_permaslug":"typesafe/jev-router","provider_name":"OpenRouter","timestamp":"2026-10-13T19:19:49.574Z","resumed_from_workflow_id":null,"config_key":"51b206840993059e0cc12bfc9d00a190","reasoning_effort":"high","max_tokens":null,"cost_tier":"high","route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.699115,"total_questions":113,"correct_answers":79,"total_cost":670.766079,"generation_time_ms":161711127,"duration_ms":20739574,"input_tokens":258763019,"output_tokens":5853080,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"cf08e19d-840d-4983-8e1a-dd13576fc84b","workflow_id":"rb1013-tau3-jevhi-r4","benchmark_type":"tau3_bench_banking","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-10-13T18:29:03.190Z","resumed_from_workflow_id":null,"config_key":"64319e9d76f22824bb9319b94f4c08f3","reasoning_effort":"high","max_tokens":null,"cost_tier":"high","route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.35567,"total_questions":194,"correct_answers":69,"total_cost":164.095526,"generation_time_ms":19992466,"duration_ms":2146190,"input_tokens":199746454,"output_tokens":1013623,"reasoning_tokens":417126,"success":true,"baseline":false},{"id":"df6154aa-8535-46e9-9261-735ad7dcb617","workflow_id":"rb1013-mmmu-jevhi-r2","benchmark_type":"mmmu_pro_vision","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-10-13T18:25:59.482Z","resumed_from_workflow_id":null,"config_key":"3d8d7a09c6300a2d24b3eed2d3f0dc3b","reasoning_effort":"high","max_tokens":null,"cost_tier":"high","route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.813295,"total_questions":1730,"correct_answers":1407,"total_cost":40.532437,"generation_time_ms":24030228,"duration_ms":3680482,"input_tokens":5938092,"output_tokens":1594454,"reasoning_tokens":1377970,"success":true,"baseline":false},{"id":"154fbc92-04e2-462f-9add-c7b5dababad8","workflow_id":"t3p1010-gem38f-high-r2","benchmark_type":"tau3_bench_banking","model_permaslug":"google/gemini-3.8-flash","provider_name":"auto-routing","timestamp":"2026-10-10T05:22:24.731Z","resumed_from_workflow_id":null,"config_key":"81a4e3b6517b51c0e741f82319b47cda","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.391753,"total_questions":192,"correct_answers":76,"total_cost":96.325353,"generation_time_ms":48742588,"duration_ms":1975731,"input_tokens":541378046,"output_tokens":4666828,"reasoning_tokens":4063502,"success":true,"baseline":true},{"id":"dbf391ea-c4ab-467a-8786-f20e4bf45379","workflow_id":"rb1009-qa-jevhi","benchmark_type":"swe_atlas_qa","model_permaslug":"typesafe/jev-router","provider_name":"OpenRouter","timestamp":"2026-10-10T05:19:37.432Z","resumed_from_workflow_id":null,"config_key":"c6cb3b0567b31914ba1eeb9ec1dca054","reasoning_effort":"high","max_tokens":null,"cost_tier":"high","route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.629032,"total_questions":248,"correct_answers":156,"total_cost":380.530255,"generation_time_ms":108538907,"duration_ms":22444432,"input_tokens":243554577,"output_tokens":3896898,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"2874193b-81a9-4528-b3c1-25a59d5e1dab","workflow_id":"t3p1010-dsv41f-high","benchmark_type":"tau3_bench_banking","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"auto-routing","timestamp":"2026-10-10T04:19:03.434Z","resumed_from_workflow_id":null,"config_key":"39f9448840d7ee6465f006b48322c4d8","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.324742,"total_questions":194,"correct_answers":63,"total_cost":20.760549,"generation_time_ms":87067766,"duration_ms":7049434,"input_tokens":289230698,"output_tokens":17203624,"reasoning_tokens":16603047,"success":true,"baseline":true},{"id":"d95b1374-85b9-4eb3-8016-76b567c1ac86","workflow_id":"t3p1010-grok47-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"x-ai/grok-4.7","provider_name":"auto-routing","timestamp":"2026-10-10T03:20:57.352Z","resumed_from_workflow_id":null,"config_key":"5385beb0bc937fdc35332f81f48e547f","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.417526,"total_questions":194,"correct_answers":81,"total_cost":449.349908,"generation_time_ms":51810406,"duration_ms":3543352,"input_tokens":581553361,"output_tokens":2091118,"reasoning_tokens":1154554,"success":true,"baseline":true},{"id":"ca112ecf-4fbc-4ff8-a6d9-1876927a5224","workflow_id":"t3p1010-kimik3-high","benchmark_type":"tau3_bench_banking","model_permaslug":"moonshotai/kimi-k3","provider_name":"auto-routing","timestamp":"2026-10-10T02:53:34.209Z","resumed_from_workflow_id":null,"config_key":"7c75df8a080d65a48493665114937dd7","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.427835,"total_questions":194,"correct_answers":83,"total_cost":188.172706,"generation_time_ms":39374566,"duration_ms":1892209,"input_tokens":351734824,"output_tokens":2022328,"reasoning_tokens":1506918,"success":true,"baseline":true},{"id":"bee816aa-0b04-405d-8183-baf1cb762c83","workflow_id":"t3p1010-glm53f-high","benchmark_type":"tau3_bench_banking","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"auto-routing","timestamp":"2026-10-10T02:49:43.206Z","resumed_from_workflow_id":null,"config_key":"387c28695b1b2954826c9128fba147a7","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.340206,"total_questions":194,"correct_answers":66,"total_cost":8.342272,"generation_time_ms":15134768,"duration_ms":1675206,"input_tokens":204614612,"output_tokens":1100290,"reasoning_tokens":698133,"success":true,"baseline":true},{"id":"2d43e25e-13f1-40e8-bb91-5a09194f2c16","workflow_id":"rb1009-dswe-jevlo","benchmark_type":"deep_swe","model_permaslug":"typesafe/jev-router","provider_name":"OpenRouter","timestamp":"2026-10-10T02:46:26.942Z","resumed_from_workflow_id":null,"config_key":"d43405c8339275beed0e25b083911498","reasoning_effort":"high","max_tokens":null,"cost_tier":"low","route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.659292,"total_questions":226,"correct_answers":149,"total_cost":84.297327,"generation_time_ms":78443274,"duration_ms":13258942,"input_tokens":138100191,"output_tokens":2932810,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"3d7d63f6-5edb-40c7-9ed8-88adb34b14f3","workflow_id":"t3p1010-glm53-high","benchmark_type":"tau3_bench_banking","model_permaslug":"z-ai/glm-5.3","provider_name":"auto-routing","timestamp":"2026-10-10T02:45:22.007Z","resumed_from_workflow_id":null,"config_key":"aa6ebd59450ff89538db75c0ca687a65","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.453608,"total_questions":194,"correct_answers":88,"total_cost":26.526296,"generation_time_ms":33859710,"duration_ms":1415007,"input_tokens":260014535,"output_tokens":3453409,"reasoning_tokens":3074358,"success":true,"baseline":true},{"id":"80a65eda-6cd3-49d1-979d-7a073f73eee1","workflow_id":"t3p1010-kimik3-low","benchmark_type":"tau3_bench_banking","model_permaslug":"moonshotai/kimi-k3","provider_name":"auto-routing","timestamp":"2026-10-10T02:44:47.837Z","resumed_from_workflow_id":null,"config_key":"281647aa8a4bdfc2e9fc856456766c41","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.381443,"total_questions":194,"correct_answers":74,"total_cost":137.376225,"generation_time_ms":30546156,"duration_ms":1364837,"input_tokens":268333343,"output_tokens":1291609,"reasoning_tokens":717985,"success":true,"baseline":true},{"id":"e6438901-698d-4589-b9ce-2c7e16ad4794","workflow_id":"t3p1010-astra-high","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-10-10T02:40:06.485Z","resumed_from_workflow_id":null,"config_key":"db4ea5eed060cabc3ecb0af667f0ee8b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.35567,"total_questions":194,"correct_answers":69,"total_cost":354.400302,"generation_time_ms":24784254,"duration_ms":1120485,"input_tokens":179951307,"output_tokens":669599,"reasoning_tokens":263680,"success":true,"baseline":true},{"id":"96f451f4-9b15-4358-b055-4e909ffc0013","workflow_id":"t3p1010-opus55-high","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:37:38.688Z","resumed_from_workflow_id":null,"config_key":"25803b2d58f64a6916cf262ca20119d3","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.42268,"total_questions":194,"correct_answers":82,"total_cost":189.853288,"generation_time_ms":22132077,"duration_ms":918688,"input_tokens":267066346,"output_tokens":1197021,"reasoning_tokens":389091,"success":true,"baseline":true},{"id":"aa8019a5-6d31-40cd-ac63-868725aff0cc","workflow_id":"t3p1010-sol61-high","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6.1-sol","provider_name":"auto-routing","timestamp":"2026-10-10T02:36:43.420Z","resumed_from_workflow_id":null,"config_key":"e46e166c284c2155e873235d6d5ff8c8","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.350515,"total_questions":194,"correct_answers":68,"total_cost":58.548217,"generation_time_ms":17832131,"duration_ms":854420,"input_tokens":204251391,"output_tokens":624880,"reasoning_tokens":212535,"success":true,"baseline":true},{"id":"4e40fe42-7b45-4b45-b93c-18621404dbf0","workflow_id":"t3p1010-opus55-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:36:37.039Z","resumed_from_workflow_id":null,"config_key":"afea1ac9cbbd6f317f36dbfa1fe47b90","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.396907,"total_questions":194,"correct_answers":77,"total_cost":178.328238,"generation_time_ms":21037363,"duration_ms":852039,"input_tokens":230127813,"output_tokens":1079482,"reasoning_tokens":281123,"success":true,"baseline":true},{"id":"8779ca9e-1ea3-4fa6-9d5f-511000c6029d","workflow_id":"t3p1010-luna-high","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-10-10T02:35:33.910Z","resumed_from_workflow_id":null,"config_key":"70e71e14269c29022b488aaabb7326a8","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.242268,"total_questions":194,"correct_answers":47,"total_cost":3.916447,"generation_time_ms":17677668,"duration_ms":805910,"input_tokens":174370261,"output_tokens":1655195,"reasoning_tokens":1313413,"success":true,"baseline":true},{"id":"5b26dbb7-6c94-4a3d-aa20-5ae3600ae033","workflow_id":"t3p1010-opus55-low","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:35:28.933Z","resumed_from_workflow_id":null,"config_key":"dffd86acd77fd474d87e64e852fd70fe","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.335052,"total_questions":194,"correct_answers":65,"total_cost":134.226022,"generation_time_ms":16544705,"duration_ms":787933,"input_tokens":159383608,"output_tokens":840360,"reasoning_tokens":158229,"success":true,"baseline":true},{"id":"d7bc8286-6786-4c4a-b22d-d46e0446f905","workflow_id":"t3p1010-astra-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-10-10T02:35:11.594Z","resumed_from_workflow_id":null,"config_key":"9a4e3bb03d7e650b9d0d7199a3c86863","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.329897,"total_questions":194,"correct_answers":64,"total_cost":288.849938,"generation_time_ms":17127060,"duration_ms":820594,"input_tokens":150560529,"output_tokens":402550,"reasoning_tokens":37448,"success":true,"baseline":true},{"id":"68bc55a3-1aef-4629-b5a7-e139a1c3696a","workflow_id":"t3p1010-sonnet55-high","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:34:21.530Z","resumed_from_workflow_id":null,"config_key":"b71cf84e0fafbe701a2460b9838df3ca","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.35567,"total_questions":194,"correct_answers":69,"total_cost":69.932633,"generation_time_ms":12684473,"duration_ms":703530,"input_tokens":194967728,"output_tokens":1051926,"reasoning_tokens":344522,"success":true,"baseline":true},{"id":"0908a549-5401-46a8-b034-191efee4cc86","workflow_id":"t3p1010-haiku55-high","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:34:13.427Z","resumed_from_workflow_id":null,"config_key":"066c140be21031cd617efe10c0d04fab","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.185567,"total_questions":194,"correct_answers":36,"total_cost":9.226977,"generation_time_ms":15045265,"duration_ms":737427,"input_tokens":178692155,"output_tokens":1912072,"reasoning_tokens":1193359,"success":true,"baseline":true},{"id":"76d20170-8e8d-4073-b88f-cf8fd2bb5a13","workflow_id":"t3p1010-astra-low","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-10-10T02:33:53.426Z","resumed_from_workflow_id":null,"config_key":"fd59e48524dc15bdc962ff0011c5b714","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.324742,"total_questions":194,"correct_answers":63,"total_cost":225.518622,"generation_time_ms":15633093,"duration_ms":744426,"input_tokens":120295738,"output_tokens":323827,"reasoning_tokens":8244,"success":true,"baseline":true},{"id":"bcb1fd24-81b9-4770-8745-a2dc00772b45","workflow_id":"t3p1010-sol61-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6.1-sol","provider_name":"auto-routing","timestamp":"2026-10-10T02:33:31.208Z","resumed_from_workflow_id":null,"config_key":"dfb79a74e5b6ccb48c10a64b50dd407a","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.371134,"total_questions":194,"correct_answers":72,"total_cost":44.665817,"generation_time_ms":13007060,"duration_ms":656208,"input_tokens":159237993,"output_tokens":385108,"reasoning_tokens":32717,"success":true,"baseline":true},{"id":"12bbeec1-f0a8-443b-99b5-83380f2c0850","workflow_id":"t3p1010-sonnet55-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:33:13.123Z","resumed_from_workflow_id":null,"config_key":"8ed0a9e82d8eaf5b6bf2ee942954e451","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.252577,"total_questions":194,"correct_answers":49,"total_cost":52.677928,"generation_time_ms":10152010,"duration_ms":631123,"input_tokens":136275446,"output_tokens":814138,"reasoning_tokens":203632,"success":true,"baseline":true},{"id":"1e37ee88-83d7-4757-bbc1-5a35c39c4882","workflow_id":"t3p1010-haiku55-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:33:03.076Z","resumed_from_workflow_id":null,"config_key":"a4bdba2681e7b5fa1c921a2a2ef69164","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.14433,"total_questions":194,"correct_answers":28,"total_cost":4.928525,"generation_time_ms":11425844,"duration_ms":663076,"input_tokens":127149954,"output_tokens":1310261,"reasoning_tokens":669863,"success":true,"baseline":true},{"id":"e4fef9b1-c7c4-4432-844e-8da786f6aeab","workflow_id":"t3p1010-sonnet55-low","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:32:48.923Z","resumed_from_workflow_id":null,"config_key":"93cf3175fd83ba81ad44132a941bfb63","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.221649,"total_questions":194,"correct_answers":43,"total_cost":46.559712,"generation_time_ms":9106967,"duration_ms":608923,"input_tokens":114440891,"output_tokens":725224,"reasoning_tokens":159136,"success":true,"baseline":true},{"id":"fcc44e66-aa46-49bf-bce3-a596524366d8","workflow_id":"t3p1010-sol61-low","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6.1-sol","provider_name":"auto-routing","timestamp":"2026-10-10T02:32:33.784Z","resumed_from_workflow_id":null,"config_key":"6ec7ea30f75f901e9a50ed4e4f7b9ba7","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.278351,"total_questions":194,"correct_answers":54,"total_cost":31.495343,"generation_time_ms":11015534,"duration_ms":602784,"input_tokens":108534772,"output_tokens":307571,"reasoning_tokens":6614,"success":true,"baseline":true},{"id":"623414cd-52f9-4188-9090-6439af1b3a2a","workflow_id":"t3p1010-haiku55-low","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:32:18.762Z","resumed_from_workflow_id":null,"config_key":"29901e57a90960abef8ebe35feaa4001","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.092784,"total_questions":194,"correct_answers":18,"total_cost":3.132722,"generation_time_ms":9061332,"duration_ms":620762,"input_tokens":94790075,"output_tokens":991538,"reasoning_tokens":415642,"success":true,"baseline":true},{"id":"ed26887a-2564-4ede-a1b9-920e427c3d95","workflow_id":"rb1009-qa-jevlo","benchmark_type":"swe_atlas_qa","model_permaslug":"typesafe/jev-router","provider_name":"OpenRouter","timestamp":"2026-10-10T02:18:02.582Z","resumed_from_workflow_id":null,"config_key":"74da8a1aa3958f2aaee060372a33af79","reasoning_effort":"high","max_tokens":null,"cost_tier":"low","route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.584677,"total_questions":248,"correct_answers":145,"total_cost":118.396803,"generation_time_ms":67161541,"duration_ms":11549582,"input_tokens":147286385,"output_tokens":2840670,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"d11b66d0-014f-43ba-a98b-28ab27d31ddd","workflow_id":"rbf-qa-dsv41f-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-10-10T01:08:14.874Z","resumed_from_workflow_id":null,"config_key":"33ef10c6ed72766b4ffe9b6fa9cab4f3","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.637097,"total_questions":124,"correct_answers":79,"total_cost":23.175303,"generation_time_ms":81778945,"duration_ms":29590874,"input_tokens":995037635,"output_tokens":5683269,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"683e6860-3177-43b0-a834-18f7fb68b501","workflow_id":"rb1009-mmmu-jevlo-r2","benchmark_type":"mmmu_pro_vision","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-10-10T00:25:11.852Z","resumed_from_workflow_id":null,"config_key":"3264220a537ea728cfdff1a5ad12009a","reasoning_effort":"high","max_tokens":null,"cost_tier":"low","route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.777746,"total_questions":3460,"correct_answers":2691,"total_cost":71.163558,"generation_time_ms":134727108,"duration_ms":4543852,"input_tokens":5570139,"output_tokens":19376931,"reasoning_tokens":18530519,"success":true,"baseline":false},{"id":"e61e7284-d8c9-4572-9349-54b6486335e6","workflow_id":"rb1009-tau3-jevlo","benchmark_type":"tau3_bench_banking","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-10-10T00:05:47.511Z","resumed_from_workflow_id":null,"config_key":"36aaea3b5c0cc7368aa85227d0d7d716","reasoning_effort":"high","max_tokens":null,"cost_tier":"low","route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.309278,"total_questions":194,"correct_answers":60,"total_cost":22.067724,"generation_time_ms":49722474,"duration_ms":3612511,"input_tokens":239316025,"output_tokens":14401227,"reasoning_tokens":13921311,"success":true,"baseline":false},{"id":"08050676-0464-4852-89d7-53b98f30a082","workflow_id":"rb1009-gpqa-jevhi","benchmark_type":"gpqa_diamond","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-10-09T23:23:36.213Z","resumed_from_workflow_id":null,"config_key":"6a664fbb639cffda2542309b352c36a4","reasoning_effort":"high","max_tokens":null,"cost_tier":"high","route_models":null,"temperature":0.5,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.929293,"total_questions":396,"correct_answers":368,"total_cost":9.550938,"generation_time_ms":13634048,"duration_ms":1087213,"input_tokens":371012,"output_tokens":1422659,"reasoning_tokens":1273722,"success":true,"baseline":false},{"id":"3956781a-18ad-4cb1-a1e3-27942b287582","workflow_id":"rb1009-gpqa-jevlo","benchmark_type":"gpqa_diamond","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-10-09T23:22:38.848Z","resumed_from_workflow_id":null,"config_key":"8dc8dc42fbe1b088953798fbd6957793","reasoning_effort":"high","max_tokens":null,"cost_tier":"low","route_models":null,"temperature":0.5,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.929293,"total_questions":396,"correct_answers":368,"total_cost":8.02682,"generation_time_ms":15879297,"duration_ms":1028848,"input_tokens":101562,"output_tokens":1744053,"reasoning_tokens":1610630,"success":true,"baseline":false},{"id":"e9b894f6-a229-4990-bc0d-4c63dde2c8a1","workflow_id":"rbf-dswe-sonnet55-hi","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T23:17:37.226Z","resumed_from_workflow_id":null,"config_key":"1df22d36ef5ff6fbc1846cbbcd46cab6","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.716814,"total_questions":113,"correct_answers":81,"total_cost":209.766246,"generation_time_ms":147335084,"duration_ms":22955226,"input_tokens":249512763,"output_tokens":15278501,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"72e92a79-4856-4a1e-82b3-c433ae2cadb1","workflow_id":"rbf-dswe-haiku55-md","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T23:15:51.582Z","resumed_from_workflow_id":null,"config_key":"ad8e7eaf6947775262f41d3b567f81db","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.522124,"total_questions":113,"correct_answers":59,"total_cost":28.585066,"generation_time_ms":160517722,"duration_ms":22852582,"input_tokens":496877404,"output_tokens":23429182,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"820d07bf-82ea-4f46-ad53-52fc2a2452ed","workflow_id":"rbf-dswe-haiku55-lo","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T22:18:56.174Z","resumed_from_workflow_id":null,"config_key":"a1970f2cb9989dca6dbb2fc5889b7a18","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.40708,"total_questions":113,"correct_answers":46,"total_cost":15.440723,"generation_time_ms":109872322,"duration_ms":19438174,"input_tokens":329871531,"output_tokens":12742228,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"043616b5-e3ec-46f2-b671-99c1c2512ac7","workflow_id":"rbf-qa-gem38f-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"google/gemini-3.8-flash","provider_name":"OpenRouter","timestamp":"2026-10-09T22:18:47.881Z","resumed_from_workflow_id":null,"config_key":"984b97255a508e21490deced28500655","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.435484,"total_questions":124,"correct_answers":54,"total_cost":129.543175,"generation_time_ms":72943244,"duration_ms":19421881,"input_tokens":831543000,"output_tokens":2782098,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"29e6e7b1-1fff-4eb0-b5e6-3649b7d2391b","workflow_id":"rbf-dswe-grok47-md","benchmark_type":"deep_swe","model_permaslug":"x-ai/grok-4.7","provider_name":"OpenRouter","timestamp":"2026-10-09T22:11:30.941Z","resumed_from_workflow_id":null,"config_key":"7f8b39971bbbe2f883cb7a13b634134b","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.353982,"total_questions":113,"correct_answers":40,"total_cost":665.622048,"generation_time_ms":82819085,"duration_ms":18993941,"input_tokens":874818702,"output_tokens":4143443,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"80fb5b08-5121-4a42-9f86-67b87bd92a57","workflow_id":"rbf-dswe-luna-hi","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-10-09T21:44:34.581Z","resumed_from_workflow_id":null,"config_key":"09944ac27ca572c06ab3376341668b02","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.530973,"total_questions":113,"correct_answers":60,"total_cost":12.477907,"generation_time_ms":107102903,"duration_ms":17374581,"input_tokens":583086014,"output_tokens":7661080,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9b8ed6a7-d49a-429e-b655-d7ad6f67c860","workflow_id":"rbf-dswe-dsv41f-hi","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-10-09T21:42:55.993Z","resumed_from_workflow_id":null,"config_key":"bfb5af1dc0240f434ee674efde542166","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.628319,"total_questions":113,"correct_answers":71,"total_cost":42.249287,"generation_time_ms":134893994,"duration_ms":17281993,"input_tokens":1864999745,"output_tokens":12554120,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"d3e51c51-c841-4b55-99ee-ec5034c9247f","workflow_id":"rbf-qa-sol61-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6.1-sol","provider_name":"OpenRouter","timestamp":"2026-10-09T21:42:19.122Z","resumed_from_workflow_id":null,"config_key":"8ad3c59663b9f093d28f8b4f901107cf","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.596774,"total_questions":124,"correct_answers":74,"total_cost":48.198367,"generation_time_ms":43272701,"duration_ms":17229122,"input_tokens":94912795,"output_tokens":1646290,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"ec109e61-cb19-4f32-9828-4a86d0609bbe","workflow_id":"rbf-qa-sol61-md","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6.1-sol","provider_name":"OpenRouter","timestamp":"2026-10-09T21:42:13.934Z","resumed_from_workflow_id":null,"config_key":"993476a4c2cd9f73b7df6d806262a9e2","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.604839,"total_questions":124,"correct_answers":75,"total_cost":27.169469,"generation_time_ms":27316281,"duration_ms":17221934,"input_tokens":45826603,"output_tokens":909432,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"3e8e5d30-84e3-46b0-a5e3-25403fd29797","workflow_id":"rbf-qa-grok47-md","benchmark_type":"swe_atlas_qa","model_permaslug":"x-ai/grok-4.7","provider_name":"OpenRouter","timestamp":"2026-10-09T21:38:26.272Z","resumed_from_workflow_id":null,"config_key":"b38dec7a04b7a719f6c7d98695d6637e","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.58871,"total_questions":124,"correct_answers":73,"total_cost":178.040576,"generation_time_ms":39176084,"duration_ms":16999272,"input_tokens":257795555,"output_tokens":1951611,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"7dd58fa3-bff2-4cc6-acb7-7f09901283af","workflow_id":"rbf-qa-haiku55-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:38:15.670Z","resumed_from_workflow_id":null,"config_key":"6428bcf38a6cbd51723d90e966090a27","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.435484,"total_questions":124,"correct_answers":54,"total_cost":5.896325,"generation_time_ms":42042884,"duration_ms":16987670,"input_tokens":184693025,"output_tokens":4279240,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9f47da29-69dd-4d93-b9b4-de06222f0121","workflow_id":"rbf-qa-sonnet55-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:29:42.871Z","resumed_from_workflow_id":null,"config_key":"eb7f4b130a53595f753cc061759219a8","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.483871,"total_questions":124,"correct_answers":60,"total_cost":32.474459,"generation_time_ms":23899413,"duration_ms":16470871,"input_tokens":63411963,"output_tokens":1402405,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"bf00c532-57e0-44a9-8094-44995e0817c0","workflow_id":"rbf-qa-sonnet55-lo","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:27:32.959Z","resumed_from_workflow_id":null,"config_key":"7bfe4c264f7ac7808a7502393fee3638","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.201613,"total_questions":124,"correct_answers":25,"total_cost":8.610245,"generation_time_ms":8484784,"duration_ms":16339959,"input_tokens":8411336,"output_tokens":435681,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"46d4d528-38f4-4ae9-b06c-8459663d4bb4","workflow_id":"rbf-qa-haiku55-md","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:26:33.667Z","resumed_from_workflow_id":null,"config_key":"887e64e7d037e033a2ef524906ad5570","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.314516,"total_questions":124,"correct_answers":39,"total_cost":1.868429,"generation_time_ms":18929614,"duration_ms":16284667,"input_tokens":57112055,"output_tokens":1631530,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"d38dc0bb-76bc-45fc-ba51-50e7e77fb84b","workflow_id":"rbf-qa-sonnet55-md","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:18:43.619Z","resumed_from_workflow_id":null,"config_key":"555530d8b781322207b8e8aae68c129a","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.330645,"total_questions":124,"correct_answers":41,"total_cost":12.150018,"generation_time_ms":11029897,"duration_ms":15809619,"input_tokens":14820198,"output_tokens":591634,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"d4ab0e1b-843b-4824-9af4-4e324112ea0b","workflow_id":"rbf-qa-sol61-lo","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6.1-sol","provider_name":"OpenRouter","timestamp":"2026-10-09T21:15:20.892Z","resumed_from_workflow_id":null,"config_key":"25e2fc07b96f0dee18566fbb32c2e708","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.516129,"total_questions":124,"correct_answers":64,"total_cost":15.818023,"generation_time_ms":19320743,"duration_ms":15609892,"input_tokens":20945097,"output_tokens":578484,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"0c5e93d8-3ee1-4155-bf0e-a10a00dfce1b","workflow_id":"rbf-qa-haiku55-lo","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:00:13.172Z","resumed_from_workflow_id":null,"config_key":"e7e53ac58ec81dec6d77f3bf8ad7d108","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.241935,"total_questions":124,"correct_answers":30,"total_cost":0.895186,"generation_time_ms":10842289,"duration_ms":14704172,"input_tokens":20222996,"output_tokens":837691,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"c7c7b27a-b97e-4588-bd52-6393fcf58779","workflow_id":"rbf-dswe-astra-hi","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-astra","provider_name":"OpenRouter","timestamp":"2026-10-09T20:04:35.935Z","resumed_from_workflow_id":null,"config_key":"6bc013c788f272191246c18a088b58e2","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.734513,"total_questions":113,"correct_answers":83,"total_cost":337.859193,"generation_time_ms":79210781,"duration_ms":11383935,"input_tokens":123639195,"output_tokens":2636227,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"69e65156-9787-405f-bbc1-f40cbbcfa844","workflow_id":"rbf-dswe-sonnet55-md","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T19:24:51.820Z","resumed_from_workflow_id":null,"config_key":"409a0f188e4e539d2dcb41624b987570","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.495575,"total_questions":113,"correct_answers":56,"total_cost":57.814809,"generation_time_ms":47733608,"duration_ms":8988820,"input_tokens":81388596,"output_tokens":3907023,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9ac28f70-e26b-45f2-9611-16985fcd1228","workflow_id":"rbf-dswe-sonnet55-lo","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T19:13:05.143Z","resumed_from_workflow_id":null,"config_key":"9b1d13ca458af55c00ad0af34997b8fb","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.539823,"total_questions":113,"correct_answers":61,"total_cost":40.117314,"generation_time_ms":33282129,"duration_ms":8282143,"input_tokens":64652931,"output_tokens":2479685,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"8b0e834a-8c57-400d-9d39-1fcb36dd664e","workflow_id":"rbf-dswe-sol61-lo","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6.1-sol","provider_name":"OpenRouter","timestamp":"2026-10-09T18:31:05.307Z","resumed_from_workflow_id":null,"config_key":"de6c8f131688cdc78c154f28b0ae2e8b","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.60177,"total_questions":113,"correct_answers":68,"total_cost":15.359054,"generation_time_ms":24945925,"duration_ms":5764307,"input_tokens":27151159,"output_tokens":670838,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"bee19db1-fbec-4247-b423-a9f8bb39383f","workflow_id":"rb-base3-1791424530-deep-swe-dsv41flash-r1.deepseek/deepseek-v4.1-flash.auto-routing","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"auto-routing","timestamp":"2026-10-08T21:33:36.641Z","resumed_from_workflow_id":"rb-base3-1791424530-deep-swe-dsv41flash.deepseek/deepseek-v4.1-flash.auto-routing","config_key":"be64be36b5760a38b6d6c43f92ea666f","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.619469,"total_questions":113,"correct_answers":70,"total_cost":47.369685,"generation_time_ms":186746875,"duration_ms":12939166,"input_tokens":2893563774,"output_tokens":18530310,"reasoning_tokens":14755412,"success":true,"baseline":true},{"id":"9fb64b24-ab88-4ac9-8079-3a4a75e8fab6","workflow_id":"rb-base3-1791424530-mmmu-pro-vision-dsv41flash.deepseek/deepseek-v4.1-flash.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.452Z","resumed_from_workflow_id":null,"config_key":"44cddfa78584772c12077796950fd5c2","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.682081,"total_questions":1730,"correct_answers":1180,"total_cost":27.704963,"generation_time_ms":219006489,"duration_ms":14023374,"input_tokens":1763806,"output_tokens":35883925,"reasoning_tokens":35362331,"success":true,"baseline":true},{"id":"84c72119-87f7-427a-bbc9-6cdb4af207a8","workflow_id":"rb-base3-1791424530-mmmu-pro-vision-gpt6luna.openai/gpt-6-luna.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.447Z","resumed_from_workflow_id":null,"config_key":"d23c6e7120417ac946e8ede97d646166","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.77341,"total_questions":1730,"correct_answers":1338,"total_cost":3.653491,"generation_time_ms":65062649,"duration_ms":4179063,"input_tokens":6436417,"output_tokens":5708300,"reasoning_tokens":5631910,"success":true,"baseline":true},{"id":"ba20beb7-9dec-4acf-9910-826578180fb2","workflow_id":"rb-base3-1791424530-gpqa-diamond-dsv41flash.deepseek/deepseek-v4.1-flash.auto-routing","benchmark_type":"gpqa_diamond","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.446Z","resumed_from_workflow_id":null,"config_key":"30894b739ad04b4a76b1e8138345ccdd","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.878788,"total_questions":198,"correct_answers":174,"total_cost":2.107504,"generation_time_ms":25270298,"duration_ms":2151452,"input_tokens":53563,"output_tokens":2827034,"reasoning_tokens":2634968,"success":true,"baseline":true},{"id":"68eecb64-c2c7-47e2-81c4-82d4556fa85c","workflow_id":"rb-base3-1791424530-swe-atlas-qa-gpt6luna.openai/gpt-6-luna.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.440Z","resumed_from_workflow_id":null,"config_key":"79c45eb112559e7daa480444659722a2","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.427419,"total_questions":124,"correct_answers":53,"total_cost":7.972291,"generation_time_ms":57699545,"duration_ms":9050163,"input_tokens":292890853,"output_tokens":5470861,"reasoning_tokens":4117299,"success":true,"baseline":true},{"id":"509b735c-8dff-4169-a25e-5da7715a13b3","workflow_id":"rb-base3-1791424530-gpqa-diamond-gpt6luna.openai/gpt-6-luna.auto-routing","benchmark_type":"gpqa_diamond","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.437Z","resumed_from_workflow_id":null,"config_key":"a1355bd8ede06ecb582682be0ac472b1","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.898477,"total_questions":197,"correct_answers":177,"total_cost":0.678113,"generation_time_ms":14442524,"duration_ms":1198304,"input_tokens":50963,"output_tokens":1345892,"reasoning_tokens":1329366,"success":true,"baseline":true},{"id":"bb3d190f-3507-4c45-abfd-36aee32da148","workflow_id":"rb-base3-1791424530-deep-swe-gpt6luna.openai/gpt-6-luna.auto-routing","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.436Z","resumed_from_workflow_id":null,"config_key":"15a8905171c21a005d427e8a7bb4a8c0","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.672566,"total_questions":113,"correct_answers":76,"total_cost":20.735924,"generation_time_ms":119781326,"duration_ms":26073578,"input_tokens":1028868484,"output_tokens":11426982,"reasoning_tokens":8669837,"success":true,"baseline":true},{"id":"a1a2d0ad-242a-485a-90fc-eead69c07a26","workflow_id":"rb-base3-1791424530-tau3-bench-banking-dsv41flash.deepseek/deepseek-v4.1-flash.auto-routing","benchmark_type":"tau3_bench_banking","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.435Z","resumed_from_workflow_id":null,"config_key":"e18e58332a2f68cc476dc54a2d805cc8","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.360825,"total_questions":97,"correct_answers":35,"total_cost":19.451398,"generation_time_ms":104362787,"duration_ms":16704229,"input_tokens":488451118,"output_tokens":11792847,"reasoning_tokens":10866497,"success":true,"baseline":true},{"id":"3d514bb8-042c-48bb-92e1-6a91eec3e472","workflow_id":"rb-base3-1791424530-swe-atlas-qa-dsv41flash.deepseek/deepseek-v4.1-flash.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.434Z","resumed_from_workflow_id":null,"config_key":"0a2b47d25f6a809b3b668e9974e61246","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.653226,"total_questions":124,"correct_answers":81,"total_cost":20.312854,"generation_time_ms":94699810,"duration_ms":14977278,"input_tokens":911271560,"output_tokens":7951664,"reasoning_tokens":5191315,"success":true,"baseline":true},{"id":"65bddcfd-c3e6-4a97-999b-51e74729106b","workflow_id":"rb-base3-1791424530-tau3-bench-banking-gpt6luna.openai/gpt-6-luna.auto-routing","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.433Z","resumed_from_workflow_id":null,"config_key":"e5de429ab9fe68229c52e0c0c3c7c080","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.309278,"total_questions":97,"correct_answers":30,"total_cost":4.427173,"generation_time_ms":22213388,"duration_ms":2749387,"input_tokens":166979163,"output_tokens":2312739,"reasoning_tokens":2086050,"success":true,"baseline":true},{"id":"2dd77b4b-70ae-4396-b583-c1104c614bef","workflow_id":"rb-base3-1791424530-deep-swe-mimo26pro-r1.xiaomi/mimo-v2.6-pro.auto-routing","benchmark_type":"deep_swe","model_permaslug":"xiaomi/mimo-v2.6-pro","provider_name":"auto-routing","timestamp":"2026-10-08T10:18:48.374Z","resumed_from_workflow_id":"rb-base3-1791424530-deep-swe-mimo26pro.xiaomi/mimo-v2.6-pro.auto-routing","config_key":"b35c2be8f2a82d631bbf3a2cfbd29c50","reasoning_effort":"xhigh","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.584071,"total_questions":113,"correct_answers":66,"total_cost":34.680837,"generation_time_ms":293113982,"duration_ms":25672354,"input_tokens":737731003,"output_tokens":10993322,"reasoning_tokens":8508338,"success":true,"baseline":true},{"id":"a40d2a92-b2c1-4a6a-b918-84f714f26a19","workflow_id":"rb-base3-1791424530-gpqa-diamond-mimo26pro.xiaomi/mimo-v2.6-pro.auto-routing","benchmark_type":"gpqa_diamond","model_permaslug":"xiaomi/mimo-v2.6-pro","provider_name":"auto-routing","timestamp":"2026-10-08T01:55:44.175Z","resumed_from_workflow_id":null,"config_key":"b9872ec008602dd478edf9cfb5f45f18","reasoning_effort":"xhigh","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.878788,"total_questions":198,"correct_answers":174,"total_cost":2.473039,"generation_time_ms":62528317,"duration_ms":7701654,"input_tokens":53225,"output_tokens":2816921,"reasoning_tokens":2532529,"success":true,"baseline":true},{"id":"52dd3ac7-cb82-497b-afe9-8f16e6ce4d39","workflow_id":"rb-base3-1791424530-mmmu-pro-vision-mimo26pro.xiaomi/mimo-v2.6-pro.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"xiaomi/mimo-v2.6-pro","provider_name":"auto-routing","timestamp":"2026-10-08T01:55:44.174Z","resumed_from_workflow_id":null,"config_key":"813ba3188272d80e99cd8bc45cf4e0e0","reasoning_effort":"xhigh","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.636416,"total_questions":1730,"correct_answers":1101,"total_cost":18.455067,"generation_time_ms":263650946,"duration_ms":11624646,"input_tokens":4707391,"output_tokens":18940270,"reasoning_tokens":16457981,"success":true,"baseline":true},{"id":"46ba2d52-b9a7-4ed8-9478-f8fec006de2f","workflow_id":"rb-base3-1791424530-tau3-bench-banking-mimo26pro.xiaomi/mimo-v2.6-pro.auto-routing","benchmark_type":"tau3_bench_banking","model_permaslug":"xiaomi/mimo-v2.6-pro","provider_name":"auto-routing","timestamp":"2026-10-08T01:55:44.171Z","resumed_from_workflow_id":null,"config_key":"4f2903df29f7613a80cd2019f26e84ed","reasoning_effort":"xhigh","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.42268,"total_questions":97,"correct_answers":41,"total_cost":12.010454,"generation_time_ms":65593602,"duration_ms":7808855,"input_tokens":259669729,"output_tokens":1797689,"reasoning_tokens":1522135,"success":true,"baseline":true},{"id":"b80d8187-03db-490c-aa8a-2ffd8a385d4a","workflow_id":"rb-base3-1791424530-swe-atlas-qa-mimo26pro.xiaomi/mimo-v2.6-pro.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"xiaomi/mimo-v2.6-pro","provider_name":"auto-routing","timestamp":"2026-10-08T01:55:44.169Z","resumed_from_workflow_id":null,"config_key":"4bfff29082a34d9d27a6500c7822451e","reasoning_effort":"xhigh","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.540323,"total_questions":124,"correct_answers":67,"total_cost":19.06682,"generation_time_ms":151914413,"duration_ms":19644112,"input_tokens":319625795,"output_tokens":5547845,"reasoning_tokens":3829011,"success":true,"baseline":true},{"id":"165ff2c9-aea9-4379-91f3-b6461a97e953","workflow_id":"rb-tb4-pilot-pareto2610p-e2-1002b","benchmark_type":"terminal-bench-4.0","model_permaslug":"unbiased/pareto-26.10-preview","provider_name":"OpenRouter","timestamp":"2026-10-03T07:00:05.466Z","resumed_from_workflow_id":null,"config_key":"02867826243ed42ad8ee880631548be9","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.349206,"total_questions":63,"correct_answers":22,"total_cost":36.005788,"generation_time_ms":190777666,"duration_ms":33368466,"input_tokens":7190621,"output_tokens":6672002,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"a1f7a473-d4aa-44ec-854c-1ec480e62c37","workflow_id":"rb-tb4-pilot-jev-e2-1002b","benchmark_type":"terminal-bench-4.0","model_permaslug":"typesafe/jev-router","provider_name":"OpenRouter","timestamp":"2026-10-03T02:42:30.538Z","resumed_from_workflow_id":null,"config_key":"9e56bcb6a7bada471f7ad750ed67a91c","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.412698,"total_questions":63,"correct_answers":26,"total_cost":0,"generation_time_ms":70347020,"duration_ms":17913538,"input_tokens":950975,"output_tokens":1047576,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"2c2c7586-3f68-4a9e-87db-9b82c71dc61a","workflow_id":"rb-tb4-pilot-pareto2610p-1002a","benchmark_type":"terminal-bench-4.0","model_permaslug":"unbiased/pareto-26.10-preview","provider_name":"OpenRouter","timestamp":"2026-10-02T22:07:34.606Z","resumed_from_workflow_id":null,"config_key":"3ceb13a59b4b87d1ea7c99ded6a4a8ef","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.349206,"total_questions":63,"correct_answers":22,"total_cost":32.880371,"generation_time_ms":106878465,"duration_ms":22357606,"input_tokens":7580670,"output_tokens":5909183,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"11dbbb81-eb97-45e3-ab25-a73649f6603c","workflow_id":"rb-tb4-pilot-jev-1002a","benchmark_type":"terminal-bench-4.0","model_permaslug":"typesafe/jev-router","provider_name":"OpenRouter","timestamp":"2026-10-02T18:13:39.072Z","resumed_from_workflow_id":null,"config_key":"d1fb1cb83d03e07c86f53b517109532c","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.396825,"total_questions":63,"correct_answers":25,"total_cost":0,"generation_time_ms":66624254,"duration_ms":10620072,"input_tokens":5288113,"output_tokens":1209915,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"e393be7f-6000-405b-aa37-28c75bdac3c3","workflow_id":"do-effort-fix-vis-gpqa-glm-5-3-898644","benchmark_type":"gpqa_diamond","model_permaslug":"z-ai/glm-5.3","provider_name":"digitalocean","timestamp":"2026-10-01T23:50:45.024Z","resumed_from_workflow_id":null,"config_key":"d5403d4ad5784feaf21eb2287da7ed2b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.823232,"total_questions":198,"correct_answers":163,"total_cost":5.130773,"generation_time_ms":32137954,"duration_ms":4626790,"input_tokens":52757,"output_tokens":1789179,"reasoning_tokens":1721645,"success":true,"baseline":true},{"id":"aa01109d-e7a4-4408-91c2-a10a193b0421","workflow_id":"do-effort-fix-vis-gpqa-glm-5-3-flash-898644","benchmark_type":"gpqa_diamond","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"digitalocean","timestamp":"2026-10-01T23:50:45.024Z","resumed_from_workflow_id":null,"config_key":"53f9423255de1acbaf388e5aa19f49cc","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.909091,"total_questions":198,"correct_answers":180,"total_cost":1.071675,"generation_time_ms":46696303,"duration_ms":5562896,"input_tokens":52757,"output_tokens":2128444,"reasoning_tokens":2090933,"success":true,"baseline":true},{"id":"a7cb0465-f55d-4bc6-b802-6c11b4c1347f","workflow_id":"rb-5c796641-tau3_bench_banking-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-09-30T22:53:29.695Z","resumed_from_workflow_id":null,"config_key":"d5e04c540a5f76c3e25277258acab745","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.360825,"total_questions":97,"correct_answers":35,"total_cost":26.893662,"generation_time_ms":29394139,"duration_ms":6160722,"input_tokens":202667821,"output_tokens":9113759,"reasoning_tokens":8885017,"success":true,"baseline":false},{"id":"a0c727bb-f76b-4dc5-ac87-bc0a27b340a8","workflow_id":"rb-5c796641-swe_atlas_qa-e2","benchmark_type":"swe_atlas_qa","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-09-30T22:53:29.692Z","resumed_from_workflow_id":null,"config_key":"91044375fd2e4d77bf218e63f9377f40","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.581967,"total_questions":122,"correct_answers":71,"total_cost":107.193426,"generation_time_ms":30094271,"duration_ms":6307620,"input_tokens":61162348,"output_tokens":1292778,"reasoning_tokens":229480,"success":true,"baseline":false},{"id":"34df85bc-f5aa-4322-90d3-e27405624556","workflow_id":"rb-5c796641-swe_atlas_qa-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-09-30T22:53:29.691Z","resumed_from_workflow_id":null,"config_key":"91044375fd2e4d77bf218e63f9377f40","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.532258,"total_questions":124,"correct_answers":66,"total_cost":107.044644,"generation_time_ms":30300239,"duration_ms":6940839,"input_tokens":61423234,"output_tokens":1310017,"reasoning_tokens":225107,"success":true,"baseline":false},{"id":"1cbcbd33-1cea-4cd1-acbc-c0bf172d7303","workflow_id":"rb-5c796641-tau3_bench_banking-e2","benchmark_type":"tau3_bench_banking","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-09-30T22:53:29.689Z","resumed_from_workflow_id":null,"config_key":"d5e04c540a5f76c3e25277258acab745","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.329897,"total_questions":97,"correct_answers":32,"total_cost":30.880732,"generation_time_ms":25071194,"duration_ms":4742255,"input_tokens":212154307,"output_tokens":6531712,"reasoning_tokens":6265704,"success":true,"baseline":false},{"id":"e400871e-5a2d-44ca-b95d-e0f989bb5f27","workflow_id":"rb-qa-1790778500-swe-atlas-qa-opus55-medium.anthropic/claude-opus-5.5.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.432Z","resumed_from_workflow_id":null,"config_key":"02fa8a90fe1a4904d374ed26564d5fbe","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.540323,"total_questions":124,"correct_answers":67,"total_cost":59.31391,"generation_time_ms":20942033,"duration_ms":6947916,"input_tokens":63448374,"output_tokens":1292646,"reasoning_tokens":228826,"success":true,"baseline":true},{"id":"14b062db-0366-4b7a-9bf7-70a7034f4dde","workflow_id":"rb-qa-1790778500-swe-atlas-qa-astra-medium.openai/gpt-6-astra.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.432Z","resumed_from_workflow_id":null,"config_key":"8abc761524a2b079cdcec8a62acfa369","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.588235,"total_questions":119,"correct_answers":70,"total_cost":193.575272,"generation_time_ms":29088950,"duration_ms":5413037,"input_tokens":60007763,"output_tokens":1223093,"reasoning_tokens":166292,"success":true,"baseline":true},{"id":"f3f508d0-04ac-49ae-8530-6bd639ea5e24","workflow_id":"rb-qa-1790778500-swe-atlas-qa-astra-low.openai/gpt-6-astra.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.430Z","resumed_from_workflow_id":null,"config_key":"ad5f63c9b1b1f1ca6b613bcb8e7cd819","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.540984,"total_questions":122,"correct_answers":66,"total_cost":108.104232,"generation_time_ms":19075250,"duration_ms":4365614,"input_tokens":26476751,"output_tokens":764146,"reasoning_tokens":37056,"success":true,"baseline":true},{"id":"db24ae71-a261-4bac-b7a5-ed227d540cd2","workflow_id":"rb-qa-1790778500-swe-atlas-qa-sol61-high.openai/gpt-6.1-sol.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6.1-sol","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.429Z","resumed_from_workflow_id":null,"config_key":"05c115d4ef54c739238702fdc5cca9b5","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.62963,"total_questions":54,"correct_answers":34,"total_cost":17.817865,"generation_time_ms":13762698,"duration_ms":3512451,"input_tokens":30020628,"output_tokens":666645,"reasoning_tokens":137416,"success":true,"baseline":true},{"id":"864edc74-f2fb-4ab3-86a4-adade5ecca27","workflow_id":"rb-qa-1790778500-swe-atlas-qa-opus55-low.anthropic/claude-opus-5.5.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.428Z","resumed_from_workflow_id":null,"config_key":"3cd9ec509a659b275448cd4227acaa7a","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.395161,"total_questions":124,"correct_answers":49,"total_cost":20.32742,"generation_time_ms":8255879,"duration_ms":3017225,"input_tokens":11210200,"output_tokens":509970,"reasoning_tokens":38550,"success":true,"baseline":true},{"id":"a92c5729-22fc-4a95-a68c-12b8907f296f","workflow_id":"rb-qa-1790778500-swe-atlas-qa-sol61-medium.openai/gpt-6.1-sol.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6.1-sol","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.408Z","resumed_from_workflow_id":null,"config_key":"7fa645808a4ef5c4d9b3afcd270c45f9","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.541667,"total_questions":72,"correct_answers":39,"total_cost":16.917425,"generation_time_ms":12607795,"duration_ms":3405369,"input_tokens":27688050,"output_tokens":628247,"reasoning_tokens":67082,"success":true,"baseline":true},{"id":"e0a1ea07-b878-4262-bf81-24ce2fc49f73","workflow_id":"rb-base-1790743549-mmmu-pro-vision-opus55.anthropic/claude-opus-5.5.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.896Z","resumed_from_workflow_id":null,"config_key":"9f0bf5d586ef5f7bacca47943d86c700","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.818497,"total_questions":1730,"correct_answers":1416,"total_cost":45.630146,"generation_time_ms":15628203,"duration_ms":796568,"input_tokens":5511760,"output_tokens":904450,"reasoning_tokens":445852,"success":true,"baseline":true},{"id":"439f3459-857e-4f99-a1f6-aeb935d4fed9","workflow_id":"rb-base-1790743549-mmmu-pro-vision-sol.openai/gpt-6-sol.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"openai/gpt-6-sol","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.886Z","resumed_from_workflow_id":null,"config_key":"53ed5b5a9a6247fe33785b143dbc819f","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.815029,"total_questions":1730,"correct_answers":1410,"total_cost":23.078274,"generation_time_ms":11934330,"duration_ms":937651,"input_tokens":6436417,"output_tokens":709088,"reasoning_tokens":644704,"success":true,"baseline":true},{"id":"8f2c03b2-137b-499a-9fc7-29876393c97f","workflow_id":"rb-base-1790743549-mmmu-pro-vision-luna.openai/gpt-6-luna.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.882Z","resumed_from_workflow_id":null,"config_key":"8ea38502a7335b299bbc9c18483384f3","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.766474,"total_questions":1730,"correct_answers":1326,"total_cost":1.533945,"generation_time_ms":14430514,"duration_ms":1066816,"input_tokens":6436417,"output_tokens":1467735,"reasoning_tokens":1397331,"success":true,"baseline":true},{"id":"68de8b60-3ca5-4e0e-9f58-f2a48f69cc32","workflow_id":"rb-base-1790743549-mmmu-pro-vision-kimik3.moonshotai/kimi-k3.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"moonshotai/kimi-k3","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.882Z","resumed_from_workflow_id":null,"config_key":"e93116204b255297476398f2ff1476be","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.809827,"total_questions":1730,"correct_answers":1401,"total_cost":35.533513,"generation_time_ms":152700961,"duration_ms":7477425,"input_tokens":7142136,"output_tokens":3556659,"reasoning_tokens":3169929,"success":true,"baseline":true},{"id":"96f53548-c13a-4569-8718-dbe9c9c398cf","workflow_id":"rb-base-1790743549-mmmu-pro-vision-glm53flash.z-ai/glm-5.3-flash.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.881Z","resumed_from_workflow_id":null,"config_key":"2433539edcfad3d031d76e6a0266e679","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.754335,"total_questions":1730,"correct_answers":1305,"total_cost":1.381525,"generation_time_ms":57606584,"duration_ms":7822715,"input_tokens":6801788,"output_tokens":2464645,"reasoning_tokens":1729364,"success":true,"baseline":true},{"id":"92f2552b-456f-458e-a721-887e08d0dfd4","workflow_id":"rb-base-1790743549-mmmu-pro-vision-astra.openai/gpt-6-astra.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.841Z","resumed_from_workflow_id":null,"config_key":"1bcac355680345eb579876033cbc541f","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.841618,"total_questions":1730,"correct_answers":1456,"total_cost":101.183043,"generation_time_ms":13537890,"duration_ms":978797,"input_tokens":6436417,"output_tokens":423424,"reasoning_tokens":311954,"success":true,"baseline":true},{"id":"ac6d97f6-0d95-4a42-9441-463cc11833e5","workflow_id":"rb-sy-pass-sol61-high-1790737700-full-swe-atlas-qa.nvidia/switchyard.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-30T03:18:28.501Z","resumed_from_workflow_id":null,"config_key":"4e0b3f1268cb0b729844b7129b9d3d99","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6.1-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.601695,"total_questions":118,"correct_answers":71,"total_cost":43.847998,"generation_time_ms":30486582,"duration_ms":7989789,"input_tokens":79757962,"output_tokens":1613620,"reasoning_tokens":334110,"success":true,"baseline":false},{"id":"1f3287f6-c1a0-489b-bdf1-e1c00b8fd4be","workflow_id":"rb-sy-pass-sol61-high-1790737700-full-tau3-bench-banking.nvidia/switchyard.auto-routing","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-30T03:11:21.814Z","resumed_from_workflow_id":null,"config_key":"0a75ec775e27ab7686c1aa2e914af5e4","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6.1-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.4,"total_questions":90,"correct_answers":36,"total_cost":26.809734,"generation_time_ms":8498550,"duration_ms":1171786,"input_tokens":93263889,"output_tokens":284784,"reasoning_tokens":100557,"success":true,"baseline":false},{"id":"ed14be86-a097-4914-b010-0445cab8a7bb","workflow_id":"rb-sy-pass-sol61-high-1790737700-full-gpqa-diamond.nvidia/switchyard.auto-routing","benchmark_type":"gpqa_diamond","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-30T03:11:21.812Z","resumed_from_workflow_id":null,"config_key":"6e81619287986a60847f4424cb08a036","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6.1-sol"],"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.934343,"total_questions":198,"correct_answers":185,"total_cost":1.321053,"generation_time_ms":2673790,"duration_ms":222590,"input_tokens":51603,"output_tokens":121643,"reasoning_tokens":89171,"success":true,"baseline":false},{"id":"31c50c0b-7c9f-4e9d-904f-6849c66c6227","workflow_id":"rb-sy-pass-sol61-high-1790729500-full-deep-swe.nvidia/switchyard.auto-routing","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-30T00:42:44.198Z","resumed_from_workflow_id":null,"config_key":"aeb39abdf82d1d1707d6337620a857d9","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6.1-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.707965,"total_questions":113,"correct_answers":80,"total_cost":52.245723,"generation_time_ms":44058820,"duration_ms":13541837,"input_tokens":96791231,"output_tokens":2552281,"reasoning_tokens":793577,"success":true,"baseline":false},{"id":"c3ca70b3-7e1c-4971-af01-2442eef3cc79","workflow_id":"rb-jevdte0-1790725200-deep-swe-jevrouter-low-stable.typesafe/jev-router.auto-routing","benchmark_type":"deep_swe","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-09-29T23:38:16.906Z","resumed_from_workflow_id":null,"config_key":"1c871a19964ca6fff859a7a4d89d35bf","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.70354,"total_questions":226,"correct_answers":159,"total_cost":178.493748,"generation_time_ms":56401260,"duration_ms":22847792,"input_tokens":105698263,"output_tokens":3069599,"reasoning_tokens":604825,"success":true,"baseline":false},{"id":"5ca9f27c-7b06-41b5-bb8f-e8b98b74b8f9","workflow_id":"deep-swe-sol61-openai-medium-e2-1790706000.openai/gpt-6.1-sol.openai","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6.1-sol","provider_name":"openai","timestamp":"2026-09-29T18:17:12.848Z","resumed_from_workflow_id":null,"config_key":"810fb60256d1c10095f0c4ea83ca2e74","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.725664,"total_questions":226,"correct_answers":164,"total_cost":67.074024,"generation_time_ms":50021715,"duration_ms":11459853,"input_tokens":118327046,"output_tokens":3191376,"reasoning_tokens":632372,"success":true,"baseline":true},{"id":"951ff29e-0de5-4af5-bcc5-b2316cc82e12","workflow_id":"rb-1790696055-deep-swe-opus55-medium-e2.anthropic/claude-opus-5.5.auto-routing","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-09-29T15:34:16.831Z","resumed_from_workflow_id":null,"config_key":"cc3a5efbe7464f90b54787fe91f9a1d3","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.676991,"total_questions":226,"correct_answers":153,"total_cost":348.041627,"generation_time_ms":94407617,"duration_ms":19716984,"input_tokens":508547299,"output_tokens":7997752,"reasoning_tokens":2761757,"success":true,"baseline":true},{"id":"e9638340-ea2a-46ec-b933-103b89eef570","workflow_id":"rb-1790696055-deep-swe-astra-medium-e2.openai/gpt-6-astra.auto-routing","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-29T15:34:16.829Z","resumed_from_workflow_id":null,"config_key":"8d5da09fb096e407e1defa0fbb54531e","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.699115,"total_questions":226,"correct_answers":158,"total_cost":450.216468,"generation_time_ms":77974058,"duration_ms":25755945,"input_tokens":133745585,"output_tokens":3797739,"reasoning_tokens":871986,"success":true,"baseline":true},{"id":"12ee805d-84e6-4c96-933b-8e0f5c017802","workflow_id":"rb-1790696055-deep-swe-astra-low-e4.openai/gpt-6-astra.auto-routing","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-29T15:34:16.453Z","resumed_from_workflow_id":null,"config_key":"57081aa848898977cae30c0f8b2921e9","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":4,"recorded_epochs":4,"accuracy":0.671875,"total_questions":448,"correct_answers":301,"total_cost":457.699549,"generation_time_ms":83805171,"duration_ms":21491652,"input_tokens":122841742,"output_tokens":3875306,"reasoning_tokens":389507,"success":true,"baseline":true},{"id":"c4012139-61e0-4458-bd49-8839a44bc48b","workflow_id":"rb-autotier-0929-pilot-medium-tau3-banking-e1r","benchmark_type":"tau3_bench_banking","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:53:33.961Z","resumed_from_workflow_id":"rb-autotier-0929-pilot-medium-tau3-banking-e5a","config_key":"1c64af50375ef20b960016f2789036fc","reasoning_effort":"medium","max_tokens":null,"cost_tier":"medium","route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":0,"accuracy":0.371134,"total_questions":97,"correct_answers":37,"total_cost":56.862418,"generation_time_ms":24529843,"duration_ms":1673239,"input_tokens":262458425,"output_tokens":4564238,"reasoning_tokens":4121706,"success":true,"baseline":false},{"id":"2070389f-50fa-45d0-adbc-d19d7a30b17e","workflow_id":"rb-autotier-0929-pilot-medium-mmmu-pro-vision-e1r","benchmark_type":"mmmu_pro_vision","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:53:33.961Z","resumed_from_workflow_id":"rb-autotier-0929-pilot-medium-mmmu-pro-vision-e5a","config_key":"cbbfc8e2cd89d104706e35f33a580cee","reasoning_effort":"high","max_tokens":null,"cost_tier":"medium","route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":0,"accuracy":0.811561,"total_questions":1730,"correct_answers":1410,"total_cost":55.572396,"generation_time_ms":35356828,"duration_ms":1748587,"input_tokens":9808462,"output_tokens":2168315,"reasoning_tokens":1732666,"success":true,"baseline":false},{"id":"9eba396d-4df4-4731-8e4d-3e97c2f56c45","workflow_id":"rb-autotier-0929-pilot-high-deep-swe-e1r","benchmark_type":"deep_swe","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:53:33.960Z","resumed_from_workflow_id":"rb-autotier-0929-pilot-high-deep-swe-e5a","config_key":"2b188c1f97cc898cc874c8748a9edd1f","reasoning_effort":"medium","max_tokens":null,"cost_tier":"high","route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":0,"accuracy":0.568142,"total_questions":113,"correct_answers":65,"total_cost":166.935065,"generation_time_ms":49131165,"duration_ms":11939005,"input_tokens":526582324,"output_tokens":5427349,"reasoning_tokens":2758713,"success":true,"baseline":false},{"id":"7d10e6eb-b125-4086-9b07-d12ed3fc17b0","workflow_id":"rb-autotier-0929-pilot-high-tau3-banking-e1r","benchmark_type":"tau3_bench_banking","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:53:33.960Z","resumed_from_workflow_id":"rb-autotier-0929-pilot-high-tau3-banking-e5a","config_key":"76fcae880329ca39d5e3ea1b38453878","reasoning_effort":"medium","max_tokens":null,"cost_tier":"high","route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":0,"accuracy":0.272165,"total_questions":97,"correct_answers":25,"total_cost":91.486418,"generation_time_ms":25683590,"duration_ms":291467,"input_tokens":244558971,"output_tokens":1280298,"reasoning_tokens":521338,"success":true,"baseline":false},{"id":"5d0c08cc-3d8c-4d06-9414-e27e5689284b","workflow_id":"rb-autotier-0929-pilot-low-swe-atlas-qa-e1r","benchmark_type":"swe_atlas_qa","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:53:33.960Z","resumed_from_workflow_id":"rb-autotier-0929-pilot-low-swe-atlas-qa-e5a","config_key":"574f9f4214b4bf0a7b9c85454efe1c09","reasoning_effort":"medium","max_tokens":null,"cost_tier":"low","route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":0,"accuracy":0.440323,"total_questions":124,"correct_answers":54,"total_cost":14.660797,"generation_time_ms":30018508,"duration_ms":11533554,"input_tokens":372286073,"output_tokens":4434514,"reasoning_tokens":2556199,"success":true,"baseline":false},{"id":"40083ed0-5983-4f35-b418-fb10eb448153","workflow_id":"rb-autotier-0929-pilot-medium-deep-swe-e1r","benchmark_type":"deep_swe","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:53:33.958Z","resumed_from_workflow_id":"rb-autotier-0929-pilot-medium-deep-swe-e5a","config_key":"4d4e22b04a0169f8f1fddbcabfe5ba5e","reasoning_effort":"medium","max_tokens":null,"cost_tier":"medium","route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":0,"accuracy":0.545133,"total_questions":113,"correct_answers":62,"total_cost":90.346356,"generation_time_ms":39682357,"duration_ms":8717049,"input_tokens":316023748,"output_tokens":3962474,"reasoning_tokens":2041331,"success":true,"baseline":false},{"id":"38b5d511-9dcf-4370-ace6-dc581c3670b2","workflow_id":"rb-autotier-0929-pilot-low-mmmu-pro-vision-e1r","benchmark_type":"mmmu_pro_vision","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:53:33.958Z","resumed_from_workflow_id":"rb-autotier-0929-pilot-low-mmmu-pro-vision-e5a","config_key":"973b8e82a0dccafbd3308942cdc14b7d","reasoning_effort":"high","max_tokens":null,"cost_tier":"low","route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":0,"accuracy":0.654451,"total_questions":1730,"correct_answers":1133,"total_cost":53.934196,"generation_time_ms":109732868,"duration_ms":9202258,"input_tokens":2939977,"output_tokens":42428272,"reasoning_tokens":42219830,"success":true,"baseline":false},{"id":"c3bf7c96-b57a-41d9-984f-e3ee1a4c0d78","workflow_id":"rb-autotier-0929-pilot-medium-swe-atlas-qa-e1r","benchmark_type":"swe_atlas_qa","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:53:33.957Z","resumed_from_workflow_id":"rb-autotier-0929-pilot-medium-swe-atlas-qa-e5a","config_key":"4917ff0acd8f4b496d00f33e20eef4e5","reasoning_effort":"medium","max_tokens":null,"cost_tier":"medium","route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":0,"accuracy":0.43871,"total_questions":124,"correct_answers":56,"total_cost":43.746989,"generation_time_ms":24043692,"duration_ms":7122863,"input_tokens":175068903,"output_tokens":1945316,"reasoning_tokens":728978,"success":true,"baseline":false},{"id":"b1465d8f-c96a-471f-9c93-e31a3d9896c3","workflow_id":"rb-autotier-0929-pilot-high-mmmu-pro-vision-e1r","benchmark_type":"mmmu_pro_vision","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:53:33.948Z","resumed_from_workflow_id":"rb-autotier-0929-pilot-high-mmmu-pro-vision-e5a","config_key":"26586fc47ac5d1e69cb56a84d21b1d50","reasoning_effort":"high","max_tokens":null,"cost_tier":"high","route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":0,"accuracy":0.798613,"total_questions":1730,"correct_answers":1383,"total_cost":63.141217,"generation_time_ms":40284331,"duration_ms":2133484,"input_tokens":9852574,"output_tokens":2821149,"reasoning_tokens":2020632,"success":true,"baseline":false},{"id":"3805357d-7c37-4791-ab7f-aefb433fc402","workflow_id":"rb-autotier-0929-pilot-low-tau3-banking-e1r","benchmark_type":"tau3_bench_banking","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:53:33.931Z","resumed_from_workflow_id":"rb-autotier-0929-pilot-low-tau3-banking-e5a","config_key":"279750060b2febec3d0cbfbbd65c8bb3","reasoning_effort":"medium","max_tokens":null,"cost_tier":"low","route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":0,"accuracy":0.082474,"total_questions":97,"correct_answers":9,"total_cost":16.628797,"generation_time_ms":25364070,"duration_ms":3775900,"input_tokens":154270971,"output_tokens":2322026,"reasoning_tokens":1912337,"success":true,"baseline":false},{"id":"c40b558f-a518-4dd5-8227-86ffa0755ea5","workflow_id":"rb-autotier-0929-pilot-low-gpqa-e5a","benchmark_type":"gpqa_diamond","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:28:17.204Z","resumed_from_workflow_id":null,"config_key":"4130e601a660d288ee8f9c782592f0aa","reasoning_effort":"high","max_tokens":null,"cost_tier":"low","route_models":null,"temperature":0.5,"sample_limit":null,"epochs":5,"recorded_epochs":5,"accuracy":0.881818,"total_questions":990,"correct_answers":873,"total_cost":8.987119,"generation_time_ms":27549228,"duration_ms":1518718,"input_tokens":264387,"output_tokens":7592683,"reasoning_tokens":7486292,"success":true,"baseline":false},{"id":"5a004b1c-d2a4-4358-abd4-882a1f8fb81e","workflow_id":"rb-autotier-0929-pilot-medium-gpqa-e5a","benchmark_type":"gpqa_diamond","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:28:17.199Z","resumed_from_workflow_id":null,"config_key":"b6dce29effda19ef3899c61cfe31fad5","reasoning_effort":"high","max_tokens":null,"cost_tier":"medium","route_models":null,"temperature":0.5,"sample_limit":null,"epochs":5,"recorded_epochs":5,"accuracy":0.875758,"total_questions":990,"correct_answers":867,"total_cost":10.428219,"generation_time_ms":28308695,"duration_ms":1459247,"input_tokens":263929,"output_tokens":7782802,"reasoning_tokens":7681279,"success":true,"baseline":false},{"id":"ffc14262-7897-481e-8c91-d874f0d3edfe","workflow_id":"rb-autotier-0929-pilot-high-gpqa-e5a","benchmark_type":"gpqa_diamond","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-29T14:28:17.172Z","resumed_from_workflow_id":null,"config_key":"63e644053b47f61726414bedfea85c39","reasoning_effort":"high","max_tokens":null,"cost_tier":"high","route_models":null,"temperature":0.5,"sample_limit":null,"epochs":5,"recorded_epochs":5,"accuracy":0.832323,"total_questions":990,"correct_answers":824,"total_cost":60.23094,"generation_time_ms":59542550,"duration_ms":3230160,"input_tokens":264143,"output_tokens":11639740,"reasoning_tokens":10643173,"success":true,"baseline":false},{"id":"b93ce9b4-fba1-4289-8ea6-2ac75732a312","workflow_id":"rb-jevv2-1790638867-mmmu-pro-vision.typesafe/jev-router.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-09-28T23:44:57.585Z","resumed_from_workflow_id":null,"config_key":"56b26da1051a84897afae17c6725605e","reasoning_effort":"auto","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.783815,"total_questions":1730,"correct_answers":1356,"total_cost":16.703059,"generation_time_ms":19571015,"duration_ms":869257,"input_tokens":3641371,"output_tokens":3136438,"reasoning_tokens":2848802,"success":true,"baseline":false},{"id":"231c1078-ddfb-496e-bc7a-d246c7d422c0","workflow_id":"rb-aoe-1790615238-mmmu-pro-vision-jevrouter.typesafe/jev-router.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-09-28T18:11:34.038Z","resumed_from_workflow_id":null,"config_key":"f7c77776a5672aff86f1295c785e0187","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.722543,"total_questions":1730,"correct_answers":1250,"total_cost":42.590188,"generation_time_ms":81217113,"duration_ms":4671498,"input_tokens":3318235,"output_tokens":25993936,"reasoning_tokens":25807004,"success":true,"baseline":false},{"id":"a61f53e9-b788-4124-9010-5c2b8e553749","workflow_id":"rb-aoe-1790615238-gpqa-diamond-jevrouter.typesafe/jev-router.auto-routing","benchmark_type":"gpqa_diamond","model_permaslug":"typesafe/jev-router","provider_name":"auto-routing","timestamp":"2026-09-28T18:11:34.031Z","resumed_from_workflow_id":null,"config_key":"fdfb7b2962b165dfc40a826e1907ffa9","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.939394,"total_questions":198,"correct_answers":186,"total_cost":3.153439,"generation_time_ms":4883840,"duration_ms":580009,"input_tokens":66329,"output_tokens":439368,"reasoning_tokens":400891,"success":true,"baseline":false},{"id":"8dadccd7-fcbc-4d68-b86e-25f859fa36c9","workflow_id":"rbdswe-1790443217-gpt6sol-high-full.openai/gpt-6-sol.auto-routing","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-sol","provider_name":"auto-routing","timestamp":"2026-09-26T17:43:46.354Z","resumed_from_workflow_id":null,"config_key":"bd2352da21f38302cc009c3f1405d29b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.610619,"total_questions":113,"correct_answers":69,"total_cost":51.659173,"generation_time_ms":24557267,"duration_ms":16533847,"input_tokens":103173185,"output_tokens":1806449,"reasoning_tokens":554395,"success":true,"baseline":true},{"id":"1ab5e75b-191f-487f-b90b-be9c2c46c613","workflow_id":"rbdswe-1790443217-kimik3-low-full.moonshotai/kimi-k3.auto-routing","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3","provider_name":"auto-routing","timestamp":"2026-09-26T17:33:18.660Z","resumed_from_workflow_id":null,"config_key":"23ef6ec4945cd9bfe22f2be4c9a20ac3","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.575221,"total_questions":113,"correct_answers":65,"total_cost":71.625343,"generation_time_ms":29971765,"duration_ms":17212798,"input_tokens":149978734,"output_tokens":2410766,"reasoning_tokens":1353329,"success":true,"baseline":true},{"id":"6cc62970-3874-475e-bb0a-10ef9eaa09ac","workflow_id":"deep-swe-opus-5-5-low-full-1790432600.anthropic/claude-opus-5.5.auto-routing","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-09-26T14:22:23.010Z","resumed_from_workflow_id":null,"config_key":"c2b08efd242c071b6740b9e1ad1f8a1b","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.610619,"total_questions":113,"correct_answers":69,"total_cost":58.612545,"generation_time_ms":17622610,"duration_ms":14674842,"input_tokens":62501008,"output_tokens":1438965,"reasoning_tokens":313717,"success":true,"baseline":true},{"id":"eef0f5dc-04ef-43a2-b0f5-94414dfeef25","workflow_id":"deep-swe-gpt-6-astra-low-full-1790429800.openai/gpt-6-astra.auto-routing","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-26T13:36:35.897Z","resumed_from_workflow_id":null,"config_key":"57081aa848898977cae30c0f8b2921e9","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.690265,"total_questions":113,"correct_answers":78,"total_cost":116.060753,"generation_time_ms":18373489,"duration_ms":3794680,"input_tokens":30706653,"output_tokens":993796,"reasoning_tokens":104484,"success":true,"baseline":true},{"id":"e5dc9b2b-9a78-42d7-a1e3-2e3f3c99b561","workflow_id":"rb-sy2-swe-atlas-qa-glm-5.3-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.492Z","resumed_from_workflow_id":null,"config_key":"65da924c80aa72e597bab844f5259b5c","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.572581,"total_questions":124,"correct_answers":71,"total_cost":79.881578,"generation_time_ms":67685398,"duration_ms":17550417,"input_tokens":390624350,"output_tokens":3621554,"reasoning_tokens":2029200,"success":true,"baseline":false},{"id":"755ac704-b820-46f8-9d83-1deee7c0c897","workflow_id":"rb-sy2-deep-swe-gpt-6-luna-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.475Z","resumed_from_workflow_id":null,"config_key":"6b6bd77e89e943b3b23a5ea919aae3d5","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-luna"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.530973,"total_questions":113,"correct_answers":60,"total_cost":7.500924,"generation_time_ms":46323089,"duration_ms":12458727,"input_tokens":393062965,"output_tokens":4442109,"reasoning_tokens":2624293,"success":true,"baseline":false},{"id":"4381aa3b-ef8c-4e56-9deb-384244a87cad","workflow_id":"rb-sy2-tau3-banking-glm-5.3-flash-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.474Z","resumed_from_workflow_id":null,"config_key":"7ad57946a6cf682846fa465e7ab177a4","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3-flash"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.371134,"total_questions":97,"correct_answers":36,"total_cost":3.500089,"generation_time_ms":23114942,"duration_ms":3085055,"input_tokens":93464243,"output_tokens":805710,"reasoning_tokens":571913,"success":true,"baseline":false},{"id":"1de7c2ab-cb68-4402-b3c1-a26c18f40f6f","workflow_id":"rb-sy2-deep-swe-glm-5.3-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.474Z","resumed_from_workflow_id":null,"config_key":"37531a6c4e5fcadb8389f451f129d3b6","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.646018,"total_questions":113,"correct_answers":73,"total_cost":165.31288,"generation_time_ms":110230755,"duration_ms":24180261,"input_tokens":805035182,"output_tokens":6760634,"reasoning_tokens":4091951,"success":true,"baseline":false},{"id":"499c3fdf-a765-48b6-8edd-7ebc1c7da89a","workflow_id":"rb-sy2-gpqa-gpt-6-sol-e1","benchmark_type":"gpqa_diamond","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.470Z","resumed_from_workflow_id":null,"config_key":"43e4a0bb2111a0207fa609a0b7298f52","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-sol"],"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.914141,"total_questions":198,"correct_answers":181,"total_cost":1.680313,"generation_time_ms":2459868,"duration_ms":830422,"input_tokens":51603,"output_tokens":157569,"reasoning_tokens":142895,"success":true,"baseline":false},{"id":"f893c2ab-ab28-4abd-9987-5becf1b36e40","workflow_id":"rb-sy2-gpqa-gpt-6-luna-e1","benchmark_type":"gpqa_diamond","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.469Z","resumed_from_workflow_id":null,"config_key":"77f89a9d4b1d3ffc9e3dbc2e6340ae0f","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-luna"],"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.868687,"total_questions":198,"correct_answers":172,"total_cost":0.194184,"generation_time_ms":3480886,"duration_ms":845517,"input_tokens":51603,"output_tokens":377906,"reasoning_tokens":361408,"success":true,"baseline":false},{"id":"ca155568-55e9-43fb-af49-ece5d0b7bfcd","workflow_id":"rb-sy2-deep-swe-kimi-k3-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.467Z","resumed_from_workflow_id":null,"config_key":"3adcee5e07b29bec643c70c9f65d2477","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["moonshotai/kimi-k3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.690265,"total_questions":113,"correct_answers":78,"total_cost":209.421451,"generation_time_ms":156955794,"duration_ms":27825503,"input_tokens":531865780,"output_tokens":6374976,"reasoning_tokens":4033413,"success":true,"baseline":false},{"id":"c2b5729c-774a-4436-a7ce-3dba6b62f0f2","workflow_id":"rb-sy2-deep-swe-opus-5.5-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.467Z","resumed_from_workflow_id":null,"config_key":"4269908affdb9f3e6b24d7a9f81559d2","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["anthropic/claude-opus-5.5"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.734513,"total_questions":113,"correct_answers":83,"total_cost":285.011051,"generation_time_ms":68572475,"duration_ms":11821433,"input_tokens":466458925,"output_tokens":6401253,"reasoning_tokens":2630333,"success":true,"baseline":false},{"id":"f8be8bb3-1d57-48b2-ae31-bea7bbbac01f","workflow_id":"rb-sy2-tau3-banking-kimi-k3-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.466Z","resumed_from_workflow_id":null,"config_key":"58e1a20ebc39b2870924bbeedfc4a2a4","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["moonshotai/kimi-k3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.381443,"total_questions":97,"correct_answers":37,"total_cost":79.256036,"generation_time_ms":23953672,"duration_ms":3661074,"input_tokens":174668796,"output_tokens":1352334,"reasoning_tokens":1055782,"success":true,"baseline":false},{"id":"7f942fdb-9cbd-4476-8493-ea594a1ddde2","workflow_id":"rb-sy2-deep-swe-glm-5.3-flash-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.466Z","resumed_from_workflow_id":null,"config_key":"c5be45086cc4476cd096912811ecd0a5","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3-flash"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.309735,"total_questions":113,"correct_answers":35,"total_cost":38.67259,"generation_time_ms":183324932,"duration_ms":43380141,"input_tokens":1346445400,"output_tokens":7552682,"reasoning_tokens":4354715,"success":true,"baseline":false},{"id":"349408f7-d74d-4e82-9c98-87ff66919e43","workflow_id":"rb-sy2-gpqa-glm-5.3-e1","benchmark_type":"gpqa_diamond","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.466Z","resumed_from_workflow_id":null,"config_key":"bc593430cc04c9fcac9b48f571c6b6ff","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3"],"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.853535,"total_questions":198,"correct_answers":169,"total_cost":3.918581,"generation_time_ms":21608838,"duration_ms":3135642,"input_tokens":52757,"output_tokens":2204128,"reasoning_tokens":2139791,"success":true,"baseline":false},{"id":"ab45d8fc-8a1e-4762-b59d-70b43382c944","workflow_id":"rb-sy2-swe-atlas-qa-gpt-6-luna-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.465Z","resumed_from_workflow_id":null,"config_key":"7546a08366221a1286545dfc909dbedd","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-luna"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.354839,"total_questions":124,"correct_answers":44,"total_cost":3.378771,"generation_time_ms":20565552,"duration_ms":6814409,"input_tokens":123903528,"output_tokens":1960772,"reasoning_tokens":1114253,"success":true,"baseline":false},{"id":"74de37d5-a739-4836-b3d7-293d5086ecd8","workflow_id":"rb-sy2-deep-swe-gpt-6-astra-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.465Z","resumed_from_workflow_id":null,"config_key":"8b94f5faaf143d2deacbec01a3046dff","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-astra"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.716814,"total_questions":113,"correct_answers":81,"total_cost":303.860228,"generation_time_ms":52143513,"duration_ms":11324406,"input_tokens":92872741,"output_tokens":2581959,"reasoning_tokens":808141,"success":true,"baseline":false},{"id":"e6dab572-4300-4544-b46a-dbef2df7219a","workflow_id":"rb-sy2-gpqa-kimi-k3-e1","benchmark_type":"gpqa_diamond","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.463Z","resumed_from_workflow_id":null,"config_key":"4f2d8ebd0d4c506317388e2708686ab9","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["moonshotai/kimi-k3"],"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.949495,"total_questions":198,"correct_answers":188,"total_cost":7.091439,"generation_time_ms":8064670,"duration_ms":1835620,"input_tokens":69209,"output_tokens":650009,"reasoning_tokens":617249,"success":true,"baseline":false},{"id":"317076e4-c08f-47b3-92ba-854767ea15f4","workflow_id":"rb-sy2-tau3-banking-glm-5.3-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.463Z","resumed_from_workflow_id":null,"config_key":"f42c85bdbd6833935dd40652771dd34d","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.381443,"total_questions":97,"correct_answers":37,"total_cost":18.243153,"generation_time_ms":11102490,"duration_ms":1704176,"input_tokens":73461209,"output_tokens":594469,"reasoning_tokens":374071,"success":true,"baseline":false},{"id":"0e74d1e5-4fba-488e-90c0-9b2652b87266","workflow_id":"rb-sy2-swe-atlas-qa-gpt-6-astra-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.463Z","resumed_from_workflow_id":null,"config_key":"386401ae0243becfb10e7473827318b1","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-astra"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.655462,"total_questions":119,"correct_answers":78,"total_cost":244.95841,"generation_time_ms":34152701,"duration_ms":6629183,"input_tokens":75023099,"output_tokens":1575736,"reasoning_tokens":330588,"success":true,"baseline":false},{"id":"d8210edd-438c-4d39-a2cb-74dc156bb386","workflow_id":"rb-sy2-swe-atlas-qa-glm-5.3-flash-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.462Z","resumed_from_workflow_id":null,"config_key":"094bac727a85f033d47ef20ec62e5211","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3-flash"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.427419,"total_questions":124,"correct_answers":53,"total_cost":24.573037,"generation_time_ms":112241756,"duration_ms":32969291,"input_tokens":758829338,"output_tokens":3334940,"reasoning_tokens":1147003,"success":true,"baseline":false},{"id":"57af040d-f063-4545-b2b4-d48808865d47","workflow_id":"rb-sy2-gpqa-gpt-6-astra-e1","benchmark_type":"gpqa_diamond","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.462Z","resumed_from_workflow_id":null,"config_key":"2ad41862447eb9db9da44cb280062fb6","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-astra"],"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.949495,"total_questions":198,"correct_answers":188,"total_cost":6.700163,"generation_time_ms":3172392,"duration_ms":783251,"input_tokens":51603,"output_tokens":123541,"reasoning_tokens":89783,"success":true,"baseline":false},{"id":"8adc4422-6c33-437c-ac67-3f59c70678fc","workflow_id":"rb-sy2-swe-atlas-qa-opus-5.5-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.461Z","resumed_from_workflow_id":null,"config_key":"a8610eb8c38c2a652164d11318f98b22","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["anthropic/claude-opus-5.5"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.556452,"total_questions":124,"correct_answers":69,"total_cost":98.336004,"generation_time_ms":28300527,"duration_ms":10123391,"input_tokens":134632195,"output_tokens":1954062,"reasoning_tokens":479825,"success":true,"baseline":false},{"id":"cbc48863-9f93-4a30-b412-5ce65e506af3","workflow_id":"rb-sy2-swe-atlas-qa-gpt-6-sol-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.460Z","resumed_from_workflow_id":null,"config_key":"7564c56b45ff6a63e21f296ca0ca67a3","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.553719,"total_questions":121,"correct_answers":67,"total_cost":42.648324,"generation_time_ms":19102401,"duration_ms":6615331,"input_tokens":81392889,"output_tokens":1072116,"reasoning_tokens":239787,"success":true,"baseline":false},{"id":"b12fd7bd-e760-476d-8ac1-19cafb752451","workflow_id":"rb-sy2-tau3-banking-gpt-6-astra-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.460Z","resumed_from_workflow_id":null,"config_key":"4cf1fc0452f66150c088ad95d6c9ca16","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-astra"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.391753,"total_questions":97,"correct_answers":38,"total_cost":181.777802,"generation_time_ms":12312741,"duration_ms":1791444,"input_tokens":93104908,"output_tokens":335460,"reasoning_tokens":133169,"success":true,"baseline":false},{"id":"a4eab3ca-57bd-4808-9fd2-a1be3fdc703c","workflow_id":"rb-sy2-tau3-banking-gpt-6-luna-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.460Z","resumed_from_workflow_id":null,"config_key":"97274dcbc07fcfbd764664c6cdd25c6c","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-luna"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.247423,"total_questions":97,"correct_answers":24,"total_cost":1.964327,"generation_time_ms":10077332,"duration_ms":1558428,"input_tokens":86079711,"output_tokens":852326,"reasoning_tokens":683991,"success":true,"baseline":false},{"id":"205cab7b-cb83-4840-a668-4762f21028bc","workflow_id":"rb-sy2-tau3-banking-opus-5.5-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.460Z","resumed_from_workflow_id":null,"config_key":"443b674d7a7c693e085aab52b11af045","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["anthropic/claude-opus-5.5"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.42268,"total_questions":97,"correct_answers":41,"total_cost":71.171822,"generation_time_ms":12229173,"duration_ms":1756792,"input_tokens":121420226,"output_tokens":590819,"reasoning_tokens":191391,"success":true,"baseline":false},{"id":"57ca9207-1524-45f7-b476-3e422bbde67a","workflow_id":"rb-sy2-gpqa-glm-5.3-flash-e1","benchmark_type":"gpqa_diamond","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.457Z","resumed_from_workflow_id":null,"config_key":"4e9e5e4226a4d488d18291c89af0e7bf","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3-flash"],"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.893401,"total_questions":197,"correct_answers":176,"total_cost":0.588095,"generation_time_ms":57948853,"duration_ms":13269456,"input_tokens":54714,"output_tokens":1906995,"reasoning_tokens":1857813,"success":true,"baseline":false},{"id":"fa77744f-616d-4566-a946-c19c9a86a08f","workflow_id":"rb-sy2-swe-atlas-qa-kimi-k3-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.436Z","resumed_from_workflow_id":null,"config_key":"3a04ee35e708ab56870b65a1fc96950b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["moonshotai/kimi-k3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.580645,"total_questions":124,"correct_answers":72,"total_cost":107.139966,"generation_time_ms":60598317,"duration_ms":15396268,"input_tokens":239186689,"output_tokens":2934668,"reasoning_tokens":1567034,"success":true,"baseline":false},{"id":"4fb0f7de-9f30-4ed2-a977-75aac8754210","workflow_id":"rb-sy2-gpqa-opus-5.5-e1","benchmark_type":"gpqa_diamond","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.436Z","resumed_from_workflow_id":null,"config_key":"f19307fecfab8ab101847cf6f278e8cf","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["anthropic/claude-opus-5.5"],"temperature":0.5,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.893939,"total_questions":198,"correct_answers":177,"total_cost":3.559004,"generation_time_ms":1977303,"duration_ms":711367,"input_tokens":66672,"output_tokens":164011,"reasoning_tokens":66896,"success":true,"baseline":false},{"id":"492f9cf9-1019-4d5f-b586-0db79ed58044","workflow_id":"rb-sy2-deep-swe-gpt-6-sol-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.431Z","resumed_from_workflow_id":null,"config_key":"c38d0de954cd7d54cd19d45520387d49","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.575221,"total_questions":113,"correct_answers":65,"total_cost":53.744126,"generation_time_ms":29321465,"duration_ms":6182622,"input_tokens":106719256,"output_tokens":1884000,"reasoning_tokens":569472,"success":true,"baseline":false},{"id":"96ba2886-4b28-4927-88ae-412bc80e267d","workflow_id":"rb-sy2-tau3-banking-gpt-6-sol-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.429Z","resumed_from_workflow_id":null,"config_key":"4ea33e0f90ee1458d47077dfe9ad0c90","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.42268,"total_questions":97,"correct_answers":41,"total_cost":52.904438,"generation_time_ms":9137863,"duration_ms":1468524,"input_tokens":137742468,"output_tokens":437920,"reasoning_tokens":226743,"success":true,"baseline":false},{"id":"a88b0560-94d9-43aa-a331-cfd22320d324","workflow_id":"rb-7741ebdf-ext5b-fugu-max-deep-swe","benchmark_type":"deep_swe","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-18T17:14:33.283Z","resumed_from_workflow_id":null,"config_key":"966cf9b20f1df4816400a69d5baf59dd","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.339286,"total_questions":224,"correct_answers":76,"total_cost":672.333734,"generation_time_ms":341844679,"duration_ms":62962733,"input_tokens":1811999956,"output_tokens":26015609,"reasoning_tokens":19580403,"success":true,"baseline":false},{"id":"da7352d2-7e87-49c3-9df5-70b71c14dcee","workflow_id":"rb-7741ebdf-ext5b-fugu-ultra-v2-swe-atlas-qa","benchmark_type":"swe_atlas_qa","model_permaslug":"sakana/fugu-ultra-v2","provider_name":"auto-routing","timestamp":"2026-09-18T17:14:33.223Z","resumed_from_workflow_id":null,"config_key":"a26832f83753b3092f51e30f4cda26a0","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.694915,"total_questions":118,"correct_answers":82,"total_cost":1051.581245,"generation_time_ms":109569714,"duration_ms":19021856,"input_tokens":568897814,"output_tokens":7444848,"reasoning_tokens":2202523,"success":true,"baseline":false},{"id":"e654a444-a0ff-4307-b763-2d1c47cfc36c","workflow_id":"rb-7741ebdf-ext4-fugu-max-mmmu-pro-vision","benchmark_type":"mmmu_pro_vision","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-18T11:55:14.198Z","resumed_from_workflow_id":null,"config_key":"c7c88655aafd2ff3f66e9a009ddf0b1f","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.794605,"total_questions":5190,"correct_answers":4124,"total_cost":100.348075,"generation_time_ms":126036381,"duration_ms":7168366,"input_tokens":8029093,"output_tokens":14302323,"reasoning_tokens":13288841,"success":true,"baseline":false},{"id":"30d03ff8-777a-475f-b04d-fb73ae05785b","workflow_id":"rb-7741ebdf-ext4-fugu-max-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-18T11:55:14.193Z","resumed_from_workflow_id":null,"config_key":"631d44e09067c0e58125bd89a32ee325","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.298969,"total_questions":97,"correct_answers":29,"total_cost":57.555494,"generation_time_ms":19408156,"duration_ms":3587601,"input_tokens":104492046,"output_tokens":1491398,"reasoning_tokens":1224420,"success":true,"baseline":false},{"id":"fc4aa9e6-eaaf-4cf2-9cc7-62e21940f167","workflow_id":"rb-7741ebdf-ext4-fugu-ultra-v2-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"sakana/fugu-ultra-v2","provider_name":"auto-routing","timestamp":"2026-09-18T11:55:14.190Z","resumed_from_workflow_id":null,"config_key":"87ecead7895f766c6093ecc3a56c2f10","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.350515,"total_questions":97,"correct_answers":34,"total_cost":275.673304,"generation_time_ms":23940758,"duration_ms":3608817,"input_tokens":131713843,"output_tokens":1327595,"reasoning_tokens":802896,"success":true,"baseline":false},{"id":"672b44c4-0d7c-41b5-92e7-17c2066cbdab","workflow_id":"rb-7741ebdf-ext4-fugu-ultra-v2-mmmu-pro-vision","benchmark_type":"mmmu_pro_vision","model_permaslug":"sakana/fugu-ultra-v2","provider_name":"auto-routing","timestamp":"2026-09-18T11:55:14.184Z","resumed_from_workflow_id":null,"config_key":"a10543c341a126603b5856b3a301ba50","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.80501,"total_questions":5190,"correct_answers":4178,"total_cost":567.894964,"generation_time_ms":218281287,"duration_ms":9977928,"input_tokens":85928291,"output_tokens":5902077,"reasoning_tokens":587726,"success":true,"baseline":false},{"id":"a511c523-45a5-4e63-9e2d-6ddb22fc3fd6","workflow_id":"rb-7741ebdf-ext4-fugu-max-swe-atlas-qa","benchmark_type":"swe_atlas_qa","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-18T11:55:13.783Z","resumed_from_workflow_id":null,"config_key":"392fe58ae6feb02dc5fca40e1e66a234","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.524194,"total_questions":124,"correct_answers":65,"total_cost":193.8577,"generation_time_ms":82553717,"duration_ms":19650054,"input_tokens":530337376,"output_tokens":6213915,"reasoning_tokens":4028249,"success":true,"baseline":false},{"id":"394d6b67-5432-40bb-9b70-68be471a5ee5","workflow_id":"rb-7741ebdf-ext2-union-alpha-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"unbiased/pareto-20260917","provider_name":"auto-routing","timestamp":"2026-09-17T13:12:55.247Z","resumed_from_workflow_id":null,"config_key":"46f623096bd58feaf9465ca01b31bc73","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.371134,"total_questions":97,"correct_answers":36,"total_cost":0.010133,"generation_time_ms":16051027,"duration_ms":2849119,"input_tokens":82274590,"output_tokens":320734,"reasoning_tokens":85,"success":true,"baseline":false},{"id":"0fa6f6e3-5dc1-48cc-af42-0d6461f94cda","workflow_id":"rb-7741ebdf-ext-fugu-max-deep-swe","benchmark_type":"deep_swe","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-16T03:06:09.565Z","resumed_from_workflow_id":null,"config_key":"966cf9b20f1df4816400a69d5baf59dd","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.383929,"total_questions":112,"correct_answers":43,"total_cost":322.464822,"generation_time_ms":166433486,"duration_ms":44515180,"input_tokens":855890840,"output_tokens":13167225,"reasoning_tokens":10921808,"success":true,"baseline":false},{"id":"82ffd18b-05b5-41f4-b0b4-29edb12978cb","workflow_id":"rb-7741ebdf-ext-fugu-max-swe-atlas-qa","benchmark_type":"swe_atlas_qa","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-16T02:46:05.137Z","resumed_from_workflow_id":null,"config_key":"392fe58ae6feb02dc5fca40e1e66a234","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.528226,"total_questions":248,"correct_answers":131,"total_cost":406.188523,"generation_time_ms":130083478,"duration_ms":22825043,"input_tokens":1045289588,"output_tokens":13086401,"reasoning_tokens":8675369,"success":true,"baseline":false},{"id":"cf5eaf24-3fa3-4ca3-9c3c-e1ee7387e810","workflow_id":"rb-7741ebdf-ext-fugu-max-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-16T02:46:05.133Z","resumed_from_workflow_id":null,"config_key":"631d44e09067c0e58125bd89a32ee325","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.304124,"total_questions":194,"correct_answers":59,"total_cost":84.783657,"generation_time_ms":23498566,"duration_ms":3122623,"input_tokens":139764660,"output_tokens":2370109,"reasoning_tokens":1945169,"success":true,"baseline":false},{"id":"a5a585d4-4756-4390-ada5-22a5be25e79e","workflow_id":"rb-7741ebdf-ext-fugu-ultra-v2-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"sakana/fugu-ultra-v2","provider_name":"auto-routing","timestamp":"2026-09-16T02:46:05.133Z","resumed_from_workflow_id":null,"config_key":"87ecead7895f766c6093ecc3a56c2f10","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.345361,"total_questions":194,"correct_answers":67,"total_cost":628.777126,"generation_time_ms":51402317,"duration_ms":4308575,"input_tokens":303783938,"output_tokens":3071479,"reasoning_tokens":1901712,"success":true,"baseline":false},{"id":"41aeb205-db2f-4f9c-a07d-9932ac2627ad","workflow_id":"rb-7741ebdf-ext-union-alpha-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"unbiased/pareto-20260917","provider_name":"auto-routing","timestamp":"2026-09-16T02:46:04.601Z","resumed_from_workflow_id":null,"config_key":"46f623096bd58feaf9465ca01b31bc73","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.350515,"total_questions":194,"correct_answers":68,"total_cost":0.005834,"generation_time_ms":17929327,"duration_ms":1692279,"input_tokens":179923246,"output_tokens":845421,"reasoning_tokens":393,"success":true,"baseline":false},{"id":"e95ba7bb-a708-44d8-918c-ab60aab3692f","workflow_id":"rb-7741ebdf-pilot-auto-search-hle","benchmark_type":"search_hle","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-15T18:51:27.288Z","resumed_from_workflow_id":null,"config_key":"e15d82a9092d75202869b7b3709fc905","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.524503,"total_questions":2061,"correct_answers":1081,"total_cost":168.79506,"generation_time_ms":254008292,"duration_ms":72025759,"input_tokens":263870077,"output_tokens":36394607,"reasoning_tokens":33677094,"success":true,"baseline":false},{"id":"d41c0935-de23-45d3-a42a-a82e4e1d3fe6","workflow_id":"rb-7741ebdf-pilot-auto-mmmu-pro-vision","benchmark_type":"mmmu_pro_vision","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-15T18:49:01.492Z","resumed_from_workflow_id":null,"config_key":"cc08532109262d3bf45695d00da9cf72","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.744316,"total_questions":5190,"correct_answers":3863,"total_cost":32.949705,"generation_time_ms":140726769,"duration_ms":10495867,"input_tokens":16233961,"output_tokens":14752498,"reasoning_tokens":13136167,"success":true,"baseline":false},{"id":"b31d55ff-2786-464c-a413-83c8104cef39","workflow_id":"rb-7741ebdf-pilot-auto-deep-swe","benchmark_type":"deep_swe","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-15T17:19:36.246Z","resumed_from_workflow_id":null,"config_key":"3fbb66e0a988cd24f2a9651673c8bff1","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.157817,"total_questions":336,"correct_answers":51,"total_cost":15.32906,"generation_time_ms":58372403,"duration_ms":16573748,"input_tokens":364344616,"output_tokens":3333815,"reasoning_tokens":1032928,"success":true,"baseline":false},{"id":"e13c5903-b77a-4d53-9482-b47e3a9b6317","workflow_id":"rb-7741ebdf-pilot-auto-swe-atlas-qa","benchmark_type":"swe_atlas_qa","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-15T14:09:23.266Z","resumed_from_workflow_id":null,"config_key":"a0616a6e6c3e742b67c28b8fc714ec7b","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.287634,"total_questions":372,"correct_answers":107,"total_cost":7.701978,"generation_time_ms":26799842,"duration_ms":6248734,"input_tokens":103438885,"output_tokens":1828664,"reasoning_tokens":250113,"success":true,"baseline":false},{"id":"310e45ac-fa81-49dd-93d2-6cd0e5e621ff","workflow_id":"rb-7741ebdf-pilot-auto-gpqa","benchmark_type":"gpqa_diamond","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-15T14:00:47.531Z","resumed_from_workflow_id":null,"config_key":"032aa9b706852f04e602639bc76d02c4","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.900673,"total_questions":594,"correct_answers":535,"total_cost":7.072428,"generation_time_ms":32471699,"duration_ms":3245131,"input_tokens":192171,"output_tokens":5413208,"reasoning_tokens":5377855,"success":true,"baseline":false},{"id":"005a50a6-f4d1-4067-b5e4-9354fc1a362c","workflow_id":"rb-7741ebdf-pilot-auto-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-15T14:00:47.340Z","resumed_from_workflow_id":null,"config_key":"9e57d33358f22bc1e824bc85f9520be9","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.137457,"total_questions":291,"correct_answers":40,"total_cost":25.221053,"generation_time_ms":26794842,"duration_ms":3227461,"input_tokens":184872969,"output_tokens":2530498,"reasoning_tokens":2024715,"success":true,"baseline":false},{"id":"d8fbce64-1a8e-4648-827c-962d3986cdad","workflow_id":"rbgq3-1789433899-pilot-gpqa.sakana/fugu-ultra-v2.auto-routing","benchmark_type":"gpqa_diamond","model_permaslug":"sakana/fugu-ultra-v2","provider_name":"auto-routing","timestamp":"2026-09-15T01:07:48.052Z","resumed_from_workflow_id":null,"config_key":"37a4dbebefa2842508adf5f32c99406b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.93266,"total_questions":594,"correct_answers":554,"total_cost":99.506314,"generation_time_ms":38593253,"duration_ms":1970735,"input_tokens":5035457,"output_tokens":2733633,"reasoning_tokens":324526,"success":true,"baseline":false},{"id":"814ad8a9-7b16-45c4-9bb2-e4cfb657b9b8","workflow_id":"rbgq3-1789433899-pilot-gpqa.sakana/fugu-ultra.auto-routing","benchmark_type":"gpqa_diamond","model_permaslug":"sakana/fugu-ultra","provider_name":"auto-routing","timestamp":"2026-09-15T01:07:47.987Z","resumed_from_workflow_id":null,"config_key":"634d38fa293f5999bdf29b2cb3d47c4b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.946128,"total_questions":594,"correct_answers":562,"total_cost":227.450382,"generation_time_ms":65098262,"duration_ms":3830881,"input_tokens":6522402,"output_tokens":6587778,"reasoning_tokens":706903,"success":true,"baseline":false},{"id":"6d78f821-57e3-4378-a725-7c5133979925","workflow_id":"rbgq3-1789433899-pilot-gpqa.openrouter/auto.auto-routing","benchmark_type":"gpqa_diamond","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-09-15T01:07:47.971Z","resumed_from_workflow_id":null,"config_key":"90fa47f67f9993814d325aefb0cf9183","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.907407,"total_questions":594,"correct_answers":539,"total_cost":1.560001,"generation_time_ms":46948387,"duration_ms":4171740,"input_tokens":163266,"output_tokens":4642534,"reasoning_tokens":4601108,"success":true,"baseline":false},{"id":"fc1568ed-851a-44a8-87a2-98139b4ee79f","workflow_id":"rbgq3-1789433899-pilot-gpqa.sakana/fugu-max.auto-routing","benchmark_type":"gpqa_diamond","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-15T01:07:47.957Z","resumed_from_workflow_id":null,"config_key":"7b1f4ced9764072b0c37201de4c6ede4","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0.5,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.885522,"total_questions":594,"correct_answers":526,"total_cost":14.959829,"generation_time_ms":44568206,"duration_ms":4589302,"input_tokens":231931,"output_tokens":2431351,"reasoning_tokens":2685693,"success":true,"baseline":false}],"truncated":false}}