{"data":{"runs":[{"id":"44851e27-8d8c-4651-98d4-8ddbc9260e01","workflow_id":"cx6n-ds-dstb40o-native-8l889","benchmark_type":"terminal-bench-4.0","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-11T03:49:47.168Z","resumed_from_workflow_id":null,"config_key":"18455b40f26894e52d9270ab4e16c622","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.25,"total_questions":12,"correct_answers":3,"total_cost":22.026422,"generation_time_ms":106781364,"duration_ms":30767168,"input_tokens":580356542,"output_tokens":14070336,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"731e7fad-aad4-4668-a967-3b21ef81b4fd","workflow_id":"pi27n-sonnet-sotb40o-native-cmmmj","benchmark_type":"terminal-bench-4.0","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-11T03:48:25.991Z","resumed_from_workflow_id":null,"config_key":"43a947c3e3f31f4de9707a78253def6c","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.3125,"total_questions":16,"correct_answers":5,"total_cost":23.039691,"generation_time_ms":56799577,"duration_ms":30680991,"input_tokens":29231049,"output_tokens":1626684,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"5eb6f1ab-75d1-404f-8b48-0c3460046376","workflow_id":"cc1n-kimi-kitb40o-native-ckj9s","benchmark_type":"terminal-bench-4.0","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-11T03:24:32.536Z","resumed_from_workflow_id":null,"config_key":"7b7abfea705d6a2180c08a451afab09c","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.6875,"total_questions":16,"correct_answers":11,"total_cost":62.353791,"generation_time_ms":70390885,"duration_ms":29254536,"input_tokens":185232764,"output_tokens":1292295,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"e98adf09-d44b-4f3a-b5c5-05ef6eadec7d","workflow_id":"pi12e-ds-dstb40o-explicit-pkvpv","benchmark_type":"terminal-bench-4.0","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-11T02:31:28.044Z","resumed_from_workflow_id":null,"config_key":"b4f6eceee07dc7865fa887bad86a71c9","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.083333,"total_questions":12,"correct_answers":1,"total_cost":4.523966,"generation_time_ms":42410859,"duration_ms":26067044,"input_tokens":237978794,"output_tokens":2158734,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"6371bcef-e9a5-4ba9-8af9-742e1ca0b43c","workflow_id":"cc0e-glm-gltb40o-explicit-ldd56","benchmark_type":"terminal-bench-4.0","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-11T00:20:09.470Z","resumed_from_workflow_id":null,"config_key":"ade90a0a17c92e0ad2e937cd028383c5","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.428571,"total_questions":14,"correct_answers":6,"total_cost":5.442021,"generation_time_ms":72174337,"duration_ms":18191470,"input_tokens":126586487,"output_tokens":2195659,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"3a7d0a46-3267-40d6-90a4-145319b4b34e","workflow_id":"pi24e-luna-lutb40o-explicit-2pjnc","benchmark_type":"terminal-bench-4.0","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-11T00:17:17.712Z","resumed_from_workflow_id":null,"config_key":"7a4b3fb3ea7a422aa2dc3c5d91b582d0","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0,"total_questions":13,"correct_answers":0,"total_cost":0.431219,"generation_time_ms":23553017,"duration_ms":18012712,"input_tokens":17779562,"output_tokens":360672,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"90859ab7-8873-4ac6-a5fb-8e5938167599","workflow_id":"pi20n-kimi-kitb40o-native-fffwl","benchmark_type":"terminal-bench-4.0","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-11T00:17:10.110Z","resumed_from_workflow_id":null,"config_key":"6468749d7c7581aa613acefa5910678e","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.6,"total_questions":15,"correct_answers":9,"total_cost":31.614031,"generation_time_ms":49079139,"duration_ms":18006110,"input_tokens":80376397,"output_tokens":780960,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"60104fc1-3165-4002-bef7-9fe315b35af6","workflow_id":"pi20e-kimi-kitb40o-explicit-nvt99","benchmark_type":"terminal-bench-4.0","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-11T00:16:36.975Z","resumed_from_workflow_id":null,"config_key":"4f6ec3b8c089a63645dee35f644b4d4d","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.625,"total_questions":16,"correct_answers":10,"total_cost":25.206479,"generation_time_ms":48184258,"duration_ms":17972975,"input_tokens":57898728,"output_tokens":581459,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"1ea26817-e120-4188-ab91-ceb65654799e","workflow_id":"pi16e-glm-gltb40o-explicit-g6xvn","benchmark_type":"terminal-bench-4.0","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-11T00:16:29.347Z","resumed_from_workflow_id":null,"config_key":"bd3b7bd712f1049ca7640414f712d6fe","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.428571,"total_questions":14,"correct_answers":6,"total_cost":4.736245,"generation_time_ms":56268945,"duration_ms":17967347,"input_tokens":121471982,"output_tokens":1629310,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"12dbfdc3-c53e-4f32-9343-448ae043ad4f","workflow_id":"cc1e-kimi-kitb40o-explicit-tgfvk","benchmark_type":"terminal-bench-4.0","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-11T00:16:14.486Z","resumed_from_workflow_id":null,"config_key":"e28c4f98f40338b85e0429ba57935a44","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.8125,"total_questions":16,"correct_answers":13,"total_cost":40.486566,"generation_time_ms":51740838,"duration_ms":17956486,"input_tokens":97710736,"output_tokens":1059628,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"462f0c63-4732-465c-a53a-b936443c0231","workflow_id":"cx9e-luna-lutb40o-explicit-cm2lc","benchmark_type":"terminal-bench-4.0","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-11T00:16:05.539Z","resumed_from_workflow_id":null,"config_key":"e3320adc3512a62911bcb32086fa48ae","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.153846,"total_questions":13,"correct_answers":2,"total_cost":1.697631,"generation_time_ms":30807340,"duration_ms":17945539,"input_tokens":89940328,"output_tokens":793847,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"7ec08523-e020-465c-a81b-234774ab2359","workflow_id":"cx9n-luna-lutb40o-native-bzrvn","benchmark_type":"terminal-bench-4.0","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-11T00:14:46.131Z","resumed_from_workflow_id":null,"config_key":"3f5eff06f24fbfb972d53a2245a3c98f","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.083333,"total_questions":12,"correct_answers":1,"total_cost":1.353952,"generation_time_ms":33552415,"duration_ms":17866131,"input_tokens":71901058,"output_tokens":831945,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"b8272d36-83bd-4181-9bfc-c05fd6deda99","workflow_id":"pi24n-luna-lutb40o-native-r6pmb","benchmark_type":"terminal-bench-4.0","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-11T00:14:36.308Z","resumed_from_workflow_id":null,"config_key":"4e1fffaebed2aff1c7c98614952f06fa","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.153846,"total_questions":13,"correct_answers":2,"total_cost":0.489168,"generation_time_ms":28711927,"duration_ms":17851308,"input_tokens":20604582,"output_tokens":398277,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"26408861-0aa2-4f4d-9fb6-7c7ba0128a81","workflow_id":"pi16n-glm-gltb40o-native-lt2x8","benchmark_type":"terminal-bench-4.0","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-11T00:14:20.741Z","resumed_from_workflow_id":null,"config_key":"972962f563dd6afc58aed5a31b390397","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.571429,"total_questions":14,"correct_answers":8,"total_cost":3.93331,"generation_time_ms":55045989,"duration_ms":17837741,"input_tokens":95296595,"output_tokens":1548560,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"0244ad44-383d-4ad1-846d-754f6dea7321","workflow_id":"cc0n-glm-gltb40o-native-m88wb","benchmark_type":"terminal-bench-4.0","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-11T00:12:57.616Z","resumed_from_workflow_id":null,"config_key":"ee588dc2725b47a61c50b86e8128f1d8","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.285714,"total_questions":14,"correct_answers":4,"total_cost":4.525895,"generation_time_ms":64574467,"duration_ms":17759616,"input_tokens":108054547,"output_tokens":1802831,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"ce85c07b-8e2a-40a7-a019-c5639bdcd8c6","workflow_id":"cx7n-luna-ludswehv-native-ns27v","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-10T22:41:06.472Z","resumed_from_workflow_id":null,"config_key":"ef07bb83e6d47c05ea0f10ed3b56b6b5","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.511905,"total_questions":168,"correct_answers":86,"total_cost":15.339875,"generation_time_ms":108804392,"duration_ms":12246472,"input_tokens":988159409,"output_tokens":6212397,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"e1b4e4e6-8528-4a8e-8626-2b3eddecc8a4","workflow_id":"cx7e-luna-ludswehv-explicit-zqwzt","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-10T22:38:25.351Z","resumed_from_workflow_id":null,"config_key":"1810949e79438b79bed0c717616d8337","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.386905,"total_questions":168,"correct_answers":65,"total_cost":14.311066,"generation_time_ms":106478202,"duration_ms":12085351,"input_tokens":915380061,"output_tokens":5773448,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"5ce665e6-7858-42fe-a1a9-2a1d74c648f7","workflow_id":"pi27e-sonnet-sotb40o-explicit-vqpgd","benchmark_type":"terminal-bench-4.0","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-10T22:06:16.481Z","resumed_from_workflow_id":null,"config_key":"6eb5c8238d3fb1cf053e98ab469c0010","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.25,"total_questions":16,"correct_answers":4,"total_cost":24.197731,"generation_time_ms":30575234,"duration_ms":10151481,"input_tokens":32097825,"output_tokens":1643347,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"e3e0712c-3b14-49ce-b657-b42d9d769fb7","workflow_id":"pi18e-kimi-kidsweo-explicit-nbz4s","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-10T20:32:38.923Z","resumed_from_workflow_id":null,"config_key":"0f504b5435c859c16688ea695522fc81","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.6,"total_questions":10,"correct_answers":6,"total_cost":14.267484,"generation_time_ms":14933656,"duration_ms":4535923,"input_tokens":42769408,"output_tokens":369431,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"e8c8203b-2ab4-4188-97fd-ea365ff57e2d","workflow_id":"pi12n-ds-dstb40o-native-cskns","benchmark_type":"terminal-bench-4.0","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-10T20:30:49.306Z","resumed_from_workflow_id":null,"config_key":"46db5600b5d39a93ee2074e97c5a48cf","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.166667,"total_questions":12,"correct_answers":2,"total_cost":3.159276,"generation_time_ms":14923141,"duration_ms":4428306,"input_tokens":118129093,"output_tokens":1775550,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"dcf85eb5-2371-4ea4-88ed-feba7ff0ec71","workflow_id":"pi14e-glm-gldsweo-explicit-2h4sj","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-10T20:25:52.174Z","resumed_from_workflow_id":null,"config_key":"c003277f07c87907cbb0282dea806781","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.846154,"total_questions":13,"correct_answers":11,"total_cost":2.612718,"generation_time_ms":20800089,"duration_ms":4130174,"input_tokens":69906272,"output_tokens":726294,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9e9a58d8-1d05-44da-9f13-84105ef2253d","workflow_id":"cx6e-ds-dstb40o-explicit-mnm64","benchmark_type":"terminal-bench-4.0","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-10T20:25:46.913Z","resumed_from_workflow_id":null,"config_key":"e075152a56943ea7da40508eea4e11c4","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.333333,"total_questions":12,"correct_answers":4,"total_cost":3.286339,"generation_time_ms":14560604,"duration_ms":4127913,"input_tokens":169478131,"output_tokens":1690446,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"6f507e5d-6b32-4f4c-94c1-74161a214c9d","workflow_id":"pi22es-luna-ludsweos-explicit-mqdx7","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-10T20:16:42.123Z","resumed_from_workflow_id":null,"config_key":"87173f54b42f3b12fb58d1a64efb9b7b","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":1,"total_questions":1,"correct_answers":1,"total_cost":0.198124,"generation_time_ms":2248516,"duration_ms":2558123,"input_tokens":15071523,"output_tokens":53568,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"ec7f2e4d-d5f2-4cd0-8d24-cb701170591b","workflow_id":"pi14n-glm-gldsweo-native-75vfm","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-10T20:14:15.588Z","resumed_from_workflow_id":null,"config_key":"90050a5e0a58ebdfd94a8099259d35a4","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.538462,"total_questions":13,"correct_answers":7,"total_cost":2.139478,"generation_time_ms":16517417,"duration_ms":3433588,"input_tokens":55435895,"output_tokens":681726,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9a431b98-43ba-44f9-95de-488ccaebe751","workflow_id":"pi10e-ds-dsdsweo-explicit-jrchp","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-10T20:13:09.745Z","resumed_from_workflow_id":null,"config_key":"01ab5162ba9ecdca44762a58c294b2ca","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.636364,"total_questions":11,"correct_answers":7,"total_cost":2.333705,"generation_time_ms":10294313,"duration_ms":3368745,"input_tokens":143377386,"output_tokens":934589,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"ee843250-d239-4c0d-8aee-979ec2030329","workflow_id":"cx7ns-luna-ludswehvs-native-xfnxm","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-10T20:10:13.877Z","resumed_from_workflow_id":null,"config_key":"4c8cd4fddf43549a6fdd24db93a5cdd8","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.333333,"total_questions":3,"correct_answers":1,"total_cost":0.447202,"generation_time_ms":2855436,"duration_ms":2169877,"input_tokens":34142605,"output_tokens":104010,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"f3dc4bbe-8c98-45b3-bb30-362e027633c1","workflow_id":"pi22n-luna-ludsweo-native-59g98","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-10T20:09:48.508Z","resumed_from_workflow_id":null,"config_key":"9b0185d5873260c7de0b8ff1262fdef7","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.666667,"total_questions":21,"correct_answers":14,"total_cost":1.270081,"generation_time_ms":12849347,"duration_ms":3164508,"input_tokens":77355198,"output_tokens":618778,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"4400d9b7-b0e9-41cf-8c43-c552b1ccc5c4","workflow_id":"pi18n-kimi-kidsweo-native-m9ggk","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-10T20:09:40.221Z","resumed_from_workflow_id":null,"config_key":"14d2d827d96e2d6e6a2c25c92ac197f8","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.8,"total_questions":10,"correct_answers":8,"total_cost":13.162338,"generation_time_ms":14729900,"duration_ms":3157221,"input_tokens":38884982,"output_tokens":359745,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"e400aeef-74ae-4862-9d69-5d93ca56b162","workflow_id":"cx7es-luna-ludswehvs-explicit-4kjvm","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-10T20:05:15.548Z","resumed_from_workflow_id":null,"config_key":"ec0b163b6d1b7c605e2ebeee20babf37","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.666667,"total_questions":3,"correct_answers":2,"total_cost":0.631732,"generation_time_ms":4147268,"duration_ms":1872548,"input_tokens":48069237,"output_tokens":162030,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"eeb87406-fc95-4c53-83da-48d15212b603","workflow_id":"pi22e-luna-ludsweo-explicit-4jvdz","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-10T20:02:42.834Z","resumed_from_workflow_id":null,"config_key":"daf54b36aa8d666948c8ae1feea196d1","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.636364,"total_questions":22,"correct_answers":14,"total_cost":1.245025,"generation_time_ms":12876885,"duration_ms":2738834,"input_tokens":75790167,"output_tokens":599966,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"67218dc8-2383-4ae4-8ed7-ab612d639052","workflow_id":"pi25n-sonnet-sodsweo-native-jhgfz","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-10T20:00:14.365Z","resumed_from_workflow_id":null,"config_key":"ebbd0c32d09e266278527f800f5b3161","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.777778,"total_questions":18,"correct_answers":14,"total_cost":7.110246,"generation_time_ms":6450579,"duration_ms":2589365,"input_tokens":6728710,"output_tokens":512904,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"f8865623-d7bf-4e88-a7db-1cc009e6c5eb","workflow_id":"pi22ns-luna-ludsweos-native-g5j9g","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-10T19:59:45.443Z","resumed_from_workflow_id":null,"config_key":"e0eeb985abb033804e08fd634bee6357","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0,"total_questions":1,"correct_answers":0,"total_cost":0.083854,"generation_time_ms":1190296,"duration_ms":1541443,"input_tokens":5775318,"output_tokens":25036,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"df8a43d0-c318-4ea6-94e9-9d103b78ed82","workflow_id":"pi10n-ds-dsdsweo-native-xhwsn","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-10T19:55:55.928Z","resumed_from_workflow_id":null,"config_key":"075f654c9343204f6af6dbccc91c10c3","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.454545,"total_questions":11,"correct_answers":5,"total_cost":1.899751,"generation_time_ms":8878100,"duration_ms":2334928,"input_tokens":111317721,"output_tokens":830935,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"2ddff710-1acd-47c0-b390-b40eb09b848b","workflow_id":"pi25e-sonnet-sodsweo-explicit-mf8b6","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-10T19:54:33.562Z","resumed_from_workflow_id":null,"config_key":"2e1f0d67324be123b045f06adda72520","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.777778,"total_questions":18,"correct_answers":14,"total_cost":5.735929,"generation_time_ms":5655551,"duration_ms":2248562,"input_tokens":5621711,"output_tokens":389107,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"91d0e16a-654d-41e9-9540-ca07589d0266","workflow_id":"rr1-pi-sonnet-tb40-old-native-hdkll","benchmark_type":"terminal-bench-4.0","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-09T13:54:26.131Z","resumed_from_workflow_id":null,"config_key":"32bba035b2ac5bbb12e88a6175da3f3d","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.333333,"total_questions":12,"correct_answers":4,"total_cost":24.230829,"generation_time_ms":75584714,"duration_ms":31786131,"input_tokens":28885210,"output_tokens":1640510,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"d8464641-d70d-4db5-b09a-933486b69c3b","workflow_id":"rr1-pi-luna-tb40-old-universal-5hg7q","benchmark_type":"terminal-bench-4.0","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-09T13:13:27.095Z","resumed_from_workflow_id":null,"config_key":"0b94e0957394fbdbd7c94ea8bf45ff99","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.090909,"total_questions":11,"correct_answers":1,"total_cost":0.224504,"generation_time_ms":51102866,"duration_ms":29328095,"input_tokens":5837871,"output_tokens":231686,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"ed573a0a-7a2c-4bdb-b7f0-962ab6ac7a73","workflow_id":"rr1-cc-kimi-tb40-old-universal-c64ms","benchmark_type":"terminal-bench-4.0","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-09T12:25:03.135Z","resumed_from_workflow_id":null,"config_key":"c5362b0cdea2a59239b14c6166deaa25","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.454545,"total_questions":11,"correct_answers":5,"total_cost":79.601871,"generation_time_ms":44279787,"duration_ms":26430135,"input_tokens":261437113,"output_tokens":1504782,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"4af85125-928a-4553-8d40-ca1cba0d04a0","workflow_id":"rr1-cc-glm-tb40-old-universal-mss26","benchmark_type":"terminal-bench-4.0","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T10:44:21.157Z","resumed_from_workflow_id":null,"config_key":"202dd4c5af5f950e713a1f9239f7791a","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.1,"total_questions":10,"correct_answers":1,"total_cost":17.592429,"generation_time_ms":49132710,"duration_ms":20388157,"input_tokens":558442088,"output_tokens":3518559,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"4a9389b6-1d91-4117-a57e-c9ad6e11c7f8","workflow_id":"rr1-cx-ds-tb40-old-native-v8l7x","benchmark_type":"terminal-bench-4.0","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T10:35:48.086Z","resumed_from_workflow_id":null,"config_key":"5a59ffcbba8e055fdd4faa223f218f50","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.181818,"total_questions":11,"correct_answers":2,"total_cost":12.695797,"generation_time_ms":80287337,"duration_ms":19874086,"input_tokens":300381312,"output_tokens":9471352,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"4dc05481-e32e-4cd3-b117-1f9b5c8c350e","workflow_id":"rr1-pi-glm-tb40-old-universal-6h6fz","benchmark_type":"terminal-bench-4.0","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T10:06:01.218Z","resumed_from_workflow_id":null,"config_key":"f1162924e1c16eae8da46ef9088477da","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.111111,"total_questions":9,"correct_answers":1,"total_cost":1.92013,"generation_time_ms":35800156,"duration_ms":18084218,"input_tokens":53110732,"output_tokens":804967,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"2cf1fc1f-8f83-467c-a0d6-816e3fd8f51d","workflow_id":"rr1-pi-kimi-tb40-old-native-ffljm","benchmark_type":"terminal-bench-4.0","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-09T10:05:06.980Z","resumed_from_workflow_id":null,"config_key":"aaab9b9e4082b981bc4d0c918be00920","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.75,"total_questions":12,"correct_answers":9,"total_cost":18.464693,"generation_time_ms":31064918,"duration_ms":18028980,"input_tokens":48962224,"output_tokens":501953,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"9d3707e0-7c95-4b9f-a63d-9321d00c3ae8","workflow_id":"rr1-pi-luna-tb40-old-native-wcl49","benchmark_type":"terminal-bench-4.0","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-09T10:05:02.221Z","resumed_from_workflow_id":null,"config_key":"7a891b67c937d99c993815884e0ff1af","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0,"total_questions":8,"correct_answers":0,"total_cost":0.50964,"generation_time_ms":31789585,"duration_ms":18023221,"input_tokens":26647981,"output_tokens":328323,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"5e4f210f-6c25-4d7e-9968-83968880d641","workflow_id":"rr1-cc-glm-tb40-old-native-rg2m9","benchmark_type":"terminal-bench-4.0","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T10:03:55.752Z","resumed_from_workflow_id":null,"config_key":"8b7463f2a5a907fc8ad092938134a593","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.2,"total_questions":10,"correct_answers":2,"total_cost":9.719226,"generation_time_ms":44157669,"duration_ms":17963752,"input_tokens":256566873,"output_tokens":2739964,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"f53eeaeb-88ba-4dc4-bad5-6adef6feabbf","workflow_id":"rr1-pi-kimi-tb40-old-universal-snhr2","benchmark_type":"terminal-bench-4.0","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-09T10:00:28.506Z","resumed_from_workflow_id":null,"config_key":"0129da787882f50a289b79a9685f42ce","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.4,"total_questions":10,"correct_answers":4,"total_cost":24.842067,"generation_time_ms":31039863,"duration_ms":17750506,"input_tokens":64330431,"output_tokens":572792,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"5012074b-75ce-41ac-863d-21e5f9f934ac","workflow_id":"rr1-cx-luna-dswe-hv-universal-rzsv9","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-09T09:18:16.648Z","resumed_from_workflow_id":null,"config_key":"6b580d56f6543625f5e231f65401718c","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.509091,"total_questions":165,"correct_answers":84,"total_cost":15.115201,"generation_time_ms":103285517,"duration_ms":15222648,"input_tokens":977581727,"output_tokens":5990008,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"43ea4c1e-f082-4aa1-b45d-4045d6cea872","workflow_id":"rr1-cx-luna-dswe-hv-native-ps4c9","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-09T08:52:58.475Z","resumed_from_workflow_id":null,"config_key":"489b3629768951ca4e298cbbcc1f12d5","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.466667,"total_questions":165,"correct_answers":77,"total_cost":14.693835,"generation_time_ms":97653046,"duration_ms":13704475,"input_tokens":935785641,"output_tokens":5971619,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"e9b02ea1-b448-40fb-82fd-2031930a6f06","workflow_id":"rr1-cc-kimi-tb40-old-native-682fw","benchmark_type":"terminal-bench-4.0","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-09T08:37:55.768Z","resumed_from_workflow_id":null,"config_key":"8c4d116a6659bedbb604ec632e795f30","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.5,"total_questions":12,"correct_answers":6,"total_cost":33.787574,"generation_time_ms":24437646,"duration_ms":12802768,"input_tokens":81964390,"output_tokens":755719,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"b5304bd9-4295-4527-ae24-0d4b471f95ca","workflow_id":"rr1-pi-sonnet-tb40-old-universal-mc2lt","benchmark_type":"terminal-bench-4.0","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-09T08:30:12.695Z","resumed_from_workflow_id":null,"config_key":"5e4d048cb39074c906e197f9a1f2229f","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.4,"total_questions":10,"correct_answers":4,"total_cost":12.519653,"generation_time_ms":24069885,"duration_ms":12332695,"input_tokens":15276632,"output_tokens":802883,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"5f17dc3d-55b7-48e6-bda7-9abefba1610e","workflow_id":"rr1-pi-ds-tb40-old-native-k59mz","benchmark_type":"terminal-bench-4.0","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T08:26:55.456Z","resumed_from_workflow_id":null,"config_key":"12f47412bb72456a2b35fac90b9644c8","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.272727,"total_questions":11,"correct_answers":3,"total_cost":3.5071,"generation_time_ms":34096762,"duration_ms":12139456,"input_tokens":150402749,"output_tokens":1816581,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"66d16442-d51f-4f48-9a82-f869e7747dde","workflow_id":"rr1-cx-ds-tb40-old-universal-r7hk8","benchmark_type":"terminal-bench-4.0","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T08:26:38.978Z","resumed_from_workflow_id":null,"config_key":"caa48e46a98805c24d496e5e8aac3cfd","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.4,"total_questions":10,"correct_answers":4,"total_cost":3.662621,"generation_time_ms":27315317,"duration_ms":12124978,"input_tokens":246503066,"output_tokens":1845110,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"de8c7526-3742-475c-80d4-a8146ebe1fe9","workflow_id":"rr1-pi-sonnet-dswe-old-native-mwjp8","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-09T08:19:37.515Z","resumed_from_workflow_id":null,"config_key":"9a34cbd22ac55d5a9cbdf3ef3e451ab2","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.461538,"total_questions":13,"correct_answers":6,"total_cost":6.855397,"generation_time_ms":17010596,"duration_ms":11698515,"input_tokens":5561350,"output_tokens":521003,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"4463369c-04cc-4aa4-84ef-c8ba8c290b5d","workflow_id":"rr1-pi-sonnet-dswe-old-universal-7h26n","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-09T08:18:14.524Z","resumed_from_workflow_id":null,"config_key":"e686448b9166b7d5d9f8d3b2627a56a1","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.666667,"total_questions":15,"correct_answers":10,"total_cost":3.841175,"generation_time_ms":14198538,"duration_ms":11615524,"input_tokens":3556215,"output_tokens":257140,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"4749ee30-53dd-4626-b41e-060fe1f7fe1a","workflow_id":"rr1-pi-ds-tb40-old-universal-5gjhr","benchmark_type":"terminal-bench-4.0","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T07:56:14.923Z","resumed_from_workflow_id":null,"config_key":"90eb9cef979d55d684bafa9d3a8488ec","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.25,"total_questions":8,"correct_answers":2,"total_cost":6.17876,"generation_time_ms":18023186,"duration_ms":10298923,"input_tokens":247571497,"output_tokens":2250416,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"5a786290-ee64-4931-b503-ef674a93861e","workflow_id":"rr1-pi-glm-tb40-old-native-9mg88","benchmark_type":"terminal-bench-4.0","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T07:30:44.813Z","resumed_from_workflow_id":null,"config_key":"761595fb489df5642dffcff67c402448","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.166667,"total_questions":6,"correct_answers":1,"total_cost":2.182973,"generation_time_ms":13606218,"duration_ms":8767813,"input_tokens":111319172,"output_tokens":981529,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"bc49866e-34b7-491e-ab62-88744ce811bc","workflow_id":"rr1-pi-glm-dswe-old-universal-5qnxh","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T06:05:30.655Z","resumed_from_workflow_id":null,"config_key":"ab45368b14fcf2d73107f56c2d458e74","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.545455,"total_questions":11,"correct_answers":6,"total_cost":2.65414,"generation_time_ms":12073682,"duration_ms":3654655,"input_tokens":85889640,"output_tokens":795769,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"85edfa81-c544-42cc-a762-95d274054f14","workflow_id":"rr1-pi-ds-dswe-old-universal-lwzqf","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T06:03:45.473Z","resumed_from_workflow_id":null,"config_key":"8dbd9a2e0fa40f2822603ace96bde3e3","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.8,"total_questions":10,"correct_answers":8,"total_cost":0.959263,"generation_time_ms":9850998,"duration_ms":3550473,"input_tokens":111497669,"output_tokens":796515,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"eff37bda-f9ce-46e6-a47f-3b936079fd30","workflow_id":"rr1-pi-glm-dswe-old-native-6w5rf","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T05:57:01.819Z","resumed_from_workflow_id":null,"config_key":"49823eb148b3f6ce5ce7cb0360c04790","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.636364,"total_questions":11,"correct_answers":7,"total_cost":2.175385,"generation_time_ms":11337530,"duration_ms":3145819,"input_tokens":69833814,"output_tokens":845806,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"44961b34-5054-4427-95f8-e11bd813c471","workflow_id":"rr1-pi-kimi-dswe-old-universal-4r2w6","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-09T05:54:28.923Z","resumed_from_workflow_id":null,"config_key":"9ab1f1b95a55199ff4dfe7ea466ca5b5","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.6,"total_questions":10,"correct_answers":6,"total_cost":12.652143,"generation_time_ms":7867523,"duration_ms":2990923,"input_tokens":52370656,"output_tokens":384356,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"5c31e5f2-5870-4211-9227-93061f593e58","workflow_id":"rr1-cx-luna-dswe-hv-universal-s-rvxkg","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-09T05:52:34.382Z","resumed_from_workflow_id":null,"config_key":"aed2d66ce95247c528d2fd4afcfedf16","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.5,"total_questions":6,"correct_answers":3,"total_cost":0.947723,"generation_time_ms":6405448,"duration_ms":2056382,"input_tokens":70155044,"output_tokens":258716,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"bc8594d3-5d25-4268-bb1d-2123caa5430c","workflow_id":"rr1-pi-kimi-dswe-old-native-9bqkm","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-09T05:51:38.204Z","resumed_from_workflow_id":null,"config_key":"a423e298de4c0a5fd7dbc04a501a27db","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.555556,"total_questions":9,"correct_answers":5,"total_cost":10.030492,"generation_time_ms":8487825,"duration_ms":2821204,"input_tokens":40080586,"output_tokens":329809,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"a0eff56a-e369-4628-a2a8-f2886ee9e72d","workflow_id":"rr1-pi-luna-dswe-old-universal-s-9kltq","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-09T05:50:39.068Z","resumed_from_workflow_id":null,"config_key":"ead9cd8b80bd7e224ba6d2fcd42743ae","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":1,"total_questions":1,"correct_answers":1,"total_cost":0.151601,"generation_time_ms":1702791,"duration_ms":1940068,"input_tokens":11083444,"output_tokens":41754,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"c12b994d-a600-4c4b-b965-27bd1a7f5c1a","workflow_id":"rr1-cx-luna-dswe-hv-native-s-gbw5h","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-09T05:48:39.521Z","resumed_from_workflow_id":null,"config_key":"028cecc5e1f5139ce6129c9def44490a","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.166667,"total_questions":6,"correct_answers":1,"total_cost":1.067665,"generation_time_ms":6438224,"duration_ms":1821521,"input_tokens":79996025,"output_tokens":287011,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"0cc7e14a-873b-404e-aaf1-b2fd31f3516e","workflow_id":"rr1-pi-ds-dswe-old-native-lkrgr","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-09T05:47:36.497Z","resumed_from_workflow_id":null,"config_key":"3c567e1c1746a84101686a74e287b976","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.454545,"total_questions":11,"correct_answers":5,"total_cost":1.377929,"generation_time_ms":11260363,"duration_ms":2581497,"input_tokens":137136713,"output_tokens":1144611,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"750180e6-47ce-40aa-8585-16c87b58545e","workflow_id":"rr1-cx-luna-tb40-old-universal-vts8v","benchmark_type":"terminal-bench-4.0","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-09T05:46:07.155Z","resumed_from_workflow_id":null,"config_key":"c033895cbd83879b56501b927ff0c2a8","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0,"total_questions":11,"correct_answers":0,"total_cost":0.604626,"generation_time_ms":7125514,"duration_ms":2492155,"input_tokens":22279085,"output_tokens":460012,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"0dabf15a-4670-44e3-a181-45ad85f97cbe","workflow_id":"rr1-cx-luna-tb40-old-native-t7dx6","benchmark_type":"terminal-bench-4.0","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-09T05:45:56.283Z","resumed_from_workflow_id":null,"config_key":"aca596c9c5e0b41be97908deca0b42af","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0,"total_questions":11,"correct_answers":0,"total_cost":0.538179,"generation_time_ms":6650702,"duration_ms":2481283,"input_tokens":16260467,"output_tokens":479561,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"34d08226-e3cd-432e-b8b1-2c6941e4d128","workflow_id":"rr1-pi-luna-dswe-old-native-rnd5g","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-09T05:42:02.676Z","resumed_from_workflow_id":null,"config_key":"83e0150f652b44e874d4617d7eace60b","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.538462,"total_questions":13,"correct_answers":7,"total_cost":0.803543,"generation_time_ms":8104466,"duration_ms":2244676,"input_tokens":50516715,"output_tokens":373249,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"4f869927-32f2-4198-86dd-47b8e1039073","workflow_id":"rr1-pi-luna-dswe-old-universal-hj66b","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-09T05:39:02.985Z","resumed_from_workflow_id":null,"config_key":"4f5fd1280d27f012a2d925248dccf221","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.333333,"total_questions":12,"correct_answers":4,"total_cost":0.69128,"generation_time_ms":7092048,"duration_ms":2063985,"input_tokens":42534855,"output_tokens":319494,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"bcdb7cfe-4f6e-452a-a311-c7410086d886","workflow_id":"cx-val-glm-dswe-native-ql4rv","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T19:08:20.144Z","resumed_from_workflow_id":null,"config_key":"08e52b254be341e6d36bb2406805f61e","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.017699,"total_questions":113,"correct_answers":2,"total_cost":0,"generation_time_ms":715153591,"duration_ms":79239144,"input_tokens":50130484373,"output_tokens":32806987,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9b35102b-d5a0-4ca9-a60a-e120b4164f05","workflow_id":"cx-val-ds-dswe-native-dpk5m","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T09:29:21.495Z","resumed_from_workflow_id":null,"config_key":"c52ee36ca251170173dcd59d9f0448cd","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.619469,"total_questions":113,"correct_answers":70,"total_cost":0,"generation_time_ms":373399308,"duration_ms":44501495,"input_tokens":2705217644,"output_tokens":65455837,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"f2c22b42-5b7c-4458-93a6-298daed41fca","workflow_id":"smoke2-cx-glm-dswe-native-hskjm","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T07:02:13.656Z","resumed_from_workflow_id":null,"config_key":"4308c6a3dbb40974d17d719686a3bb57","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":1,"epochs":1,"recorded_epochs":1,"accuracy":0,"total_questions":1,"correct_answers":0,"total_cost":10.177473,"generation_time_ms":6686946,"duration_ms":6949656,"input_tokens":329811404,"output_tokens":120357,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"f6b4a040-4944-4a63-aa86-c05995ed203b","workflow_id":"smoke3g-glm-dswe-universal-ghpjh","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T05:43:24.586Z","resumed_from_workflow_id":null,"config_key":"4a60a9674a3118a2808f16a71a064394","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":1,"epochs":1,"recorded_epochs":1,"accuracy":0,"total_questions":1,"correct_answers":0,"total_cost":0.053784,"generation_time_ms":469525,"duration_ms":705586,"input_tokens":1371971,"output_tokens":39499,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"0b579aff-b744-4ea8-8e88-2f55d6cd06e2","workflow_id":"smoke3k-kimi-dswe-native-6lndw","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-08T05:43:24.529Z","resumed_from_workflow_id":null,"config_key":"9d2484221a37601135569f3b6f39b9db","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":1,"epochs":1,"recorded_epochs":1,"accuracy":1,"total_questions":1,"correct_answers":1,"total_cost":1.430163,"generation_time_ms":371133,"duration_ms":705529,"input_tokens":5801829,"output_tokens":36637,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"cab19b76-07ae-4a99-a7d0-6ff5bf77d6a7","workflow_id":"cx-glm-dswes-native-9tb5r","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T05:42:42.215Z","resumed_from_workflow_id":null,"config_key":"052bf3d975b8293990cba4f91f356956","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0,"total_questions":10,"correct_answers":0,"total_cost":0,"generation_time_ms":56743820,"duration_ms":11722215,"input_tokens":3672674426,"output_tokens":2613211,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"fc6d19a5-f83b-477c-8254-ad23f1ea2ccf","workflow_id":"cc-glm-dswes-native-jsxrr","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T05:14:48.659Z","resumed_from_workflow_id":null,"config_key":"2ac1568809a29b2f4d0df71417cddbb5","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.2,"total_questions":10,"correct_answers":2,"total_cost":3.714002,"generation_time_ms":17409837,"duration_ms":10050659,"input_tokens":89475583,"output_tokens":739364,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"e161c50e-6c75-4c99-aef4-515846c4870d","workflow_id":"cx-ds-dswes-native-bzcn2","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T05:06:02.445Z","resumed_from_workflow_id":null,"config_key":"51092d2ab9145ad9b5d2517c664e9187","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.6,"total_questions":10,"correct_answers":6,"total_cost":20.257805,"generation_time_ms":40306741,"duration_ms":9523445,"input_tokens":283371694,"output_tokens":6986223,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"ce8eb4b5-1e75-4aaf-8bdf-a872a5ac8589","workflow_id":"cx-val-glm-dswe-universal-8hlnb","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T03:35:49.791Z","resumed_from_workflow_id":null,"config_key":"b11aa786a81cd0eec91902bb70a09bd0","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.353982,"total_questions":113,"correct_answers":40,"total_cost":0,"generation_time_ms":147554162,"duration_ms":23288791,"input_tokens":2601722385,"output_tokens":11074139,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"8c170ca6-15c2-4bc4-8823-ebfbc42d5c57","workflow_id":"cx-ds-dswes-universal-dhqtf","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T03:28:53.221Z","resumed_from_workflow_id":null,"config_key":"fb12e4e28fb03b9f02b64fb9edac7294","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.6,"total_questions":10,"correct_answers":6,"total_cost":1.608549,"generation_time_ms":11012200,"duration_ms":3694221,"input_tokens":127085188,"output_tokens":1108957,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"181632ba-b7f7-4a98-8438-f75b609241a7","workflow_id":"cc-glm-dswes-universal-5q4td","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T03:18:47.284Z","resumed_from_workflow_id":null,"config_key":"2de6fc49c43934b0cb482f035dab7296","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.4,"total_questions":10,"correct_answers":4,"total_cost":3.126915,"generation_time_ms":9787187,"duration_ms":3089284,"input_tokens":84156334,"output_tokens":637134,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"50a6447a-5438-471f-8a5d-ea0351e8ca33","workflow_id":"cc-ds-dswes-native-5f5cj","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T03:12:03.994Z","resumed_from_workflow_id":null,"config_key":"12894da0f3812e5ddfdec33e6bb9898e","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.5,"total_questions":10,"correct_answers":5,"total_cost":1.893972,"generation_time_ms":6782005,"duration_ms":2686994,"input_tokens":101028373,"output_tokens":957396,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"781d78d3-8eec-4a45-bc51-13dbb79ebf59","workflow_id":"cc-ds-dswes-universal-9xx89","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T03:01:29.136Z","resumed_from_workflow_id":null,"config_key":"fefcf67d3a81e2a506a5bd227aeb5288","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.4,"total_questions":10,"correct_answers":4,"total_cost":2.471671,"generation_time_ms":7656009,"duration_ms":2052136,"input_tokens":100347395,"output_tokens":836119,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"1418285b-61a7-4f58-a27e-f002fa50e70c","workflow_id":"cx-glm-dswes-universal-srr7s","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T03:01:18.709Z","resumed_from_workflow_id":null,"config_key":"cc54ef7055ab19979b8aa15adf749911","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.3,"total_questions":10,"correct_answers":3,"total_cost":2.462367,"generation_time_ms":5998007,"duration_ms":2038709,"input_tokens":60339550,"output_tokens":539050,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"36c09db2-0ebb-465d-9b8b-9a23d9d10176","workflow_id":"cx-luna-dswes-native-ntfvt","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-08T02:56:14.930Z","resumed_from_workflow_id":null,"config_key":"91454480b5ef0f0a9f4e6f35006692c9","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.4,"total_questions":10,"correct_answers":4,"total_cost":0.908528,"generation_time_ms":5115436,"duration_ms":1734930,"input_tokens":53691747,"output_tokens":388610,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"a5b75cf9-a339-466e-ab8d-64bb78ee0aeb","workflow_id":"cc-sonnet-dswes-universal-thlhq","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-08T02:49:21.086Z","resumed_from_workflow_id":null,"config_key":"ce65b174bc39e99bd406993a3db58683","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.4,"total_questions":10,"correct_answers":4,"total_cost":3.634996,"generation_time_ms":2081080,"duration_ms":1322086,"input_tokens":7455512,"output_tokens":183818,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"f6625b28-dc56-4583-9fe8-b92ddc98fbf6","workflow_id":"cx-luna-dswes-universal-rkm5k","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-08T02:48:58.010Z","resumed_from_workflow_id":null,"config_key":"9ba07f2d5589a46280c940da23f5fb43","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.2,"total_questions":10,"correct_answers":2,"total_cost":0.640956,"generation_time_ms":4267195,"duration_ms":1298010,"input_tokens":36055120,"output_tokens":311715,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"ab3f591d-c7c9-4915-8e33-1cc0f5be7df7","workflow_id":"cc-sonnet-dswes-native-8742f","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-08T02:48:31.350Z","resumed_from_workflow_id":null,"config_key":"074aa2be7ee0557764e1973a546338b9","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.4,"total_questions":10,"correct_answers":4,"total_cost":3.595095,"generation_time_ms":2103385,"duration_ms":1273350,"input_tokens":7846935,"output_tokens":180852,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"7fbef958-b32b-420b-b057-2341352cf54c","workflow_id":"ccab4-glm-dswe-glm-dswe60-universal-next-ng9tn","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T02:26:10.676Z","resumed_from_workflow_id":null,"config_key":"9b447fdedb974e41efc8044142206a10","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.627119,"total_questions":59,"correct_answers":37,"total_cost":22.139395,"generation_time_ms":90879695,"duration_ms":17195676,"input_tokens":554726017,"output_tokens":6420162,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"4acd931a-c2a9-4bb6-8dcf-3ae40e2c203e","workflow_id":"cx-val-kimi-dswe-native-xf52q","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-08T02:00:36.666Z","resumed_from_workflow_id":null,"config_key":"f8b942d54b0dd3ff35afccd67084155b","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.663717,"total_questions":113,"correct_answers":75,"total_cost":431.937574,"generation_time_ms":127334411,"duration_ms":17575666,"input_tokens":1365956379,"output_tokens":6775461,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"5aa33844-338c-472c-9ff8-fc412be00039","workflow_id":"ccab4-glm-dswe-glm-dswe60-native-94qs6","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T00:59:15.858Z","resumed_from_workflow_id":null,"config_key":"eb41b4816ffc6199d40ab75e3e5427bb","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.508475,"total_questions":59,"correct_answers":30,"total_cost":20.981975,"generation_time_ms":68008678,"duration_ms":11980858,"input_tokens":539989443,"output_tokens":5981272,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"b58621a3-5924-438d-926a-eba63de622b2","workflow_id":"cx-val-ds-dswe-universal-267bq","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T00:58:28.311Z","resumed_from_workflow_id":null,"config_key":"13bb1317b630dbcebf561ce1049f50ed","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.619469,"total_questions":113,"correct_answers":70,"total_cost":24.486884,"generation_time_ms":101651231,"duration_ms":13848311,"input_tokens":1437369080,"output_tokens":11365047,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"0577891b-1158-464f-9432-ec638bf6467d","workflow_id":"cx-val-kimi-dswe-universal-5lfmx","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-08T00:35:55.232Z","resumed_from_workflow_id":null,"config_key":"4a4410c8f35edfd122be1421aac5dd86","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.79646,"total_questions":113,"correct_answers":90,"total_cost":236.163519,"generation_time_ms":88773594,"duration_ms":12494232,"input_tokens":693818238,"output_tokens":5771531,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"306b0fec-e6a8-466d-b3a0-b735d23ef870","workflow_id":"cxfix-glm-glm-dswe30-native-4ltxv","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-08T00:21:33.768Z","resumed_from_workflow_id":null,"config_key":"1c34dad2655c8d9da2a50fb9c887b426","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":30,"epochs":1,"recorded_epochs":1,"accuracy":0,"total_questions":30,"correct_answers":0,"total_cost":0,"generation_time_ms":180247601,"duration_ms":29250768,"input_tokens":13753875023,"output_tokens":7878407,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"c44274a1-53e7-424a-a032-183be454d19e","workflow_id":"cx-val-luna-luna-dswe-cm0-ft75m","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-08T00:07:01.583Z","resumed_from_workflow_id":null,"config_key":"2528b0b9d927badd9333126140630a6f","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.380531,"total_questions":113,"correct_answers":43,"total_cost":11.186998,"generation_time_ms":67955773,"duration_ms":10759583,"input_tokens":745688388,"output_tokens":4146963,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"3e47b239-bf4a-4709-873d-8770bd35e388","workflow_id":"cx-val-luna-luna-dswe-cm1-2sqkr","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-08T00:04:44.581Z","resumed_from_workflow_id":null,"config_key":"bc699987eaadb182d2e9fe9694812f4a","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.40708,"total_questions":113,"correct_answers":46,"total_cost":9.969399,"generation_time_ms":70343394,"duration_ms":10622581,"input_tokens":637969428,"output_tokens":4240529,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"6779235c-0944-4361-a288-7c0ba5ca15cf","workflow_id":"cx-search-glm-dswe-native-pvdzm","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-07T21:01:56.657Z","resumed_from_workflow_id":null,"config_key":"1aa2ae21ae0e47bead742706ec4cda9f","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.044248,"total_questions":113,"correct_answers":5,"total_cost":0,"generation_time_ms":646908456,"duration_ms":80694657,"input_tokens":49631709796,"output_tokens":22464154,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"d80d7938-63fd-413b-8d96-a7574d8d8e1f","workflow_id":"kd-pi-kimi-dswe10-native-v4nq4","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-07T19:41:11.725Z","resumed_from_workflow_id":null,"config_key":"97fb4c69f78c469ea3e9458903956241","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":10,"epochs":1,"recorded_epochs":1,"accuracy":0.8,"total_questions":10,"correct_answers":8,"total_cost":10.823668,"generation_time_ms":6974166,"duration_ms":2424725,"input_tokens":1808996,"output_tokens":302144,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"06eb824a-1761-471c-bdac-2dabf5470bc9","workflow_id":"cxfix-glm-glm-dswe30-universal-dpgmk","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-07T19:34:09.726Z","resumed_from_workflow_id":null,"config_key":"ce27c260b47af7b3e29d4d174db9e89d","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":30,"epochs":1,"recorded_epochs":1,"accuracy":0.433333,"total_questions":30,"correct_answers":13,"total_cost":12.010601,"generation_time_ms":28210957,"duration_ms":12005726,"input_tokens":266197344,"output_tokens":2000970,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"368dfdec-d173-40a7-8377-dc5da627343a","workflow_id":"kd-cc-kimi-dswe10-native-s5mx9","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-07T19:31:03.195Z","resumed_from_workflow_id":null,"config_key":"c845720db789fc2adc3ae09052565a93","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":10,"epochs":1,"recorded_epochs":1,"accuracy":0.7,"total_questions":10,"correct_answers":7,"total_cost":25.776445,"generation_time_ms":7115866,"duration_ms":1817195,"input_tokens":64420265,"output_tokens":473325,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"65bc1512-9732-4b3e-a1e6-7baad350ac3f","workflow_id":"cxfix-luna-cm1-luna-dswe30-universal-trn94","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-07T17:34:16.217Z","resumed_from_workflow_id":null,"config_key":"113a58f3db864afaf2cfd50f1f788c42","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":30,"epochs":1,"recorded_epochs":1,"accuracy":0.366667,"total_questions":30,"correct_answers":11,"total_cost":2.387066,"generation_time_ms":18012554,"duration_ms":4812217,"input_tokens":158942723,"output_tokens":931073,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"81f5c787-e9aa-4e6b-8a80-eb150ce5aebc","workflow_id":"cxfix-luna-cm1-luna-dswe30-native-wjc2m","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-07T17:32:47.831Z","resumed_from_workflow_id":null,"config_key":"29d9609b212d49cd775f99d374bdebee","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":30,"epochs":1,"recorded_epochs":1,"accuracy":0.333333,"total_questions":30,"correct_answers":10,"total_cost":2.74894,"generation_time_ms":20683407,"duration_ms":4723831,"input_tokens":171654765,"output_tokens":1237387,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"4f1e18a2-1de1-4600-b432-7d302233acb2","workflow_id":"cx-search-ds-dswe-native-scjx5","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-07T11:25:39.893Z","resumed_from_workflow_id":null,"config_key":"6841ddcbc1f0acd301d9d4773de95a74","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.60177,"total_questions":113,"correct_answers":68,"total_cost":0,"generation_time_ms":401314824,"duration_ms":46117893,"input_tokens":2539534738,"output_tokens":66645634,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"95dbbb09-90d7-48ed-82cb-4c720283b8eb","workflow_id":"cc-ow-native-glm-dswe-native-582h6","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-07T10:19:25.787Z","resumed_from_workflow_id":null,"config_key":"b46a93749141c7691d82771f482e8582","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.628319,"total_questions":113,"correct_answers":71,"total_cost":41.861539,"generation_time_ms":123986950,"duration_ms":19979787,"input_tokens":1117416932,"output_tokens":10736004,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"330a0b4e-489e-494a-bf6e-956e4c7bb3b9","workflow_id":"cc-ow-uni-glm-dswe-ts0-universal-gz84p","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-07T09:28:36.790Z","resumed_from_workflow_id":null,"config_key":"29b3c73268c048d5eb2fd2e149f588ad","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.557522,"total_questions":113,"correct_answers":63,"total_cost":40.916336,"generation_time_ms":124903473,"duration_ms":16928790,"input_tokens":1051344908,"output_tokens":9617961,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"80229960-3d4d-4a38-a1ae-159745856a95","workflow_id":"cc-ow-uni-glm-dswe-ts1-universal-bwk7z","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-07T09:28:15.290Z","resumed_from_workflow_id":null,"config_key":"7ce844e3549060c02b39198edce6347c","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.522124,"total_questions":113,"correct_answers":59,"total_cost":40.009846,"generation_time_ms":129700835,"duration_ms":16907290,"input_tokens":1047689130,"output_tokens":10050734,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"18d00425-e47c-4ac4-985d-15d9198be698","workflow_id":"cc-ow-uni-ds-dswe-ts0-universal-2lfc6","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-07T09:05:58.863Z","resumed_from_workflow_id":null,"config_key":"a1e2502b07075dbf3ffee95fb4ff27e5","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.716814,"total_questions":113,"correct_answers":81,"total_cost":27.474997,"generation_time_ms":113216271,"duration_ms":15571863,"input_tokens":1378702420,"output_tokens":10632000,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"2e1e27d0-120f-458a-9d44-f5f656ada004","workflow_id":"cc-ow-uni-ds-dswe-ts1-universal-xlh4z","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-07T09:02:55.156Z","resumed_from_workflow_id":null,"config_key":"f15635e7436c04a9e481336950836000","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.707965,"total_questions":113,"correct_answers":80,"total_cost":26.252033,"generation_time_ms":109541826,"duration_ms":15388156,"input_tokens":1416750841,"output_tokens":11176320,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"d0d0cba8-8918-44ce-9f47-5d5c5078b269","workflow_id":"cc-ow-native-ds-dswe-native-rlbwv","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-07T09:01:49.989Z","resumed_from_workflow_id":null,"config_key":"bc6c51dda882522e1b2fe81052c30405","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.619469,"total_questions":113,"correct_answers":70,"total_cost":27.378254,"generation_time_ms":113483596,"duration_ms":15323989,"input_tokens":1455029739,"output_tokens":11586721,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"00eebc91-0826-4aad-9b82-b4431aa25d16","workflow_id":"cc-ow-native-kimi-dswe-native-s66wj","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-07T08:28:17.002Z","resumed_from_workflow_id":null,"config_key":"8bb410e9ec9d0ffc42668eee95c5c473","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.681416,"total_questions":113,"correct_answers":77,"total_cost":372.244795,"generation_time_ms":92841963,"duration_ms":13310002,"input_tokens":988431058,"output_tokens":6253691,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"45ee9acc-1624-4cbb-809f-a19edbed6207","workflow_id":"cc-ow-uni-kimi-dswe-ts0-universal-tdhd4","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-07T08:08:11.063Z","resumed_from_workflow_id":null,"config_key":"e5603b25bf50a9ab82c348c19ea585d7","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.725664,"total_questions":113,"correct_answers":82,"total_cost":368.349663,"generation_time_ms":88216694,"duration_ms":12102063,"input_tokens":993532744,"output_tokens":6391848,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"d8537238-a3b0-478d-b8df-d0494a1a52a9","workflow_id":"cc-ow-uni-kimi-dswe-ts1-universal-cnsjb","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-07T07:54:17.507Z","resumed_from_workflow_id":null,"config_key":"204972b9d205f6fff0758341727c1c1c","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.831858,"total_questions":113,"correct_answers":94,"total_cost":254.609251,"generation_time_ms":78481297,"duration_ms":11268507,"input_tokens":717681331,"output_tokens":4698772,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"d17f3325-cdd5-4a7d-a5f9-c52b58c71a37","workflow_id":"cx-search-kimi-dswe-native-gzmr4","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-07T03:27:24.601Z","resumed_from_workflow_id":null,"config_key":"76e838508cac052080197ff7b3ed7dcc","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.637168,"total_questions":113,"correct_answers":72,"total_cost":410.743011,"generation_time_ms":124546574,"duration_ms":17421601,"input_tokens":1239334662,"output_tokens":6535290,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"4cb47693-a1f0-494d-8518-ac8cded2264a","workflow_id":"cx-search-ds-dswe-universal-9z4lj","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-11-07T02:42:39.507Z","resumed_from_workflow_id":null,"config_key":"08b92a99736e99951a20e15da19753e4","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.646018,"total_questions":113,"correct_answers":73,"total_cost":22.064799,"generation_time_ms":105394384,"duration_ms":14737507,"input_tokens":1387535391,"output_tokens":10397315,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"f5cbf23d-4c3c-4249-9803-184b02683eff","workflow_id":"cx-search-kimi-dswe-universal-xj57g","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-11-07T02:41:29.496Z","resumed_from_workflow_id":null,"config_key":"c45616e601eaf188e3f7009b7e2381b3","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.707965,"total_questions":113,"correct_answers":80,"total_cost":257.327858,"generation_time_ms":108963991,"duration_ms":14666496,"input_tokens":632690652,"output_tokens":5631184,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"82341414-9198-47ec-9ca4-135440008307","workflow_id":"cx-search-glm-dswe-universal-df5b7","benchmark_type":"deep_swe","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"OpenRouter","timestamp":"2026-11-07T02:22:49.722Z","resumed_from_workflow_id":null,"config_key":"df2df04436d788f0ff41d8cd24df3884","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.424779,"total_questions":113,"correct_answers":48,"total_cost":38.181891,"generation_time_ms":95936487,"duration_ms":13546722,"input_tokens":1141368545,"output_tokens":6357640,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"c18d207b-5544-4dd2-9bdb-8c6161d66dcb","workflow_id":"cx-luna-uni-luna-dswe-cm1-ts0-universal-xr6w2","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-07T01:54:27.645Z","resumed_from_workflow_id":null,"config_key":"543c6e499b47e19c24df95e22c68bec5","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.318584,"total_questions":113,"correct_answers":36,"total_cost":10.30761,"generation_time_ms":73433389,"duration_ms":11847645,"input_tokens":626363542,"output_tokens":4108665,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9118fa5a-edee-464b-80f4-965ee426b5cd","workflow_id":"cx-luna-uni-luna-dswe-cm0-ts0-universal-fqfns","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-07T01:42:11.882Z","resumed_from_workflow_id":null,"config_key":"1a0ee9ac232bb37c3c41ccc4f5d07d73","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.424779,"total_questions":113,"correct_answers":48,"total_cost":11.031578,"generation_time_ms":72047351,"duration_ms":11111882,"input_tokens":716536838,"output_tokens":4261149,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"cd7169b8-f9c6-4396-988b-86f58cc9e8e8","workflow_id":"cx-luna-uni-luna-dswe-cm1-ts1-universal-t2xrr","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-07T01:41:10.860Z","resumed_from_workflow_id":null,"config_key":"9ab23227af516321785622c1858f0b73","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.309735,"total_questions":113,"correct_answers":35,"total_cost":10.043619,"generation_time_ms":72576092,"duration_ms":11049860,"input_tokens":633360514,"output_tokens":4206430,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"6b2b1c98-e82e-4266-8a12-32b4001ec930","workflow_id":"cx-luna-native-luna-dswe-cm1-native-mwdj5","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-07T01:34:40.000Z","resumed_from_workflow_id":null,"config_key":"b74838d1353cce92ed67f88d24237802","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.389381,"total_questions":113,"correct_answers":44,"total_cost":9.802467,"generation_time_ms":69431662,"duration_ms":10661000,"input_tokens":612117920,"output_tokens":4191177,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"b3a5fa1e-1853-4ac6-a284-98ba9513656a","workflow_id":"cx-luna-native-luna-dswe-cm0-native-hgxcs","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-07T01:33:31.585Z","resumed_from_workflow_id":null,"config_key":"73aadf66416e26a0b8743096c4b3902d","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.300885,"total_questions":113,"correct_answers":34,"total_cost":10.467163,"generation_time_ms":65696908,"duration_ms":10592585,"input_tokens":689937797,"output_tokens":3754919,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"3a0f5cdb-c37c-48fa-92bd-319921c87774","workflow_id":"cx-luna-uni-luna-dswe-cm0-ts1-universal-dhvx6","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-11-07T01:33:25.463Z","resumed_from_workflow_id":null,"config_key":"c1529903c1e9a72b7304f767ee8b848f","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.415929,"total_questions":113,"correct_answers":47,"total_cost":10.709676,"generation_time_ms":71472351,"duration_ms":10585463,"input_tokens":697243252,"output_tokens":4197306,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9592226a-2179-4ad2-94cf-f4b7e5db0c38","workflow_id":"cc-sonnet-uni-sonnet-dswe-ts1-universal-dqzkp","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-07T00:48:33.248Z","resumed_from_workflow_id":null,"config_key":"5d449031bd98da88ed23bce38dff9341","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.628319,"total_questions":113,"correct_answers":71,"total_cost":46.261183,"generation_time_ms":35831228,"duration_ms":7894248,"input_tokens":100691332,"output_tokens":2246387,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"ce92b210-d239-4601-8161-54cc0e575fb7","workflow_id":"cc-sonnet-native-sonnet-dswe-native-5kjdj","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-07T00:29:03.794Z","resumed_from_workflow_id":null,"config_key":"b9bfd9050cf2f9d8866f1affb98c5434","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.566372,"total_questions":113,"correct_answers":64,"total_cost":47.539859,"generation_time_ms":35252013,"duration_ms":6725794,"input_tokens":115243282,"output_tokens":2192582,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"8c947729-6a1d-4c7b-a90e-94f4b8f2d371","workflow_id":"cc-sonnet-uni-sonnet-dswe-ts0-universal-rxdvz","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-11-07T00:23:27.208Z","resumed_from_workflow_id":null,"config_key":"d8357b12af6a23748a88c43f21190f69","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.584071,"total_questions":113,"correct_answers":66,"total_cost":44.616824,"generation_time_ms":32301602,"duration_ms":6389208,"input_tokens":93292948,"output_tokens":2230892,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"1949ccd5-65a7-4c80-b3f8-43b5af8f3a30","workflow_id":"tb4-opus55-hi-1105","benchmark_type":"terminal-bench-4.0","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"OpenRouter","timestamp":"2026-11-05T16:19:06.754Z","resumed_from_workflow_id":null,"config_key":"db87ef2ed494d23c122e7b627a310b22","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":0,"accuracy":0.530612,"total_questions":49,"correct_answers":31,"total_cost":1107.233377,"generation_time_ms":584021526,"duration_ms":50250754,"input_tokens":5581200,"output_tokens":30662964,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"cc2afa62-a1e3-4a16-8d21-8f7ccf3c2a0b","workflow_id":"rbn04-tb4-auto","benchmark_type":"terminal-bench-4.0","model_permaslug":"openrouter/auto","provider_name":"OpenRouter","timestamp":"2026-11-04T16:38:32.739Z","resumed_from_workflow_id":null,"config_key":"a2b6b5c7a015d3ac35d1a1a1685ad0b3","reasoning_effort":"high","max_tokens":null,"cost_tier":"medium","route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":0,"accuracy":0.456897,"total_questions":58,"correct_answers":32,"total_cost":76.940175,"generation_time_ms":87291077,"duration_ms":9320739,"input_tokens":2736582,"output_tokens":2819674,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"997a137d-0b80-4cbe-9ccf-5c4ad4089059","workflow_id":"rbn04-dswe-auto","benchmark_type":"deep_swe","model_permaslug":"openrouter/auto","provider_name":"OpenRouter","timestamp":"2026-11-04T16:01:09.159Z","resumed_from_workflow_id":null,"config_key":"1522c795a3201c38f313ba3ae673d340","reasoning_effort":"high","max_tokens":null,"cost_tier":"medium","route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.690265,"total_questions":226,"correct_answers":156,"total_cost":129.892809,"generation_time_ms":130224703,"duration_ms":7079159,"input_tokens":284350190,"output_tokens":4447238,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"cbc2da74-1d42-4eac-ad69-f706467678a6","workflow_id":"rbn04-tau3-acode2","benchmark_type":"tau3_bench_banking","model_permaslug":"openrouter/auto-code","provider_name":"auto-routing","timestamp":"2026-11-04T15:50:35.568Z","resumed_from_workflow_id":null,"config_key":"b04d1481e50e2f7aa45ed08ae737c714","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.525773,"total_questions":194,"correct_answers":102,"total_cost":115.742786,"generation_time_ms":68565814,"duration_ms":2140568,"input_tokens":333833119,"output_tokens":2013350,"reasoning_tokens":1336713,"success":true,"baseline":false},{"id":"3e1d3244-2404-4c9b-b0e4-d2213cd4d9a9","workflow_id":"rbn03-tb4-acode","benchmark_type":"terminal-bench-4.0","model_permaslug":"openrouter/auto-code","provider_name":"OpenRouter","timestamp":"2026-11-04T11:52:28.679Z","resumed_from_workflow_id":null,"config_key":"e599632ede3f880c934a928e57eba6f3","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":0,"accuracy":0.5,"total_questions":59,"correct_answers":44,"total_cost":292.610023,"generation_time_ms":250524431,"duration_ms":56309679,"input_tokens":9335122,"output_tokens":5168202,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"5c21a566-862e-4fea-a645-8fa89c72ec99","workflow_id":"rbn03-dswe-acode","benchmark_type":"deep_swe","model_permaslug":"openrouter/auto-code","provider_name":"OpenRouter","timestamp":"2026-11-04T07:46:44.010Z","resumed_from_workflow_id":null,"config_key":"33589962a802229647dbeb0e0057fe0e","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.743363,"total_questions":226,"correct_answers":168,"total_cost":446.648502,"generation_time_ms":205749655,"duration_ms":41562010,"input_tokens":390779104,"output_tokens":12507027,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"c145c033-2833-4a6b-9f6c-57330e21681e","workflow_id":"rbn03-qa-acode","benchmark_type":"swe_atlas_qa","model_permaslug":"openrouter/auto-code","provider_name":"OpenRouter","timestamp":"2026-11-04T05:41:57.858Z","resumed_from_workflow_id":null,"config_key":"d5726e330e5b0ab8665efe33012c9d7f","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.625,"total_questions":248,"correct_answers":155,"total_cost":162.626197,"generation_time_ms":63647122,"duration_ms":34072858,"input_tokens":217043598,"output_tokens":3173960,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"5af62a68-fb03-4619-8289-af2e1f935f32","workflow_id":"rbn03-qa-auto","benchmark_type":"swe_atlas_qa","model_permaslug":"openrouter/auto","provider_name":"OpenRouter","timestamp":"2026-11-04T05:35:45.035Z","resumed_from_workflow_id":null,"config_key":"a5d7a16125322f87df2d6ca864a68c43","reasoning_effort":"high","max_tokens":null,"cost_tier":"medium","route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.596774,"total_questions":248,"correct_answers":148,"total_cost":89.073864,"generation_time_ms":73154987,"duration_ms":33701035,"input_tokens":274303009,"output_tokens":3298604,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"bee144a2-0e12-45b9-a52e-5fb20e48d524","workflow_id":"rbn03-tau3-automedr","benchmark_type":"tau3_bench_banking","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-11-04T04:03:19.665Z","resumed_from_workflow_id":null,"config_key":"89b358305ec6e3309b313d8f43887ec2","reasoning_effort":"high","max_tokens":null,"cost_tier":"medium","route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.530928,"total_questions":194,"correct_answers":103,"total_cost":116.099414,"generation_time_ms":62297172,"duration_ms":4542665,"input_tokens":342031770,"output_tokens":1892821,"reasoning_tokens":1231315,"success":true,"baseline":false},{"id":"5a01c136-92bd-4199-a61b-82ed7d083c87","workflow_id":"rbn03-mmmu-auto","benchmark_type":"mmmu_pro_vision","model_permaslug":"openrouter/auto","provider_name":"auto-routing","timestamp":"2026-11-04T03:41:59.803Z","resumed_from_workflow_id":null,"config_key":"471872ad8e10b786627302fc105b3d17","reasoning_effort":"high","max_tokens":null,"cost_tier":"medium","route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.800867,"total_questions":3460,"correct_answers":2771,"total_cost":67.882331,"generation_time_ms":129048808,"duration_ms":26959803,"input_tokens":9351116,"output_tokens":13236880,"reasoning_tokens":12696370,"success":true,"baseline":false},{"id":"f903303f-2d24-4a4c-a5c2-ac20201ecaef","workflow_id":"rbn03-mmmu-acode","benchmark_type":"mmmu_pro_vision","model_permaslug":"openrouter/auto-code","provider_name":"auto-routing","timestamp":"2026-11-03T22:07:48.038Z","resumed_from_workflow_id":null,"config_key":"387499cd6d663b6b8501fcc6e9ce221a","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.82341,"total_questions":3460,"correct_answers":2849,"total_cost":87.709242,"generation_time_ms":21657151,"duration_ms":6906038,"input_tokens":11976708,"output_tokens":949475,"reasoning_tokens":492509,"success":true,"baseline":false},{"id":"7379bcea-a441-489e-a3a8-8fef1070e4f2","workflow_id":"rbk31p-dswe-e2","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-10-28T22:28:20.818Z","resumed_from_workflow_id":null,"config_key":"5e5e0bdf93948857b2b50b80411cebf9","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.778761,"total_questions":113,"correct_answers":88,"total_cost":95.855886,"generation_time_ms":125837901,"duration_ms":15772818,"input_tokens":243469019,"output_tokens":2956035,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"77ee5d8b-b7a6-4934-ba43-a24e62ac2dd4","workflow_id":"rbk31p-qa-e2","benchmark_type":"swe_atlas_qa","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-10-28T20:55:12.162Z","resumed_from_workflow_id":null,"config_key":"9f56a9c66249b3dcae6eebcf45b64325","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.580645,"total_questions":124,"correct_answers":72,"total_cost":57.926013,"generation_time_ms":70454280,"duration_ms":10182162,"input_tokens":125688830,"output_tokens":1702850,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"fb78e98d-1a51-46ed-845a-007f6ba75454","workflow_id":"rbk31p-mmmu-e2","benchmark_type":"mmmu_pro_vision","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"auto-routing","timestamp":"2026-10-28T19:15:53.308Z","resumed_from_workflow_id":null,"config_key":"e84c6b8df981d89c8bd29e2b3172c5f2","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.818497,"total_questions":1730,"correct_answers":1416,"total_cost":27.538908,"generation_time_ms":50769488,"duration_ms":4224308,"input_tokens":7142376,"output_tokens":1325479,"reasoning_tokens":1186085,"success":true,"baseline":true},{"id":"0ac95c13-f529-4300-a59a-0cd462bac0bf","workflow_id":"rbk31p-tau3-e2","benchmark_type":"tau3_bench_banking","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"auto-routing","timestamp":"2026-10-28T18:40:08.414Z","resumed_from_workflow_id":null,"config_key":"11be9368d3abba8ed0bbdbf8c8839592","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.546392,"total_questions":97,"correct_answers":53,"total_cost":64.313204,"generation_time_ms":35190249,"duration_ms":2077414,"input_tokens":176969433,"output_tokens":877782,"reasoning_tokens":535162,"success":true,"baseline":true},{"id":"a344ca26-f058-4583-90c1-ad0ade89596c","workflow_id":"rbop55-dswe-hi","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"OpenRouter","timestamp":"2026-10-28T14:06:27.652Z","resumed_from_workflow_id":null,"config_key":"518062ee052ffdefa621509ab9ceffd3","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.705357,"total_questions":112,"correct_answers":79,"total_cost":868.874065,"generation_time_ms":348658727,"duration_ms":43857652,"input_tokens":526590878,"output_tokens":31963866,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"17c6adc4-bfc7-4907-9598-e89b792278d2","workflow_id":"rbsn55-dswe-hi","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-28T08:11:33.684Z","resumed_from_workflow_id":null,"config_key":"7b218a547ef8c0ff2d54bf8ad50be6b4","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.663717,"total_questions":113,"correct_answers":75,"total_cost":210.057059,"generation_time_ms":145200436,"duration_ms":22562684,"input_tokens":233877886,"output_tokens":15565771,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"1ee66cd8-7be1-4b49-b389-e0592a7b73e3","workflow_id":"rbsy-ag-qa-p2","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"OpenRouter","timestamp":"2026-10-28T05:02:31.802Z","resumed_from_workflow_id":null,"config_key":"86d564653f5d174354e7af01ab535d9e","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["google/gemini-4-argon","google/gemini-3.8-flash"],"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.604839,"total_questions":124,"correct_answers":75,"total_cost":158.327125,"generation_time_ms":93680162,"duration_ms":12390802,"input_tokens":665111130,"output_tokens":2065157,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"acde9730-3bb9-47db-b89e-e712cca107b6","workflow_id":"rbsy-os-dswe-p1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"OpenRouter","timestamp":"2026-10-28T00:33:36.930Z","resumed_from_workflow_id":null,"config_key":"7d3e061953a2c5591de010473d77b575","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["anthropic/claude-opus-5.5","anthropic/claude-sonnet-5.5"],"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":0,"accuracy":0.6875,"total_questions":112,"correct_answers":77,"total_cost":738.270285,"generation_time_ms":301637057,"duration_ms":37910930,"input_tokens":461912092,"output_tokens":28001385,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"8bab90f9-5f6c-4604-bb0d-55e5d2f797ba","workflow_id":"rbsy-as-dswe-p1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"OpenRouter","timestamp":"2026-10-27T17:17:20.254Z","resumed_from_workflow_id":null,"config_key":"6ceebef77b65408fbcde39be935dc3f0","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-astra","openai/gpt-6.1-sol"],"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.734513,"total_questions":113,"correct_answers":83,"total_cost":287.248523,"generation_time_ms":84947829,"duration_ms":11733254,"input_tokens":134105296,"output_tokens":2781785,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"cec6adb6-b2e6-422f-9903-21e629d49805","workflow_id":"rbsy-sl-dswe-p1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"OpenRouter","timestamp":"2026-10-27T17:13:49.220Z","resumed_from_workflow_id":null,"config_key":"d571383e58ca6e58eacbab56a28913d7","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6.1-sol","openai/gpt-6-luna"],"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.752212,"total_questions":113,"correct_answers":85,"total_cost":53.265002,"generation_time_ms":85094360,"duration_ms":11521220,"input_tokens":187829772,"output_tokens":3705514,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"a991f98c-1e3e-4706-85a2-49126c21c8b2","workflow_id":"rbt61-tb4-hi","benchmark_type":"terminal-bench-4.0","model_permaslug":"openai/gpt-6.1-terra","provider_name":"OpenRouter","timestamp":"2026-10-27T13:27:44.690Z","resumed_from_workflow_id":null,"config_key":"c3434b75e3a6f3e6351c784626ddb79a","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.408333,"total_questions":118,"correct_answers":48,"total_cost":47.189346,"generation_time_ms":168450881,"duration_ms":36413690,"input_tokens":119476,"output_tokens":3370984,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"387c2d4b-cd54-4f78-8773-4b47a4e19f7d","workflow_id":"rb1026-tb4-pareto2610","benchmark_type":"terminal-bench-4.0","model_permaslug":"unbiased/pareto-26.10","provider_name":"OpenRouter","timestamp":"2026-10-27T11:49:08.046Z","resumed_from_workflow_id":null,"config_key":"86cf4540ae0d9e9c3c29b703fdbc33a5","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.425,"total_questions":120,"correct_answers":51,"total_cost":35.355151,"generation_time_ms":164454848,"duration_ms":27631046,"input_tokens":14408663,"output_tokens":5703767,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"27b2373d-09fc-4a96-b762-601a6e96b463","workflow_id":"rbt61-dswe-hi","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6.1-terra","provider_name":"OpenRouter","timestamp":"2026-10-27T08:00:32.421Z","resumed_from_workflow_id":null,"config_key":"630fde535e99e1792dff18d50efd87a0","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.727679,"total_questions":223,"correct_answers":162,"total_cost":69.133806,"generation_time_ms":111588842,"duration_ms":16781421,"input_tokens":499308298,"output_tokens":4975496,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"a13fb2e5-ddec-4656-a75a-47aa168a3be7","workflow_id":"rbt61-qa-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6.1-terra","provider_name":"OpenRouter","timestamp":"2026-10-27T06:09:14.007Z","resumed_from_workflow_id":null,"config_key":"bf8316097a482260a2b8ea9fb7f8138e","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.53629,"total_questions":248,"correct_answers":133,"total_cost":38.769532,"generation_time_ms":53097809,"duration_ms":10104007,"input_tokens":173397094,"output_tokens":2234225,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"01f01d21-043a-4382-af53-72de88c49c08","workflow_id":"rbt61-mmmu-hi","benchmark_type":"mmmu_pro_vision","model_permaslug":"openai/gpt-6.1-terra","provider_name":"auto-routing","timestamp":"2026-10-27T04:06:41.334Z","resumed_from_workflow_id":null,"config_key":"44b2d01c1797f8050f4c5a27b96bdf5c","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.803468,"total_questions":3460,"correct_answers":2780,"total_cost":20.750303,"generation_time_ms":21304900,"duration_ms":2753334,"input_tokens":12873338,"output_tokens":949461,"reasoning_tokens":790951,"success":true,"baseline":true},{"id":"35f228ed-6361-4f5d-8c41-2ff59de0f8f8","workflow_id":"rbt61-tau3-hi","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6.1-terra","provider_name":"auto-routing","timestamp":"2026-10-27T03:34:14.883Z","resumed_from_workflow_id":null,"config_key":"0709e8756b6d6500728e49d435af7ee1","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.329897,"total_questions":194,"correct_answers":64,"total_cost":33.711239,"generation_time_ms":14784914,"duration_ms":805883,"input_tokens":226450967,"output_tokens":622873,"reasoning_tokens":243771,"success":true,"baseline":true},{"id":"41da7e6c-6adf-4820-acb9-1bf6d1062705","workflow_id":"rbk31p-dswe-hi","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-10-26T22:29:47.760Z","resumed_from_workflow_id":null,"config_key":"473e20567649462781a1bccd37cf02f7","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.778761,"total_questions":113,"correct_answers":88,"total_cost":94.827642,"generation_time_ms":109289447,"duration_ms":14141760,"input_tokens":244601401,"output_tokens":2857140,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9bfde096-8b2a-413d-a8d2-f73c68967abc","workflow_id":"rbk31p-qa-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"OpenRouter","timestamp":"2026-10-26T21:06:27.251Z","resumed_from_workflow_id":null,"config_key":"da2ddae1f78906c369679aa5def2f596","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.580645,"total_questions":124,"correct_answers":72,"total_cost":59.540662,"generation_time_ms":64322761,"duration_ms":9140251,"input_tokens":131756937,"output_tokens":1717965,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"097b6b50-78c3-46b3-894b-7d754102780d","workflow_id":"rbk31p-mmmu-hi","benchmark_type":"mmmu_pro_vision","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"auto-routing","timestamp":"2026-10-26T19:22:06.071Z","resumed_from_workflow_id":null,"config_key":"db73e71cb0461df877364dfc4d6abfcb","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.816763,"total_questions":1730,"correct_answers":1413,"total_cost":27.31192,"generation_time_ms":44797868,"duration_ms":3574071,"input_tokens":7142376,"output_tokens":1312444,"reasoning_tokens":1162000,"success":true,"baseline":true},{"id":"0c0b4a73-3f37-4931-a11a-7e6f855d7e77","workflow_id":"rbk31p-tau3-hi","benchmark_type":"tau3_bench_banking","model_permaslug":"moonshotai/kimi-k3.1-pro","provider_name":"auto-routing","timestamp":"2026-10-26T18:59:18.934Z","resumed_from_workflow_id":null,"config_key":"5a29a54272d6536130a2d6c61cad7023","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.546392,"total_questions":97,"correct_answers":53,"total_cost":61.760362,"generation_time_ms":31082044,"duration_ms":2096934,"input_tokens":173025669,"output_tokens":795161,"reasoning_tokens":450616,"success":true,"baseline":true},{"id":"d7abd2e4-ffaf-456a-8ccb-0b1c1570056d","workflow_id":"rbsy-os-tb4-e2","benchmark_type":"terminal-bench-4.0","model_permaslug":"nvidia/switchyard","provider_name":"OpenRouter","timestamp":"2026-10-23T16:41:50.898Z","resumed_from_workflow_id":null,"config_key":"33a316d20cff854b42706e01fb37556f","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["anthropic/claude-opus-5.5","anthropic/claude-sonnet-5.5"],"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.45,"total_questions":60,"correct_answers":27,"total_cost":233.601671,"generation_time_ms":180791819,"duration_ms":39409898,"input_tokens":5436,"output_tokens":3607157,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"6f66105a-1284-4834-ba4c-8399953e409b","workflow_id":"rbsy-ag-tb4-e2","benchmark_type":"terminal-bench-4.0","model_permaslug":"nvidia/switchyard","provider_name":"OpenRouter","timestamp":"2026-10-23T16:28:08.932Z","resumed_from_workflow_id":null,"config_key":"64ddc888021608ff3759492cd2057f4e","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["google/gemini-4-argon","google/gemini-3.8-flash"],"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.35,"total_questions":60,"correct_answers":21,"total_cost":4.987909,"generation_time_ms":206464033,"duration_ms":38570932,"input_tokens":118501006,"output_tokens":6659010,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"dcee0d8c-39e1-46ee-b7e1-518a00b02ebe","workflow_id":"rbgarg-dswe-hi","benchmark_type":"deep_swe","model_permaslug":"google/gemini-4-argon","provider_name":"OpenRouter","timestamp":"2026-10-23T13:31:25.257Z","resumed_from_workflow_id":null,"config_key":"8d6fa50369d0ba49ee225f58457a3916","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.761062,"total_questions":113,"correct_answers":86,"total_cost":352.227198,"generation_time_ms":180106600,"duration_ms":21676257,"input_tokens":1745104830,"output_tokens":6668831,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"046b4ef2-c7df-4a00-8ded-190c482e7876","workflow_id":"rbsy-os-qa-e2","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"OpenRouter","timestamp":"2026-10-23T11:28:30.830Z","resumed_from_workflow_id":null,"config_key":"bdd0586edf36cf0be6cc166d0161bf4c","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["anthropic/claude-opus-5.5","anthropic/claude-sonnet-5.5"],"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.564516,"total_questions":124,"correct_answers":70,"total_cost":99.359428,"generation_time_ms":52105550,"duration_ms":14766830,"input_tokens":112714098,"output_tokens":2200808,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"b281db76-f30c-4fe4-9eed-c1f224654554","workflow_id":"rbgarg-qa-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"google/gemini-4-argon","provider_name":"OpenRouter","timestamp":"2026-10-23T10:51:00.311Z","resumed_from_workflow_id":null,"config_key":"10f1783de4fe21a3e5cc39e07a86248c","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.58871,"total_questions":124,"correct_answers":73,"total_cost":155.306596,"generation_time_ms":95817883,"duration_ms":12050311,"input_tokens":639765593,"output_tokens":2531647,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"6fec06e9-8277-4196-b55a-799d7566e42c","workflow_id":"rbds1017-qa-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"deepseek/deepseek-v4.1-flash-1017","provider_name":"OpenRouter","timestamp":"2026-10-23T09:28:55.962Z","resumed_from_workflow_id":null,"config_key":"5689e52ebcdf36db8970c4805480ccb8","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.580645,"total_questions":248,"correct_answers":144,"total_cost":23.56028,"generation_time_ms":84922306,"duration_ms":13073962,"input_tokens":1077748369,"output_tokens":7347724,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"3c57ac3a-68ae-4ce3-a6b2-28a9d754b83d","workflow_id":"rbg38f-mmmu-hi","benchmark_type":"mmmu_pro_vision","model_permaslug":"google/gemini-3.8-flash","provider_name":"auto-routing","timestamp":"2026-10-23T08:42:58.312Z","resumed_from_workflow_id":null,"config_key":"dfc67ab7a64132180b624dcd8f13068b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.815029,"total_questions":1730,"correct_answers":1410,"total_cost":40.319382,"generation_time_ms":78276185,"duration_ms":4576312,"input_tokens":2125126,"output_tokens":10326810,"reasoning_tokens":9853919,"success":true,"baseline":true},{"id":"f58b76bd-6865-4cc3-bab9-6fac3dfe46b3","workflow_id":"rbgarg-mmmu-hi","benchmark_type":"mmmu_pro_vision","model_permaslug":"google/gemini-4-argon","provider_name":"auto-routing","timestamp":"2026-10-23T08:34:30.750Z","resumed_from_workflow_id":null,"config_key":"16a8dbe6b5e6041ed3c0cd21bbcd3062","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.869942,"total_questions":1730,"correct_answers":1505,"total_cost":46.46712,"generation_time_ms":62179362,"duration_ms":4072750,"input_tokens":2193245,"output_tokens":4208063,"reasoning_tokens":3655757,"success":true,"baseline":true},{"id":"b4e94a53-7f1f-4e8b-b549-1dead66e0996","workflow_id":"rbgarg-tau3-hi","benchmark_type":"tau3_bench_banking","model_permaslug":"google/gemini-4-argon","provider_name":"auto-routing","timestamp":"2026-10-23T08:17:26.120Z","resumed_from_workflow_id":null,"config_key":"b27cc2fe7ff8748bc377dbf5d394aefa","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.463918,"total_questions":97,"correct_answers":45,"total_cost":91.435361,"generation_time_ms":61377940,"duration_ms":3046120,"input_tokens":325064608,"output_tokens":2543894,"reasoning_tokens":2143769,"success":true,"baseline":true},{"id":"65107e65-8e3a-469a-9499-8879a28811bb","workflow_id":"rbds1017-mmmu-hi","benchmark_type":"mmmu_pro_vision","model_permaslug":"deepseek/deepseek-v4.1-flash-1017","provider_name":"auto-routing","timestamp":"2026-10-23T07:39:30.612Z","resumed_from_workflow_id":null,"config_key":"2ecc3d77d70499c57a86b0f812df8ba2","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.72659,"total_questions":3460,"correct_answers":2514,"total_cost":22.045348,"generation_time_ms":170924779,"duration_ms":6509612,"input_tokens":3415310,"output_tokens":34575216,"reasoning_tokens":34198954,"success":true,"baseline":true},{"id":"39c37971-1876-4700-9b26-9bfbf8c8a5bc","workflow_id":"rbsy-os-tau3-e2","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-10-23T07:33:05.677Z","resumed_from_workflow_id":null,"config_key":"c1c0dad2958aab474893039de2c34d4e","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["anthropic/claude-opus-5.5","anthropic/claude-sonnet-5.5"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.350515,"total_questions":97,"correct_answers":34,"total_cost":75.497071,"generation_time_ms":10013216,"duration_ms":623677,"input_tokens":107130121,"output_tokens":575931,"reasoning_tokens":204773,"success":true,"baseline":false},{"id":"a4a0bc72-a58d-420e-ae68-6cb29b323902","workflow_id":"rbds1017-tau3-hi","benchmark_type":"tau3_bench_banking","model_permaslug":"deepseek/deepseek-v4.1-flash-1017","provider_name":"auto-routing","timestamp":"2026-10-23T06:39:14.234Z","resumed_from_workflow_id":null,"config_key":"0e54fa8a7c92dd37bf0245dff8cff26d","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.247423,"total_questions":194,"correct_answers":48,"total_cost":6.187869,"generation_time_ms":25576221,"duration_ms":2892234,"input_tokens":155929592,"output_tokens":4158384,"reasoning_tokens":3706583,"success":true,"baseline":true},{"id":"f278a2ac-cac3-4dd0-bf85-0234629fbded","workflow_id":"rbsy-as-tb4-p1","benchmark_type":"terminal-bench-4.0","model_permaslug":"nvidia/switchyard","provider_name":"OpenRouter","timestamp":"2026-10-16T20:15:31.776Z","resumed_from_workflow_id":null,"config_key":"ad394c2d56f326040f99a2bc0f8ef724","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-astra","openai/gpt-6.1-sol"],"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.45,"total_questions":60,"correct_answers":27,"total_cost":156.193089,"generation_time_ms":46877410,"duration_ms":9036776,"input_tokens":7002,"output_tokens":1092932,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"5bd15309-5b32-47e3-9706-8fce79139ad9","workflow_id":"rbsy-sl-tb4-p1","benchmark_type":"terminal-bench-4.0","model_permaslug":"nvidia/switchyard","provider_name":"OpenRouter","timestamp":"2026-10-16T20:14:17.564Z","resumed_from_workflow_id":null,"config_key":"c4c15e6e16bab22964bb8182ba679b03","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6.1-sol","openai/gpt-6-luna"],"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.333333,"total_questions":60,"correct_answers":20,"total_cost":26.641259,"generation_time_ms":42954796,"duration_ms":8892564,"input_tokens":150898,"output_tokens":1265421,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"270f22c0-1ba0-4611-98a0-b47fe72cd492","workflow_id":"rbsy-ag-tau3-p1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-10-16T18:54:43.533Z","resumed_from_workflow_id":null,"config_key":"2da21f80627b8b9d33ad7c8ce4fd6cfc","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["google/gemini-4-argon","google/gemini-3.8-flash"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.474227,"total_questions":97,"correct_answers":46,"total_cost":72.150116,"generation_time_ms":40794557,"duration_ms":1973533,"input_tokens":260576183,"output_tokens":2207024,"reasoning_tokens":1885659,"success":true,"baseline":false},{"id":"d2443e77-8071-48b1-b4c5-9494a836ba8e","workflow_id":"rbsy-sl-tau3-p1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-10-16T18:41:32.581Z","resumed_from_workflow_id":null,"config_key":"070e4a0d392626013498744203c64863","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6.1-sol","openai/gpt-6-luna"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.268041,"total_questions":97,"correct_answers":26,"total_cost":20.198682,"generation_time_ms":14992336,"duration_ms":1173581,"input_tokens":90491995,"output_tokens":658355,"reasoning_tokens":480319,"success":true,"baseline":false},{"id":"9921dbf5-5d82-486d-9ab7-7f9cb93c53a0","workflow_id":"rbsy-as-tau3-p1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-10-16T18:39:34.863Z","resumed_from_workflow_id":null,"config_key":"2f768a5485d3bcd3033ee21f2086fea1","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-astra","openai/gpt-6.1-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.381443,"total_questions":97,"correct_answers":37,"total_cost":133.314829,"generation_time_ms":14042783,"duration_ms":1060863,"input_tokens":94663425,"output_tokens":336626,"reasoning_tokens":132344,"success":true,"baseline":false},{"id":"f1ae6c02-4c17-45ef-8487-d012a0463f31","workflow_id":"rbp2610-dswe-10160209","benchmark_type":"deep_swe","model_permaslug":"unbiased/pareto-26.10","provider_name":"OpenRouter","timestamp":"2026-10-16T04:27:58.845Z","resumed_from_workflow_id":null,"config_key":"1b2f717aa7802ea5bce7571dbc5420f1","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.743363,"total_questions":113,"correct_answers":84,"total_cost":21.868471,"generation_time_ms":50632327,"duration_ms":8287845,"input_tokens":111770138,"output_tokens":1995742,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"3c0026a7-2326-4162-9420-0bd88496e09a","workflow_id":"rbp2610-dswe-10152339","benchmark_type":"deep_swe","model_permaslug":"unbiased/pareto-26.10","provider_name":"OpenRouter","timestamp":"2026-10-16T02:00:21.730Z","resumed_from_workflow_id":null,"config_key":"d3984cce521023be918d320d8967c526","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.725664,"total_questions":113,"correct_answers":82,"total_cost":21.716664,"generation_time_ms":50931529,"duration_ms":8445730,"input_tokens":106374919,"output_tokens":1920461,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"c11ccf71-16d1-405f-b6a3-aaa14dc14c57","workflow_id":"rbp2610p-qa-10151409","benchmark_type":"swe_atlas_qa","model_permaslug":"unbiased/pareto-26.10-preview","provider_name":"OpenRouter","timestamp":"2026-10-15T15:57:30.964Z","resumed_from_workflow_id":null,"config_key":"19ae85d36afb99344808d02eca14ecc7","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.604839,"total_questions":124,"correct_answers":75,"total_cost":17.531838,"generation_time_ms":35978373,"duration_ms":6481964,"input_tokens":78202785,"output_tokens":1425715,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"be84bfe9-b65d-4ef6-bb92-3401d9c01fe1","workflow_id":"rbp2610p-mmmu-10151409","benchmark_type":"mmmu_pro_vision","model_permaslug":"unbiased/pareto-26.10-preview","provider_name":"auto-routing","timestamp":"2026-10-15T14:59:05.363Z","resumed_from_workflow_id":null,"config_key":"d793697202561c1533aa216f0c8c0f1d","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.754913,"total_questions":1730,"correct_answers":1306,"total_cost":25.331648,"generation_time_ms":28917626,"duration_ms":2978363,"input_tokens":7333192,"output_tokens":6177446,"reasoning_tokens":0,"success":true,"baseline":false},{"id":"f3622ea7-1aeb-4c58-b3bf-d595e01e7dfd","workflow_id":"rbp2610p-tau3-10151409","benchmark_type":"tau3_bench_banking","model_permaslug":"unbiased/pareto-26.10-preview","provider_name":"auto-routing","timestamp":"2026-10-15T14:26:21.572Z","resumed_from_workflow_id":null,"config_key":"643e797d5479734f71407719df8b9267","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.350515,"total_questions":97,"correct_answers":34,"total_cost":14.861935,"generation_time_ms":10003783,"duration_ms":1013572,"input_tokens":93877806,"output_tokens":332590,"reasoning_tokens":0,"success":true,"baseline":false},{"id":"154fbc92-04e2-462f-9add-c7b5dababad8","workflow_id":"t3p1010-gem38f-high-r2","benchmark_type":"tau3_bench_banking","model_permaslug":"google/gemini-3.8-flash","provider_name":"auto-routing","timestamp":"2026-10-10T05:22:24.731Z","resumed_from_workflow_id":null,"config_key":"81a4e3b6517b51c0e741f82319b47cda","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.391753,"total_questions":192,"correct_answers":76,"total_cost":96.325353,"generation_time_ms":48742588,"duration_ms":1975731,"input_tokens":541378046,"output_tokens":4666828,"reasoning_tokens":4063502,"success":true,"baseline":true},{"id":"2874193b-81a9-4528-b3c1-25a59d5e1dab","workflow_id":"t3p1010-dsv41f-high","benchmark_type":"tau3_bench_banking","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"auto-routing","timestamp":"2026-10-10T04:19:03.434Z","resumed_from_workflow_id":null,"config_key":"39f9448840d7ee6465f006b48322c4d8","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.324742,"total_questions":194,"correct_answers":63,"total_cost":20.760549,"generation_time_ms":87067766,"duration_ms":7049434,"input_tokens":289230698,"output_tokens":17203624,"reasoning_tokens":16603047,"success":true,"baseline":true},{"id":"d95b1374-85b9-4eb3-8016-76b567c1ac86","workflow_id":"t3p1010-grok47-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"x-ai/grok-4.7","provider_name":"auto-routing","timestamp":"2026-10-10T03:20:57.352Z","resumed_from_workflow_id":null,"config_key":"5385beb0bc937fdc35332f81f48e547f","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.417526,"total_questions":194,"correct_answers":81,"total_cost":449.349908,"generation_time_ms":51810406,"duration_ms":3543352,"input_tokens":581553361,"output_tokens":2091118,"reasoning_tokens":1154554,"success":true,"baseline":true},{"id":"ca112ecf-4fbc-4ff8-a6d9-1876927a5224","workflow_id":"t3p1010-kimik3-high","benchmark_type":"tau3_bench_banking","model_permaslug":"moonshotai/kimi-k3","provider_name":"auto-routing","timestamp":"2026-10-10T02:53:34.209Z","resumed_from_workflow_id":null,"config_key":"7c75df8a080d65a48493665114937dd7","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.427835,"total_questions":194,"correct_answers":83,"total_cost":188.172706,"generation_time_ms":39374566,"duration_ms":1892209,"input_tokens":351734824,"output_tokens":2022328,"reasoning_tokens":1506918,"success":true,"baseline":true},{"id":"bee816aa-0b04-405d-8183-baf1cb762c83","workflow_id":"t3p1010-glm53f-high","benchmark_type":"tau3_bench_banking","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"auto-routing","timestamp":"2026-10-10T02:49:43.206Z","resumed_from_workflow_id":null,"config_key":"387c28695b1b2954826c9128fba147a7","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.340206,"total_questions":194,"correct_answers":66,"total_cost":8.342272,"generation_time_ms":15134768,"duration_ms":1675206,"input_tokens":204614612,"output_tokens":1100290,"reasoning_tokens":698133,"success":true,"baseline":true},{"id":"80a65eda-6cd3-49d1-979d-7a073f73eee1","workflow_id":"t3p1010-kimik3-low","benchmark_type":"tau3_bench_banking","model_permaslug":"moonshotai/kimi-k3","provider_name":"auto-routing","timestamp":"2026-10-10T02:44:47.837Z","resumed_from_workflow_id":null,"config_key":"281647aa8a4bdfc2e9fc856456766c41","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.381443,"total_questions":194,"correct_answers":74,"total_cost":137.376225,"generation_time_ms":30546156,"duration_ms":1364837,"input_tokens":268333343,"output_tokens":1291609,"reasoning_tokens":717985,"success":true,"baseline":true},{"id":"e6438901-698d-4589-b9ce-2c7e16ad4794","workflow_id":"t3p1010-astra-high","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-10-10T02:40:06.485Z","resumed_from_workflow_id":null,"config_key":"db4ea5eed060cabc3ecb0af667f0ee8b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.35567,"total_questions":194,"correct_answers":69,"total_cost":354.400302,"generation_time_ms":24784254,"duration_ms":1120485,"input_tokens":179951307,"output_tokens":669599,"reasoning_tokens":263680,"success":true,"baseline":true},{"id":"96f451f4-9b15-4358-b055-4e909ffc0013","workflow_id":"t3p1010-opus55-high","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:37:38.688Z","resumed_from_workflow_id":null,"config_key":"25803b2d58f64a6916cf262ca20119d3","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.42268,"total_questions":194,"correct_answers":82,"total_cost":189.853288,"generation_time_ms":22132077,"duration_ms":918688,"input_tokens":267066346,"output_tokens":1197021,"reasoning_tokens":389091,"success":true,"baseline":true},{"id":"aa8019a5-6d31-40cd-ac63-868725aff0cc","workflow_id":"t3p1010-sol61-high","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6.1-sol","provider_name":"auto-routing","timestamp":"2026-10-10T02:36:43.420Z","resumed_from_workflow_id":null,"config_key":"e46e166c284c2155e873235d6d5ff8c8","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.350515,"total_questions":194,"correct_answers":68,"total_cost":58.548217,"generation_time_ms":17832131,"duration_ms":854420,"input_tokens":204251391,"output_tokens":624880,"reasoning_tokens":212535,"success":true,"baseline":true},{"id":"4e40fe42-7b45-4b45-b93c-18621404dbf0","workflow_id":"t3p1010-opus55-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:36:37.039Z","resumed_from_workflow_id":null,"config_key":"afea1ac9cbbd6f317f36dbfa1fe47b90","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.396907,"total_questions":194,"correct_answers":77,"total_cost":178.328238,"generation_time_ms":21037363,"duration_ms":852039,"input_tokens":230127813,"output_tokens":1079482,"reasoning_tokens":281123,"success":true,"baseline":true},{"id":"8779ca9e-1ea3-4fa6-9d5f-511000c6029d","workflow_id":"t3p1010-luna-high","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-10-10T02:35:33.910Z","resumed_from_workflow_id":null,"config_key":"70e71e14269c29022b488aaabb7326a8","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.242268,"total_questions":194,"correct_answers":47,"total_cost":3.916447,"generation_time_ms":17677668,"duration_ms":805910,"input_tokens":174370261,"output_tokens":1655195,"reasoning_tokens":1313413,"success":true,"baseline":true},{"id":"5b26dbb7-6c94-4a3d-aa20-5ae3600ae033","workflow_id":"t3p1010-opus55-low","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:35:28.933Z","resumed_from_workflow_id":null,"config_key":"dffd86acd77fd474d87e64e852fd70fe","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.335052,"total_questions":194,"correct_answers":65,"total_cost":134.226022,"generation_time_ms":16544705,"duration_ms":787933,"input_tokens":159383608,"output_tokens":840360,"reasoning_tokens":158229,"success":true,"baseline":true},{"id":"d7bc8286-6786-4c4a-b22d-d46e0446f905","workflow_id":"t3p1010-astra-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-10-10T02:35:11.594Z","resumed_from_workflow_id":null,"config_key":"9a4e3bb03d7e650b9d0d7199a3c86863","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.329897,"total_questions":194,"correct_answers":64,"total_cost":288.849938,"generation_time_ms":17127060,"duration_ms":820594,"input_tokens":150560529,"output_tokens":402550,"reasoning_tokens":37448,"success":true,"baseline":true},{"id":"68bc55a3-1aef-4629-b5a7-e139a1c3696a","workflow_id":"t3p1010-sonnet55-high","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:34:21.530Z","resumed_from_workflow_id":null,"config_key":"b71cf84e0fafbe701a2460b9838df3ca","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.35567,"total_questions":194,"correct_answers":69,"total_cost":69.932633,"generation_time_ms":12684473,"duration_ms":703530,"input_tokens":194967728,"output_tokens":1051926,"reasoning_tokens":344522,"success":true,"baseline":true},{"id":"0908a549-5401-46a8-b034-191efee4cc86","workflow_id":"t3p1010-haiku55-high","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:34:13.427Z","resumed_from_workflow_id":null,"config_key":"066c140be21031cd617efe10c0d04fab","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.185567,"total_questions":194,"correct_answers":36,"total_cost":9.226977,"generation_time_ms":15045265,"duration_ms":737427,"input_tokens":178692155,"output_tokens":1912072,"reasoning_tokens":1193359,"success":true,"baseline":true},{"id":"76d20170-8e8d-4073-b88f-cf8fd2bb5a13","workflow_id":"t3p1010-astra-low","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-10-10T02:33:53.426Z","resumed_from_workflow_id":null,"config_key":"fd59e48524dc15bdc962ff0011c5b714","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.324742,"total_questions":194,"correct_answers":63,"total_cost":225.518622,"generation_time_ms":15633093,"duration_ms":744426,"input_tokens":120295738,"output_tokens":323827,"reasoning_tokens":8244,"success":true,"baseline":true},{"id":"bcb1fd24-81b9-4770-8745-a2dc00772b45","workflow_id":"t3p1010-sol61-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6.1-sol","provider_name":"auto-routing","timestamp":"2026-10-10T02:33:31.208Z","resumed_from_workflow_id":null,"config_key":"dfb79a74e5b6ccb48c10a64b50dd407a","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.371134,"total_questions":194,"correct_answers":72,"total_cost":44.665817,"generation_time_ms":13007060,"duration_ms":656208,"input_tokens":159237993,"output_tokens":385108,"reasoning_tokens":32717,"success":true,"baseline":true},{"id":"12bbeec1-f0a8-443b-99b5-83380f2c0850","workflow_id":"t3p1010-sonnet55-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:33:13.123Z","resumed_from_workflow_id":null,"config_key":"8ed0a9e82d8eaf5b6bf2ee942954e451","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.252577,"total_questions":194,"correct_answers":49,"total_cost":52.677928,"generation_time_ms":10152010,"duration_ms":631123,"input_tokens":136275446,"output_tokens":814138,"reasoning_tokens":203632,"success":true,"baseline":true},{"id":"1e37ee88-83d7-4757-bbc1-5a35c39c4882","workflow_id":"t3p1010-haiku55-medium","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:33:03.076Z","resumed_from_workflow_id":null,"config_key":"a4bdba2681e7b5fa1c921a2a2ef69164","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.14433,"total_questions":194,"correct_answers":28,"total_cost":4.928525,"generation_time_ms":11425844,"duration_ms":663076,"input_tokens":127149954,"output_tokens":1310261,"reasoning_tokens":669863,"success":true,"baseline":true},{"id":"e4fef9b1-c7c4-4432-844e-8da786f6aeab","workflow_id":"t3p1010-sonnet55-low","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:32:48.923Z","resumed_from_workflow_id":null,"config_key":"93cf3175fd83ba81ad44132a941bfb63","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.221649,"total_questions":194,"correct_answers":43,"total_cost":46.559712,"generation_time_ms":9106967,"duration_ms":608923,"input_tokens":114440891,"output_tokens":725224,"reasoning_tokens":159136,"success":true,"baseline":true},{"id":"fcc44e66-aa46-49bf-bce3-a596524366d8","workflow_id":"t3p1010-sol61-low","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6.1-sol","provider_name":"auto-routing","timestamp":"2026-10-10T02:32:33.784Z","resumed_from_workflow_id":null,"config_key":"6ec7ea30f75f901e9a50ed4e4f7b9ba7","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.278351,"total_questions":194,"correct_answers":54,"total_cost":31.495343,"generation_time_ms":11015534,"duration_ms":602784,"input_tokens":108534772,"output_tokens":307571,"reasoning_tokens":6614,"success":true,"baseline":true},{"id":"623414cd-52f9-4188-9090-6439af1b3a2a","workflow_id":"t3p1010-haiku55-low","benchmark_type":"tau3_bench_banking","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"auto-routing","timestamp":"2026-10-10T02:32:18.762Z","resumed_from_workflow_id":null,"config_key":"29901e57a90960abef8ebe35feaa4001","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.092784,"total_questions":194,"correct_answers":18,"total_cost":3.132722,"generation_time_ms":9061332,"duration_ms":620762,"input_tokens":94790075,"output_tokens":991538,"reasoning_tokens":415642,"success":true,"baseline":true},{"id":"d11b66d0-014f-43ba-a98b-28ab27d31ddd","workflow_id":"rbf-qa-dsv41f-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-10-10T01:08:14.874Z","resumed_from_workflow_id":null,"config_key":"33ef10c6ed72766b4ffe9b6fa9cab4f3","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.637097,"total_questions":124,"correct_answers":79,"total_cost":23.175303,"generation_time_ms":81778945,"duration_ms":29590874,"input_tokens":995037635,"output_tokens":5683269,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"72e92a79-4856-4a1e-82b3-c433ae2cadb1","workflow_id":"rbf-dswe-haiku55-md","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T23:15:51.582Z","resumed_from_workflow_id":null,"config_key":"ad8e7eaf6947775262f41d3b567f81db","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.522124,"total_questions":113,"correct_answers":59,"total_cost":28.585066,"generation_time_ms":160517722,"duration_ms":22852582,"input_tokens":496877404,"output_tokens":23429182,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"820d07bf-82ea-4f46-ad53-52fc2a2452ed","workflow_id":"rbf-dswe-haiku55-lo","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T22:18:56.174Z","resumed_from_workflow_id":null,"config_key":"a1970f2cb9989dca6dbb2fc5889b7a18","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.40708,"total_questions":113,"correct_answers":46,"total_cost":15.440723,"generation_time_ms":109872322,"duration_ms":19438174,"input_tokens":329871531,"output_tokens":12742228,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"043616b5-e3ec-46f2-b671-99c1c2512ac7","workflow_id":"rbf-qa-gem38f-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"google/gemini-3.8-flash","provider_name":"OpenRouter","timestamp":"2026-10-09T22:18:47.881Z","resumed_from_workflow_id":null,"config_key":"984b97255a508e21490deced28500655","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.435484,"total_questions":124,"correct_answers":54,"total_cost":129.543175,"generation_time_ms":72943244,"duration_ms":19421881,"input_tokens":831543000,"output_tokens":2782098,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"29e6e7b1-1fff-4eb0-b5e6-3649b7d2391b","workflow_id":"rbf-dswe-grok47-md","benchmark_type":"deep_swe","model_permaslug":"x-ai/grok-4.7","provider_name":"OpenRouter","timestamp":"2026-10-09T22:11:30.941Z","resumed_from_workflow_id":null,"config_key":"7f8b39971bbbe2f883cb7a13b634134b","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.353982,"total_questions":113,"correct_answers":40,"total_cost":665.622048,"generation_time_ms":82819085,"duration_ms":18993941,"input_tokens":874818702,"output_tokens":4143443,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"80fb5b08-5121-4a42-9f86-67b87bd92a57","workflow_id":"rbf-dswe-luna-hi","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-10-09T21:44:34.581Z","resumed_from_workflow_id":null,"config_key":"09944ac27ca572c06ab3376341668b02","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.530973,"total_questions":113,"correct_answers":60,"total_cost":12.477907,"generation_time_ms":107102903,"duration_ms":17374581,"input_tokens":583086014,"output_tokens":7661080,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9b8ed6a7-d49a-429e-b655-d7ad6f67c860","workflow_id":"rbf-dswe-dsv41f-hi","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"OpenRouter","timestamp":"2026-10-09T21:42:55.993Z","resumed_from_workflow_id":null,"config_key":"bfb5af1dc0240f434ee674efde542166","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.628319,"total_questions":113,"correct_answers":71,"total_cost":42.249287,"generation_time_ms":134893994,"duration_ms":17281993,"input_tokens":1864999745,"output_tokens":12554120,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"d3e51c51-c841-4b55-99ee-ec5034c9247f","workflow_id":"rbf-qa-sol61-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6.1-sol","provider_name":"OpenRouter","timestamp":"2026-10-09T21:42:19.122Z","resumed_from_workflow_id":null,"config_key":"8ad3c59663b9f093d28f8b4f901107cf","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.596774,"total_questions":124,"correct_answers":74,"total_cost":48.198367,"generation_time_ms":43272701,"duration_ms":17229122,"input_tokens":94912795,"output_tokens":1646290,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"ec109e61-cb19-4f32-9828-4a86d0609bbe","workflow_id":"rbf-qa-sol61-md","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6.1-sol","provider_name":"OpenRouter","timestamp":"2026-10-09T21:42:13.934Z","resumed_from_workflow_id":null,"config_key":"993476a4c2cd9f73b7df6d806262a9e2","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.604839,"total_questions":124,"correct_answers":75,"total_cost":27.169469,"generation_time_ms":27316281,"duration_ms":17221934,"input_tokens":45826603,"output_tokens":909432,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"3e8e5d30-84e3-46b0-a5e3-25403fd29797","workflow_id":"rbf-qa-grok47-md","benchmark_type":"swe_atlas_qa","model_permaslug":"x-ai/grok-4.7","provider_name":"OpenRouter","timestamp":"2026-10-09T21:38:26.272Z","resumed_from_workflow_id":null,"config_key":"b38dec7a04b7a719f6c7d98695d6637e","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.58871,"total_questions":124,"correct_answers":73,"total_cost":178.040576,"generation_time_ms":39176084,"duration_ms":16999272,"input_tokens":257795555,"output_tokens":1951611,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"7dd58fa3-bff2-4cc6-acb7-7f09901283af","workflow_id":"rbf-qa-haiku55-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:38:15.670Z","resumed_from_workflow_id":null,"config_key":"6428bcf38a6cbd51723d90e966090a27","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.435484,"total_questions":124,"correct_answers":54,"total_cost":5.896325,"generation_time_ms":42042884,"duration_ms":16987670,"input_tokens":184693025,"output_tokens":4279240,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9f47da29-69dd-4d93-b9b4-de06222f0121","workflow_id":"rbf-qa-sonnet55-hi","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:29:42.871Z","resumed_from_workflow_id":null,"config_key":"eb7f4b130a53595f753cc061759219a8","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.483871,"total_questions":124,"correct_answers":60,"total_cost":32.474459,"generation_time_ms":23899413,"duration_ms":16470871,"input_tokens":63411963,"output_tokens":1402405,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"bf00c532-57e0-44a9-8094-44995e0817c0","workflow_id":"rbf-qa-sonnet55-lo","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:27:32.959Z","resumed_from_workflow_id":null,"config_key":"7bfe4c264f7ac7808a7502393fee3638","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.201613,"total_questions":124,"correct_answers":25,"total_cost":8.610245,"generation_time_ms":8484784,"duration_ms":16339959,"input_tokens":8411336,"output_tokens":435681,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"46d4d528-38f4-4ae9-b06c-8459663d4bb4","workflow_id":"rbf-qa-haiku55-md","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:26:33.667Z","resumed_from_workflow_id":null,"config_key":"887e64e7d037e033a2ef524906ad5570","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.314516,"total_questions":124,"correct_answers":39,"total_cost":1.868429,"generation_time_ms":18929614,"duration_ms":16284667,"input_tokens":57112055,"output_tokens":1631530,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"d38dc0bb-76bc-45fc-ba51-50e7e77fb84b","workflow_id":"rbf-qa-sonnet55-md","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:18:43.619Z","resumed_from_workflow_id":null,"config_key":"555530d8b781322207b8e8aae68c129a","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.330645,"total_questions":124,"correct_answers":41,"total_cost":12.150018,"generation_time_ms":11029897,"duration_ms":15809619,"input_tokens":14820198,"output_tokens":591634,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"d4ab0e1b-843b-4824-9af4-4e324112ea0b","workflow_id":"rbf-qa-sol61-lo","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6.1-sol","provider_name":"OpenRouter","timestamp":"2026-10-09T21:15:20.892Z","resumed_from_workflow_id":null,"config_key":"25e2fc07b96f0dee18566fbb32c2e708","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.516129,"total_questions":124,"correct_answers":64,"total_cost":15.818023,"generation_time_ms":19320743,"duration_ms":15609892,"input_tokens":20945097,"output_tokens":578484,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"0c5e93d8-3ee1-4155-bf0e-a10a00dfce1b","workflow_id":"rbf-qa-haiku55-lo","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-haiku-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T21:00:13.172Z","resumed_from_workflow_id":null,"config_key":"e7e53ac58ec81dec6d77f3bf8ad7d108","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.241935,"total_questions":124,"correct_answers":30,"total_cost":0.895186,"generation_time_ms":10842289,"duration_ms":14704172,"input_tokens":20222996,"output_tokens":837691,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"c7c7b27a-b97e-4588-bd52-6393fcf58779","workflow_id":"rbf-dswe-astra-hi","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-astra","provider_name":"OpenRouter","timestamp":"2026-10-09T20:04:35.935Z","resumed_from_workflow_id":null,"config_key":"6bc013c788f272191246c18a088b58e2","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.734513,"total_questions":113,"correct_answers":83,"total_cost":337.859193,"generation_time_ms":79210781,"duration_ms":11383935,"input_tokens":123639195,"output_tokens":2636227,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"69e65156-9787-405f-bbc1-f40cbbcfa844","workflow_id":"rbf-dswe-sonnet55-md","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T19:24:51.820Z","resumed_from_workflow_id":null,"config_key":"409a0f188e4e539d2dcb41624b987570","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.495575,"total_questions":113,"correct_answers":56,"total_cost":57.814809,"generation_time_ms":47733608,"duration_ms":8988820,"input_tokens":81388596,"output_tokens":3907023,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"9ac28f70-e26b-45f2-9611-16985fcd1228","workflow_id":"rbf-dswe-sonnet55-lo","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-sonnet-5.5","provider_name":"OpenRouter","timestamp":"2026-10-09T19:13:05.143Z","resumed_from_workflow_id":null,"config_key":"9b1d13ca458af55c00ad0af34997b8fb","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.539823,"total_questions":113,"correct_answers":61,"total_cost":40.117314,"generation_time_ms":33282129,"duration_ms":8282143,"input_tokens":64652931,"output_tokens":2479685,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"8b0e834a-8c57-400d-9d39-1fcb36dd664e","workflow_id":"rbf-dswe-sol61-lo","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6.1-sol","provider_name":"OpenRouter","timestamp":"2026-10-09T18:31:05.307Z","resumed_from_workflow_id":null,"config_key":"de6c8f131688cdc78c154f28b0ae2e8b","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.60177,"total_questions":113,"correct_answers":68,"total_cost":15.359054,"generation_time_ms":24945925,"duration_ms":5764307,"input_tokens":27151159,"output_tokens":670838,"reasoning_tokens":null,"success":true,"baseline":true},{"id":"bee19db1-fbec-4247-b423-a9f8bb39383f","workflow_id":"rb-base3-1791424530-deep-swe-dsv41flash-r1.deepseek/deepseek-v4.1-flash.auto-routing","benchmark_type":"deep_swe","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"auto-routing","timestamp":"2026-10-08T21:33:36.641Z","resumed_from_workflow_id":"rb-base3-1791424530-deep-swe-dsv41flash.deepseek/deepseek-v4.1-flash.auto-routing","config_key":"be64be36b5760a38b6d6c43f92ea666f","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.619469,"total_questions":113,"correct_answers":70,"total_cost":47.369685,"generation_time_ms":186746875,"duration_ms":12939166,"input_tokens":2893563774,"output_tokens":18530310,"reasoning_tokens":14755412,"success":true,"baseline":true},{"id":"9fb64b24-ab88-4ac9-8079-3a4a75e8fab6","workflow_id":"rb-base3-1791424530-mmmu-pro-vision-dsv41flash.deepseek/deepseek-v4.1-flash.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.452Z","resumed_from_workflow_id":null,"config_key":"44cddfa78584772c12077796950fd5c2","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.682081,"total_questions":1730,"correct_answers":1180,"total_cost":27.704963,"generation_time_ms":219006489,"duration_ms":14023374,"input_tokens":1763806,"output_tokens":35883925,"reasoning_tokens":35362331,"success":true,"baseline":true},{"id":"84c72119-87f7-427a-bbc9-6cdb4af207a8","workflow_id":"rb-base3-1791424530-mmmu-pro-vision-gpt6luna.openai/gpt-6-luna.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.447Z","resumed_from_workflow_id":null,"config_key":"d23c6e7120417ac946e8ede97d646166","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.77341,"total_questions":1730,"correct_answers":1338,"total_cost":3.653491,"generation_time_ms":65062649,"duration_ms":4179063,"input_tokens":6436417,"output_tokens":5708300,"reasoning_tokens":5631910,"success":true,"baseline":true},{"id":"68eecb64-c2c7-47e2-81c4-82d4556fa85c","workflow_id":"rb-base3-1791424530-swe-atlas-qa-gpt6luna.openai/gpt-6-luna.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.440Z","resumed_from_workflow_id":null,"config_key":"79c45eb112559e7daa480444659722a2","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.427419,"total_questions":124,"correct_answers":53,"total_cost":7.972291,"generation_time_ms":57699545,"duration_ms":9050163,"input_tokens":292890853,"output_tokens":5470861,"reasoning_tokens":4117299,"success":true,"baseline":true},{"id":"bb3d190f-3507-4c45-abfd-36aee32da148","workflow_id":"rb-base3-1791424530-deep-swe-gpt6luna.openai/gpt-6-luna.auto-routing","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.436Z","resumed_from_workflow_id":null,"config_key":"15a8905171c21a005d427e8a7bb4a8c0","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.672566,"total_questions":113,"correct_answers":76,"total_cost":20.735924,"generation_time_ms":119781326,"duration_ms":26073578,"input_tokens":1028868484,"output_tokens":11426982,"reasoning_tokens":8669837,"success":true,"baseline":true},{"id":"a1a2d0ad-242a-485a-90fc-eead69c07a26","workflow_id":"rb-base3-1791424530-tau3-bench-banking-dsv41flash.deepseek/deepseek-v4.1-flash.auto-routing","benchmark_type":"tau3_bench_banking","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.435Z","resumed_from_workflow_id":null,"config_key":"e18e58332a2f68cc476dc54a2d805cc8","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.360825,"total_questions":97,"correct_answers":35,"total_cost":19.451398,"generation_time_ms":104362787,"duration_ms":16704229,"input_tokens":488451118,"output_tokens":11792847,"reasoning_tokens":10866497,"success":true,"baseline":true},{"id":"3d514bb8-042c-48bb-92e1-6a91eec3e472","workflow_id":"rb-base3-1791424530-swe-atlas-qa-dsv41flash.deepseek/deepseek-v4.1-flash.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"deepseek/deepseek-v4.1-flash","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.434Z","resumed_from_workflow_id":null,"config_key":"0a2b47d25f6a809b3b668e9974e61246","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.653226,"total_questions":124,"correct_answers":81,"total_cost":20.312854,"generation_time_ms":94699810,"duration_ms":14977278,"input_tokens":911271560,"output_tokens":7951664,"reasoning_tokens":5191315,"success":true,"baseline":true},{"id":"65bddcfd-c3e6-4a97-999b-51e74729106b","workflow_id":"rb-base3-1791424530-tau3-bench-banking-gpt6luna.openai/gpt-6-luna.auto-routing","benchmark_type":"tau3_bench_banking","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-10-08T13:29:57.433Z","resumed_from_workflow_id":null,"config_key":"e5de429ab9fe68229c52e0c0c3c7c080","reasoning_effort":"max","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.309278,"total_questions":97,"correct_answers":30,"total_cost":4.427173,"generation_time_ms":22213388,"duration_ms":2749387,"input_tokens":166979163,"output_tokens":2312739,"reasoning_tokens":2086050,"success":true,"baseline":true},{"id":"2dd77b4b-70ae-4396-b583-c1104c614bef","workflow_id":"rb-base3-1791424530-deep-swe-mimo26pro-r1.xiaomi/mimo-v2.6-pro.auto-routing","benchmark_type":"deep_swe","model_permaslug":"xiaomi/mimo-v2.6-pro","provider_name":"auto-routing","timestamp":"2026-10-08T10:18:48.374Z","resumed_from_workflow_id":"rb-base3-1791424530-deep-swe-mimo26pro.xiaomi/mimo-v2.6-pro.auto-routing","config_key":"b35c2be8f2a82d631bbf3a2cfbd29c50","reasoning_effort":"xhigh","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.584071,"total_questions":113,"correct_answers":66,"total_cost":34.680837,"generation_time_ms":293113982,"duration_ms":25672354,"input_tokens":737731003,"output_tokens":10993322,"reasoning_tokens":8508338,"success":true,"baseline":true},{"id":"52dd3ac7-cb82-497b-afe9-8f16e6ce4d39","workflow_id":"rb-base3-1791424530-mmmu-pro-vision-mimo26pro.xiaomi/mimo-v2.6-pro.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"xiaomi/mimo-v2.6-pro","provider_name":"auto-routing","timestamp":"2026-10-08T01:55:44.174Z","resumed_from_workflow_id":null,"config_key":"813ba3188272d80e99cd8bc45cf4e0e0","reasoning_effort":"xhigh","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.636416,"total_questions":1730,"correct_answers":1101,"total_cost":18.455067,"generation_time_ms":263650946,"duration_ms":11624646,"input_tokens":4707391,"output_tokens":18940270,"reasoning_tokens":16457981,"success":true,"baseline":true},{"id":"46ba2d52-b9a7-4ed8-9478-f8fec006de2f","workflow_id":"rb-base3-1791424530-tau3-bench-banking-mimo26pro.xiaomi/mimo-v2.6-pro.auto-routing","benchmark_type":"tau3_bench_banking","model_permaslug":"xiaomi/mimo-v2.6-pro","provider_name":"auto-routing","timestamp":"2026-10-08T01:55:44.171Z","resumed_from_workflow_id":null,"config_key":"4f2903df29f7613a80cd2019f26e84ed","reasoning_effort":"xhigh","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.42268,"total_questions":97,"correct_answers":41,"total_cost":12.010454,"generation_time_ms":65593602,"duration_ms":7808855,"input_tokens":259669729,"output_tokens":1797689,"reasoning_tokens":1522135,"success":true,"baseline":true},{"id":"b80d8187-03db-490c-aa8a-2ffd8a385d4a","workflow_id":"rb-base3-1791424530-swe-atlas-qa-mimo26pro.xiaomi/mimo-v2.6-pro.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"xiaomi/mimo-v2.6-pro","provider_name":"auto-routing","timestamp":"2026-10-08T01:55:44.169Z","resumed_from_workflow_id":null,"config_key":"4bfff29082a34d9d27a6500c7822451e","reasoning_effort":"xhigh","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.540323,"total_questions":124,"correct_answers":67,"total_cost":19.06682,"generation_time_ms":151914413,"duration_ms":19644112,"input_tokens":319625795,"output_tokens":5547845,"reasoning_tokens":3829011,"success":true,"baseline":true},{"id":"6ad30245-0b7d-4273-b0e5-67a4afc1fec4","workflow_id":"rb-tb4-pilot-luna-1002b","benchmark_type":"terminal-bench-4.0","model_permaslug":"openai/gpt-6-luna","provider_name":"OpenRouter","timestamp":"2026-10-03T08:08:01.519Z","resumed_from_workflow_id":null,"config_key":"16bf772a5ce7df72422dc7c61d3c4f48","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.047619,"total_questions":63,"correct_answers":3,"total_cost":5.798473,"generation_time_ms":154283903,"duration_ms":37446519,"input_tokens":97275,"output_tokens":3723340,"reasoning_tokens":null,"success":false,"baseline":true},{"id":"165ff2c9-aea9-4379-91f3-b6461a97e953","workflow_id":"rb-tb4-pilot-pareto2610p-e2-1002b","benchmark_type":"terminal-bench-4.0","model_permaslug":"unbiased/pareto-26.10-preview","provider_name":"OpenRouter","timestamp":"2026-10-03T07:00:05.466Z","resumed_from_workflow_id":null,"config_key":"02867826243ed42ad8ee880631548be9","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.349206,"total_questions":63,"correct_answers":22,"total_cost":36.005788,"generation_time_ms":190777666,"duration_ms":33368466,"input_tokens":7190621,"output_tokens":6672002,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"2c2c7586-3f68-4a9e-87db-9b82c71dc61a","workflow_id":"rb-tb4-pilot-pareto2610p-1002a","benchmark_type":"terminal-bench-4.0","model_permaslug":"unbiased/pareto-26.10-preview","provider_name":"OpenRouter","timestamp":"2026-10-02T22:07:34.606Z","resumed_from_workflow_id":null,"config_key":"3ceb13a59b4b87d1ea7c99ded6a4a8ef","reasoning_effort":null,"max_tokens":null,"cost_tier":null,"route_models":null,"temperature":null,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.349206,"total_questions":63,"correct_answers":22,"total_cost":32.880371,"generation_time_ms":106878465,"duration_ms":22357606,"input_tokens":7580670,"output_tokens":5909183,"reasoning_tokens":null,"success":true,"baseline":false},{"id":"26d529c2-f673-429e-ad3f-a67732034c6e","workflow_id":"rb-sy-mmmu-1790861068-full-dsflash-sol.nvidia/switchyard.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-10-01T13:34:10.229Z","resumed_from_workflow_id":null,"config_key":"1a75f7c33fadf257f8954b4ddd0c040b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["deepseek/deepseek-v4.1-flash","openai/gpt-6-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.769942,"total_questions":1730,"correct_answers":1332,"total_cost":19.301089,"generation_time_ms":72733589,"duration_ms":8962658,"input_tokens":4530162,"output_tokens":9568749,"reasoning_tokens":8807756,"success":true,"baseline":false},{"id":"253454d5-578f-4d26-be7e-da29e9ad4ed5","workflow_id":"rb-sy-mmmu-1790861068-full-sol-astra.nvidia/switchyard.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-10-01T13:34:10.227Z","resumed_from_workflow_id":null,"config_key":"7695125d93e259acefc31bd6712c4a26","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-sol","openai/gpt-6-astra"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.824277,"total_questions":1730,"correct_answers":1426,"total_cost":70.356625,"generation_time_ms":15001942,"duration_ms":1587004,"input_tokens":6436417,"output_tokens":506800,"reasoning_tokens":416692,"success":true,"baseline":false},{"id":"cefeb9ce-677f-4563-b69d-148569e49126","workflow_id":"rb-sy-mmmu-1790861068-full-glm-opus.nvidia/switchyard.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-10-01T13:34:10.202Z","resumed_from_workflow_id":null,"config_key":"70e4f63814209a40cc33c7be819f8f87","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3-flash","anthropic/claude-opus-5.5"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.794798,"total_questions":1730,"correct_answers":1375,"total_cost":29.445201,"generation_time_ms":39152093,"duration_ms":4928727,"input_tokens":5949345,"output_tokens":1959001,"reasoning_tokens":1193205,"success":true,"baseline":false},{"id":"e400871e-5a2d-44ca-b95d-e0f989bb5f27","workflow_id":"rb-qa-1790778500-swe-atlas-qa-opus55-medium.anthropic/claude-opus-5.5.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.432Z","resumed_from_workflow_id":null,"config_key":"02fa8a90fe1a4904d374ed26564d5fbe","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.540323,"total_questions":124,"correct_answers":67,"total_cost":59.31391,"generation_time_ms":20942033,"duration_ms":6947916,"input_tokens":63448374,"output_tokens":1292646,"reasoning_tokens":228826,"success":true,"baseline":true},{"id":"14b062db-0366-4b7a-9bf7-70a7034f4dde","workflow_id":"rb-qa-1790778500-swe-atlas-qa-astra-medium.openai/gpt-6-astra.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.432Z","resumed_from_workflow_id":null,"config_key":"8abc761524a2b079cdcec8a62acfa369","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.588235,"total_questions":119,"correct_answers":70,"total_cost":193.575272,"generation_time_ms":29088950,"duration_ms":5413037,"input_tokens":60007763,"output_tokens":1223093,"reasoning_tokens":166292,"success":true,"baseline":true},{"id":"f3f508d0-04ac-49ae-8530-6bd639ea5e24","workflow_id":"rb-qa-1790778500-swe-atlas-qa-astra-low.openai/gpt-6-astra.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.430Z","resumed_from_workflow_id":null,"config_key":"ad5f63c9b1b1f1ca6b613bcb8e7cd819","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.540984,"total_questions":122,"correct_answers":66,"total_cost":108.104232,"generation_time_ms":19075250,"duration_ms":4365614,"input_tokens":26476751,"output_tokens":764146,"reasoning_tokens":37056,"success":true,"baseline":true},{"id":"db24ae71-a261-4bac-b7a5-ed227d540cd2","workflow_id":"rb-qa-1790778500-swe-atlas-qa-sol61-high.openai/gpt-6.1-sol.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6.1-sol","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.429Z","resumed_from_workflow_id":null,"config_key":"05c115d4ef54c739238702fdc5cca9b5","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.62963,"total_questions":54,"correct_answers":34,"total_cost":17.817865,"generation_time_ms":13762698,"duration_ms":3512451,"input_tokens":30020628,"output_tokens":666645,"reasoning_tokens":137416,"success":true,"baseline":true},{"id":"864edc74-f2fb-4ab3-86a4-adade5ecca27","workflow_id":"rb-qa-1790778500-swe-atlas-qa-opus55-low.anthropic/claude-opus-5.5.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.428Z","resumed_from_workflow_id":null,"config_key":"3cd9ec509a659b275448cd4227acaa7a","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.395161,"total_questions":124,"correct_answers":49,"total_cost":20.32742,"generation_time_ms":8255879,"duration_ms":3017225,"input_tokens":11210200,"output_tokens":509970,"reasoning_tokens":38550,"success":true,"baseline":true},{"id":"a92c5729-22fc-4a95-a68c-12b8907f296f","workflow_id":"rb-qa-1790778500-swe-atlas-qa-sol61-medium.openai/gpt-6.1-sol.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"openai/gpt-6.1-sol","provider_name":"auto-routing","timestamp":"2026-09-30T14:30:41.408Z","resumed_from_workflow_id":null,"config_key":"7fa645808a4ef5c4d9b3afcd270c45f9","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.541667,"total_questions":72,"correct_answers":39,"total_cost":16.917425,"generation_time_ms":12607795,"duration_ms":3405369,"input_tokens":27688050,"output_tokens":628247,"reasoning_tokens":67082,"success":true,"baseline":true},{"id":"e0a1ea07-b878-4262-bf81-24ce2fc49f73","workflow_id":"rb-base-1790743549-mmmu-pro-vision-opus55.anthropic/claude-opus-5.5.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.896Z","resumed_from_workflow_id":null,"config_key":"9f0bf5d586ef5f7bacca47943d86c700","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.818497,"total_questions":1730,"correct_answers":1416,"total_cost":45.630146,"generation_time_ms":15628203,"duration_ms":796568,"input_tokens":5511760,"output_tokens":904450,"reasoning_tokens":445852,"success":true,"baseline":true},{"id":"439f3459-857e-4f99-a1f6-aeb935d4fed9","workflow_id":"rb-base-1790743549-mmmu-pro-vision-sol.openai/gpt-6-sol.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"openai/gpt-6-sol","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.886Z","resumed_from_workflow_id":null,"config_key":"53ed5b5a9a6247fe33785b143dbc819f","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.815029,"total_questions":1730,"correct_answers":1410,"total_cost":23.078274,"generation_time_ms":11934330,"duration_ms":937651,"input_tokens":6436417,"output_tokens":709088,"reasoning_tokens":644704,"success":true,"baseline":true},{"id":"8f2c03b2-137b-499a-9fc7-29876393c97f","workflow_id":"rb-base-1790743549-mmmu-pro-vision-luna.openai/gpt-6-luna.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"openai/gpt-6-luna","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.882Z","resumed_from_workflow_id":null,"config_key":"8ea38502a7335b299bbc9c18483384f3","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.766474,"total_questions":1730,"correct_answers":1326,"total_cost":1.533945,"generation_time_ms":14430514,"duration_ms":1066816,"input_tokens":6436417,"output_tokens":1467735,"reasoning_tokens":1397331,"success":true,"baseline":true},{"id":"68de8b60-3ca5-4e0e-9f58-f2a48f69cc32","workflow_id":"rb-base-1790743549-mmmu-pro-vision-kimik3.moonshotai/kimi-k3.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"moonshotai/kimi-k3","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.882Z","resumed_from_workflow_id":null,"config_key":"e93116204b255297476398f2ff1476be","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.809827,"total_questions":1730,"correct_answers":1401,"total_cost":35.533513,"generation_time_ms":152700961,"duration_ms":7477425,"input_tokens":7142136,"output_tokens":3556659,"reasoning_tokens":3169929,"success":true,"baseline":true},{"id":"96f53548-c13a-4569-8718-dbe9c9c398cf","workflow_id":"rb-base-1790743549-mmmu-pro-vision-glm53flash.z-ai/glm-5.3-flash.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"z-ai/glm-5.3-flash","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.881Z","resumed_from_workflow_id":null,"config_key":"2433539edcfad3d031d76e6a0266e679","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.754335,"total_questions":1730,"correct_answers":1305,"total_cost":1.381525,"generation_time_ms":57606584,"duration_ms":7822715,"input_tokens":6801788,"output_tokens":2464645,"reasoning_tokens":1729364,"success":true,"baseline":true},{"id":"92f2552b-456f-458e-a721-887e08d0dfd4","workflow_id":"rb-base-1790743549-mmmu-pro-vision-astra.openai/gpt-6-astra.auto-routing","benchmark_type":"mmmu_pro_vision","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-30T04:48:34.841Z","resumed_from_workflow_id":null,"config_key":"1bcac355680345eb579876033cbc541f","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.841618,"total_questions":1730,"correct_answers":1456,"total_cost":101.183043,"generation_time_ms":13537890,"duration_ms":978797,"input_tokens":6436417,"output_tokens":423424,"reasoning_tokens":311954,"success":true,"baseline":true},{"id":"ac6d97f6-0d95-4a42-9441-463cc11833e5","workflow_id":"rb-sy-pass-sol61-high-1790737700-full-swe-atlas-qa.nvidia/switchyard.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-30T03:18:28.501Z","resumed_from_workflow_id":null,"config_key":"4e0b3f1268cb0b729844b7129b9d3d99","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6.1-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.601695,"total_questions":118,"correct_answers":71,"total_cost":43.847998,"generation_time_ms":30486582,"duration_ms":7989789,"input_tokens":79757962,"output_tokens":1613620,"reasoning_tokens":334110,"success":true,"baseline":false},{"id":"1f3287f6-c1a0-489b-bdf1-e1c00b8fd4be","workflow_id":"rb-sy-pass-sol61-high-1790737700-full-tau3-bench-banking.nvidia/switchyard.auto-routing","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-30T03:11:21.814Z","resumed_from_workflow_id":null,"config_key":"0a75ec775e27ab7686c1aa2e914af5e4","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6.1-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.4,"total_questions":90,"correct_answers":36,"total_cost":26.809734,"generation_time_ms":8498550,"duration_ms":1171786,"input_tokens":93263889,"output_tokens":284784,"reasoning_tokens":100557,"success":true,"baseline":false},{"id":"31c50c0b-7c9f-4e9d-904f-6849c66c6227","workflow_id":"rb-sy-pass-sol61-high-1790729500-full-deep-swe.nvidia/switchyard.auto-routing","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-30T00:42:44.198Z","resumed_from_workflow_id":null,"config_key":"aeb39abdf82d1d1707d6337620a857d9","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6.1-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.707965,"total_questions":113,"correct_answers":80,"total_cost":52.245723,"generation_time_ms":44058820,"duration_ms":13541837,"input_tokens":96791231,"output_tokens":2552281,"reasoning_tokens":793577,"success":true,"baseline":false},{"id":"5ca9f27c-7b06-41b5-bb8f-e8b98b74b8f9","workflow_id":"deep-swe-sol61-openai-medium-e2-1790706000.openai/gpt-6.1-sol.openai","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6.1-sol","provider_name":"openai","timestamp":"2026-09-29T18:17:12.848Z","resumed_from_workflow_id":null,"config_key":"810fb60256d1c10095f0c4ea83ca2e74","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.725664,"total_questions":226,"correct_answers":164,"total_cost":67.074024,"generation_time_ms":50021715,"duration_ms":11459853,"input_tokens":118327046,"output_tokens":3191376,"reasoning_tokens":632372,"success":true,"baseline":true},{"id":"951ff29e-0de5-4af5-bcc5-b2316cc82e12","workflow_id":"rb-1790696055-deep-swe-opus55-medium-e2.anthropic/claude-opus-5.5.auto-routing","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-09-29T15:34:16.831Z","resumed_from_workflow_id":null,"config_key":"cc3a5efbe7464f90b54787fe91f9a1d3","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.676991,"total_questions":226,"correct_answers":153,"total_cost":348.041627,"generation_time_ms":94407617,"duration_ms":19716984,"input_tokens":508547299,"output_tokens":7997752,"reasoning_tokens":2761757,"success":true,"baseline":true},{"id":"e9638340-ea2a-46ec-b933-103b89eef570","workflow_id":"rb-1790696055-deep-swe-astra-medium-e2.openai/gpt-6-astra.auto-routing","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-29T15:34:16.829Z","resumed_from_workflow_id":null,"config_key":"8d5da09fb096e407e1defa0fbb54531e","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.699115,"total_questions":226,"correct_answers":158,"total_cost":450.216468,"generation_time_ms":77974058,"duration_ms":25755945,"input_tokens":133745585,"output_tokens":3797739,"reasoning_tokens":871986,"success":true,"baseline":true},{"id":"12ee805d-84e6-4c96-933b-8e0f5c017802","workflow_id":"rb-1790696055-deep-swe-astra-low-e4.openai/gpt-6-astra.auto-routing","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-29T15:34:16.453Z","resumed_from_workflow_id":null,"config_key":"57081aa848898977cae30c0f8b2921e9","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":4,"recorded_epochs":4,"accuracy":0.671875,"total_questions":448,"correct_answers":301,"total_cost":457.699549,"generation_time_ms":83805171,"duration_ms":21491652,"input_tokens":122841742,"output_tokens":3875306,"reasoning_tokens":389507,"success":true,"baseline":true},{"id":"e69161b6-6d72-4b75-bb44-ab9858893052","workflow_id":"rb-sy-stage-20260928-pilot-glm-opus-deep-swe.nvidia/switchyard.auto-routing","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-28T14:56:18.246Z","resumed_from_workflow_id":null,"config_key":"7f3ebf67d6d6b3633964becb16825dc4","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3-flash","anthropic/claude-opus-5.5"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.734513,"total_questions":113,"correct_answers":83,"total_cost":278.709375,"generation_time_ms":117868130,"duration_ms":26270177,"input_tokens":546734940,"output_tokens":7461590,"reasoning_tokens":3475562,"success":true,"baseline":false},{"id":"19104d15-17a1-451a-a351-b01b0e6b0f82","workflow_id":"rb-sy-stage-20260928-pilot-sol-astra-deep-swe.nvidia/switchyard.auto-routing","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-28T14:56:08.188Z","resumed_from_workflow_id":null,"config_key":"c5c095f2f9d4551ee086d7d8d2088b57","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-sol","openai/gpt-6-astra"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.707965,"total_questions":113,"correct_answers":80,"total_cost":265.411643,"generation_time_ms":57563169,"duration_ms":9155644,"input_tokens":105438411,"output_tokens":2607101,"reasoning_tokens":845881,"success":true,"baseline":false},{"id":"5f638962-30e9-4494-a673-21ae12f56604","workflow_id":"rb-sy-stage-20260928-pilot-glm-opus-swe-atlas-qa.nvidia/switchyard.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-28T14:50:11.135Z","resumed_from_workflow_id":null,"config_key":"162b502d132b25d978db76fa5f650fc1","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3-flash","anthropic/claude-opus-5.5"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.596774,"total_questions":124,"correct_answers":74,"total_cost":108.075767,"generation_time_ms":52644430,"duration_ms":22891503,"input_tokens":197824631,"output_tokens":2469204,"reasoning_tokens":668104,"success":true,"baseline":false},{"id":"9b215542-7ea9-46d9-96be-b695ad544180","workflow_id":"rb-sy-stage-20260928-pilot-dsflash-sol-swe-atlas-qa.nvidia/switchyard.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-28T14:49:01.614Z","resumed_from_workflow_id":null,"config_key":"5fb554833e024a5fb22387ec3604e517","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["deepseek/deepseek-v4.1-flash","openai/gpt-6-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.528926,"total_questions":121,"correct_answers":64,"total_cost":43.455474,"generation_time_ms":27198750,"duration_ms":6816026,"input_tokens":83033103,"output_tokens":1099795,"reasoning_tokens":248245,"success":true,"baseline":false},{"id":"dabc46f0-b52d-44b8-aaac-76ad9efd0741","workflow_id":"rb-sy-stage-20260928-pilot-sol-astra-swe-atlas-qa.nvidia/switchyard.auto-routing","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-28T14:48:57.925Z","resumed_from_workflow_id":null,"config_key":"08e3d4bafcecc2fbd328bb6b05c4d6d5","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-sol","openai/gpt-6-astra"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.661157,"total_questions":121,"correct_answers":80,"total_cost":239.469606,"generation_time_ms":38268416,"duration_ms":6037318,"input_tokens":84678797,"output_tokens":1594949,"reasoning_tokens":359038,"success":true,"baseline":false},{"id":"26d76c5c-87ae-429b-8a70-b9a954a3fd31","workflow_id":"rb-sy-stage-20260928-pilot-dsflash-sol-deep-swe.nvidia/switchyard.auto-routing","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-28T14:48:57.081Z","resumed_from_workflow_id":null,"config_key":"cc377abc81409f491389b7b6ca5b64ac","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["deepseek/deepseek-v4.1-flash","openai/gpt-6-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.654867,"total_questions":113,"correct_answers":74,"total_cost":54.609263,"generation_time_ms":42740952,"duration_ms":27691799,"input_tokens":111542190,"output_tokens":1869373,"reasoning_tokens":566533,"success":true,"baseline":false},{"id":"33e518fd-139e-4198-9f7b-c0f61286833b","workflow_id":"rb-sy-stage-20260928-pilot-sol-astra-tau3-bench-banking.nvidia/switchyard.auto-routing","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-28T14:45:19.048Z","resumed_from_workflow_id":null,"config_key":"abb54e67630b27ee43768b467abf9aef","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-sol","openai/gpt-6-astra"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.42268,"total_questions":97,"correct_answers":41,"total_cost":180.814433,"generation_time_ms":15217797,"duration_ms":2280052,"input_tokens":117746882,"output_tokens":382096,"reasoning_tokens":176463,"success":true,"baseline":false},{"id":"8ff712d6-b234-4c43-a925-652dd9fe6397","workflow_id":"rb-sy-stage-20260928-pilot-glm-opus-tau3-bench-banking.nvidia/switchyard.auto-routing","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-28T14:45:19.047Z","resumed_from_workflow_id":null,"config_key":"5ebd68c6786f9162f387885e3a56aab1","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3-flash","anthropic/claude-opus-5.5"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.381443,"total_questions":97,"correct_answers":37,"total_cost":64.555316,"generation_time_ms":21866675,"duration_ms":3502826,"input_tokens":120906477,"output_tokens":730239,"reasoning_tokens":252531,"success":true,"baseline":false},{"id":"2c05e78a-02ce-4d3b-a189-d2ff08ccd708","workflow_id":"rb-sy-stage-20260928-pilot-dsflash-sol-tau3-bench-banking.nvidia/switchyard.auto-routing","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-28T14:45:19.043Z","resumed_from_workflow_id":null,"config_key":"fa0fe735bebc6de11e3431dd92e78cb7","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["deepseek/deepseek-v4.1-flash","openai/gpt-6-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.412371,"total_questions":97,"correct_answers":40,"total_cost":57.285809,"generation_time_ms":16066067,"duration_ms":2355981,"input_tokens":148367970,"output_tokens":444285,"reasoning_tokens":225153,"success":true,"baseline":false},{"id":"8dadccd7-fcbc-4d68-b86e-25f859fa36c9","workflow_id":"rbdswe-1790443217-gpt6sol-high-full.openai/gpt-6-sol.auto-routing","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-sol","provider_name":"auto-routing","timestamp":"2026-09-26T17:43:46.354Z","resumed_from_workflow_id":null,"config_key":"bd2352da21f38302cc009c3f1405d29b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.610619,"total_questions":113,"correct_answers":69,"total_cost":51.659173,"generation_time_ms":24557267,"duration_ms":16533847,"input_tokens":103173185,"output_tokens":1806449,"reasoning_tokens":554395,"success":true,"baseline":true},{"id":"1ab5e75b-191f-487f-b90b-be9c2c46c613","workflow_id":"rbdswe-1790443217-kimik3-low-full.moonshotai/kimi-k3.auto-routing","benchmark_type":"deep_swe","model_permaslug":"moonshotai/kimi-k3","provider_name":"auto-routing","timestamp":"2026-09-26T17:33:18.660Z","resumed_from_workflow_id":null,"config_key":"23ef6ec4945cd9bfe22f2be4c9a20ac3","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.575221,"total_questions":113,"correct_answers":65,"total_cost":71.625343,"generation_time_ms":29971765,"duration_ms":17212798,"input_tokens":149978734,"output_tokens":2410766,"reasoning_tokens":1353329,"success":true,"baseline":true},{"id":"6cc62970-3874-475e-bb0a-10ef9eaa09ac","workflow_id":"deep-swe-opus-5-5-low-full-1790432600.anthropic/claude-opus-5.5.auto-routing","benchmark_type":"deep_swe","model_permaslug":"anthropic/claude-opus-5.5","provider_name":"auto-routing","timestamp":"2026-09-26T14:22:23.010Z","resumed_from_workflow_id":null,"config_key":"c2b08efd242c071b6740b9e1ad1f8a1b","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.610619,"total_questions":113,"correct_answers":69,"total_cost":58.612545,"generation_time_ms":17622610,"duration_ms":14674842,"input_tokens":62501008,"output_tokens":1438965,"reasoning_tokens":313717,"success":true,"baseline":true},{"id":"eef0f5dc-04ef-43a2-b0f5-94414dfeef25","workflow_id":"deep-swe-gpt-6-astra-low-full-1790429800.openai/gpt-6-astra.auto-routing","benchmark_type":"deep_swe","model_permaslug":"openai/gpt-6-astra","provider_name":"auto-routing","timestamp":"2026-09-26T13:36:35.897Z","resumed_from_workflow_id":null,"config_key":"57081aa848898977cae30c0f8b2921e9","reasoning_effort":"low","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.690265,"total_questions":113,"correct_answers":78,"total_cost":116.060753,"generation_time_ms":18373489,"duration_ms":3794680,"input_tokens":30706653,"output_tokens":993796,"reasoning_tokens":104484,"success":true,"baseline":true},{"id":"e5dc9b2b-9a78-42d7-a1e3-2e3f3c99b561","workflow_id":"rb-sy2-swe-atlas-qa-glm-5.3-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.492Z","resumed_from_workflow_id":null,"config_key":"65da924c80aa72e597bab844f5259b5c","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.572581,"total_questions":124,"correct_answers":71,"total_cost":79.881578,"generation_time_ms":67685398,"duration_ms":17550417,"input_tokens":390624350,"output_tokens":3621554,"reasoning_tokens":2029200,"success":true,"baseline":false},{"id":"755ac704-b820-46f8-9d83-1deee7c0c897","workflow_id":"rb-sy2-deep-swe-gpt-6-luna-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.475Z","resumed_from_workflow_id":null,"config_key":"6b6bd77e89e943b3b23a5ea919aae3d5","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-luna"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.530973,"total_questions":113,"correct_answers":60,"total_cost":7.500924,"generation_time_ms":46323089,"duration_ms":12458727,"input_tokens":393062965,"output_tokens":4442109,"reasoning_tokens":2624293,"success":true,"baseline":false},{"id":"4381aa3b-ef8c-4e56-9deb-384244a87cad","workflow_id":"rb-sy2-tau3-banking-glm-5.3-flash-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.474Z","resumed_from_workflow_id":null,"config_key":"7ad57946a6cf682846fa465e7ab177a4","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3-flash"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.371134,"total_questions":97,"correct_answers":36,"total_cost":3.500089,"generation_time_ms":23114942,"duration_ms":3085055,"input_tokens":93464243,"output_tokens":805710,"reasoning_tokens":571913,"success":true,"baseline":false},{"id":"1de7c2ab-cb68-4402-b3c1-a26c18f40f6f","workflow_id":"rb-sy2-deep-swe-glm-5.3-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.474Z","resumed_from_workflow_id":null,"config_key":"37531a6c4e5fcadb8389f451f129d3b6","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.646018,"total_questions":113,"correct_answers":73,"total_cost":165.31288,"generation_time_ms":110230755,"duration_ms":24180261,"input_tokens":805035182,"output_tokens":6760634,"reasoning_tokens":4091951,"success":true,"baseline":false},{"id":"ca155568-55e9-43fb-af49-ece5d0b7bfcd","workflow_id":"rb-sy2-deep-swe-kimi-k3-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.467Z","resumed_from_workflow_id":null,"config_key":"3adcee5e07b29bec643c70c9f65d2477","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["moonshotai/kimi-k3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.690265,"total_questions":113,"correct_answers":78,"total_cost":209.421451,"generation_time_ms":156955794,"duration_ms":27825503,"input_tokens":531865780,"output_tokens":6374976,"reasoning_tokens":4033413,"success":true,"baseline":false},{"id":"c2b5729c-774a-4436-a7ce-3dba6b62f0f2","workflow_id":"rb-sy2-deep-swe-opus-5.5-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.467Z","resumed_from_workflow_id":null,"config_key":"4269908affdb9f3e6b24d7a9f81559d2","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["anthropic/claude-opus-5.5"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.734513,"total_questions":113,"correct_answers":83,"total_cost":285.011051,"generation_time_ms":68572475,"duration_ms":11821433,"input_tokens":466458925,"output_tokens":6401253,"reasoning_tokens":2630333,"success":true,"baseline":false},{"id":"f8be8bb3-1d57-48b2-ae31-bea7bbbac01f","workflow_id":"rb-sy2-tau3-banking-kimi-k3-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.466Z","resumed_from_workflow_id":null,"config_key":"58e1a20ebc39b2870924bbeedfc4a2a4","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["moonshotai/kimi-k3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.381443,"total_questions":97,"correct_answers":37,"total_cost":79.256036,"generation_time_ms":23953672,"duration_ms":3661074,"input_tokens":174668796,"output_tokens":1352334,"reasoning_tokens":1055782,"success":true,"baseline":false},{"id":"7f942fdb-9cbd-4476-8493-ea594a1ddde2","workflow_id":"rb-sy2-deep-swe-glm-5.3-flash-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.466Z","resumed_from_workflow_id":null,"config_key":"c5be45086cc4476cd096912811ecd0a5","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3-flash"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.309735,"total_questions":113,"correct_answers":35,"total_cost":38.67259,"generation_time_ms":183324932,"duration_ms":43380141,"input_tokens":1346445400,"output_tokens":7552682,"reasoning_tokens":4354715,"success":true,"baseline":false},{"id":"ab45d8fc-8a1e-4762-b59d-70b43382c944","workflow_id":"rb-sy2-swe-atlas-qa-gpt-6-luna-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.465Z","resumed_from_workflow_id":null,"config_key":"7546a08366221a1286545dfc909dbedd","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-luna"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.354839,"total_questions":124,"correct_answers":44,"total_cost":3.378771,"generation_time_ms":20565552,"duration_ms":6814409,"input_tokens":123903528,"output_tokens":1960772,"reasoning_tokens":1114253,"success":true,"baseline":false},{"id":"74de37d5-a739-4836-b3d7-293d5086ecd8","workflow_id":"rb-sy2-deep-swe-gpt-6-astra-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.465Z","resumed_from_workflow_id":null,"config_key":"8b94f5faaf143d2deacbec01a3046dff","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-astra"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.716814,"total_questions":113,"correct_answers":81,"total_cost":303.860228,"generation_time_ms":52143513,"duration_ms":11324406,"input_tokens":92872741,"output_tokens":2581959,"reasoning_tokens":808141,"success":true,"baseline":false},{"id":"317076e4-c08f-47b3-92ba-854767ea15f4","workflow_id":"rb-sy2-tau3-banking-glm-5.3-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.463Z","resumed_from_workflow_id":null,"config_key":"f42c85bdbd6833935dd40652771dd34d","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.381443,"total_questions":97,"correct_answers":37,"total_cost":18.243153,"generation_time_ms":11102490,"duration_ms":1704176,"input_tokens":73461209,"output_tokens":594469,"reasoning_tokens":374071,"success":true,"baseline":false},{"id":"0e74d1e5-4fba-488e-90c0-9b2652b87266","workflow_id":"rb-sy2-swe-atlas-qa-gpt-6-astra-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.463Z","resumed_from_workflow_id":null,"config_key":"386401ae0243becfb10e7473827318b1","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-astra"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.655462,"total_questions":119,"correct_answers":78,"total_cost":244.95841,"generation_time_ms":34152701,"duration_ms":6629183,"input_tokens":75023099,"output_tokens":1575736,"reasoning_tokens":330588,"success":true,"baseline":false},{"id":"d8210edd-438c-4d39-a2cb-74dc156bb386","workflow_id":"rb-sy2-swe-atlas-qa-glm-5.3-flash-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.462Z","resumed_from_workflow_id":null,"config_key":"094bac727a85f033d47ef20ec62e5211","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["z-ai/glm-5.3-flash"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.427419,"total_questions":124,"correct_answers":53,"total_cost":24.573037,"generation_time_ms":112241756,"duration_ms":32969291,"input_tokens":758829338,"output_tokens":3334940,"reasoning_tokens":1147003,"success":true,"baseline":false},{"id":"8adc4422-6c33-437c-ac67-3f59c70678fc","workflow_id":"rb-sy2-swe-atlas-qa-opus-5.5-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.461Z","resumed_from_workflow_id":null,"config_key":"a8610eb8c38c2a652164d11318f98b22","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["anthropic/claude-opus-5.5"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.556452,"total_questions":124,"correct_answers":69,"total_cost":98.336004,"generation_time_ms":28300527,"duration_ms":10123391,"input_tokens":134632195,"output_tokens":1954062,"reasoning_tokens":479825,"success":true,"baseline":false},{"id":"cbc48863-9f93-4a30-b412-5ce65e506af3","workflow_id":"rb-sy2-swe-atlas-qa-gpt-6-sol-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.460Z","resumed_from_workflow_id":null,"config_key":"7564c56b45ff6a63e21f296ca0ca67a3","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.553719,"total_questions":121,"correct_answers":67,"total_cost":42.648324,"generation_time_ms":19102401,"duration_ms":6615331,"input_tokens":81392889,"output_tokens":1072116,"reasoning_tokens":239787,"success":true,"baseline":false},{"id":"b12fd7bd-e760-476d-8ac1-19cafb752451","workflow_id":"rb-sy2-tau3-banking-gpt-6-astra-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.460Z","resumed_from_workflow_id":null,"config_key":"4cf1fc0452f66150c088ad95d6c9ca16","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-astra"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.391753,"total_questions":97,"correct_answers":38,"total_cost":181.777802,"generation_time_ms":12312741,"duration_ms":1791444,"input_tokens":93104908,"output_tokens":335460,"reasoning_tokens":133169,"success":true,"baseline":false},{"id":"a4eab3ca-57bd-4808-9fd2-a1be3fdc703c","workflow_id":"rb-sy2-tau3-banking-gpt-6-luna-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.460Z","resumed_from_workflow_id":null,"config_key":"97274dcbc07fcfbd764664c6cdd25c6c","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-luna"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.247423,"total_questions":97,"correct_answers":24,"total_cost":1.964327,"generation_time_ms":10077332,"duration_ms":1558428,"input_tokens":86079711,"output_tokens":852326,"reasoning_tokens":683991,"success":true,"baseline":false},{"id":"205cab7b-cb83-4840-a668-4762f21028bc","workflow_id":"rb-sy2-tau3-banking-opus-5.5-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.460Z","resumed_from_workflow_id":null,"config_key":"443b674d7a7c693e085aab52b11af045","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["anthropic/claude-opus-5.5"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.42268,"total_questions":97,"correct_answers":41,"total_cost":71.171822,"generation_time_ms":12229173,"duration_ms":1756792,"input_tokens":121420226,"output_tokens":590819,"reasoning_tokens":191391,"success":true,"baseline":false},{"id":"fa77744f-616d-4566-a946-c19c9a86a08f","workflow_id":"rb-sy2-swe-atlas-qa-kimi-k3-e1","benchmark_type":"swe_atlas_qa","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.436Z","resumed_from_workflow_id":null,"config_key":"3a04ee35e708ab56870b65a1fc96950b","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["moonshotai/kimi-k3"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.580645,"total_questions":124,"correct_answers":72,"total_cost":107.139966,"generation_time_ms":60598317,"duration_ms":15396268,"input_tokens":239186689,"output_tokens":2934668,"reasoning_tokens":1567034,"success":true,"baseline":false},{"id":"492f9cf9-1019-4d5f-b586-0db79ed58044","workflow_id":"rb-sy2-deep-swe-gpt-6-sol-e1","benchmark_type":"deep_swe","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.431Z","resumed_from_workflow_id":null,"config_key":"c38d0de954cd7d54cd19d45520387d49","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.575221,"total_questions":113,"correct_answers":65,"total_cost":53.744126,"generation_time_ms":29321465,"duration_ms":6182622,"input_tokens":106719256,"output_tokens":1884000,"reasoning_tokens":569472,"success":true,"baseline":false},{"id":"96ba2886-4b28-4927-88ae-412bc80e267d","workflow_id":"rb-sy2-tau3-banking-gpt-6-sol-e1","benchmark_type":"tau3_bench_banking","model_permaslug":"nvidia/switchyard","provider_name":"auto-routing","timestamp":"2026-09-23T21:40:42.429Z","resumed_from_workflow_id":null,"config_key":"4ea33e0f90ee1458d47077dfe9ad0c90","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":["openai/gpt-6-sol"],"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.42268,"total_questions":97,"correct_answers":41,"total_cost":52.904438,"generation_time_ms":9137863,"duration_ms":1468524,"input_tokens":137742468,"output_tokens":437920,"reasoning_tokens":226743,"success":true,"baseline":false},{"id":"a88b0560-94d9-43aa-a331-cfd22320d324","workflow_id":"rb-7741ebdf-ext5b-fugu-max-deep-swe","benchmark_type":"deep_swe","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-18T17:14:33.283Z","resumed_from_workflow_id":null,"config_key":"966cf9b20f1df4816400a69d5baf59dd","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.339286,"total_questions":224,"correct_answers":76,"total_cost":672.333734,"generation_time_ms":341844679,"duration_ms":62962733,"input_tokens":1811999956,"output_tokens":26015609,"reasoning_tokens":19580403,"success":true,"baseline":false},{"id":"da7352d2-7e87-49c3-9df5-70b71c14dcee","workflow_id":"rb-7741ebdf-ext5b-fugu-ultra-v2-swe-atlas-qa","benchmark_type":"swe_atlas_qa","model_permaslug":"sakana/fugu-ultra-v2","provider_name":"auto-routing","timestamp":"2026-09-18T17:14:33.223Z","resumed_from_workflow_id":null,"config_key":"a26832f83753b3092f51e30f4cda26a0","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.694915,"total_questions":118,"correct_answers":82,"total_cost":1051.581245,"generation_time_ms":109569714,"duration_ms":19021856,"input_tokens":568897814,"output_tokens":7444848,"reasoning_tokens":2202523,"success":true,"baseline":false},{"id":"e654a444-a0ff-4307-b763-2d1c47cfc36c","workflow_id":"rb-7741ebdf-ext4-fugu-max-mmmu-pro-vision","benchmark_type":"mmmu_pro_vision","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-18T11:55:14.198Z","resumed_from_workflow_id":null,"config_key":"c7c88655aafd2ff3f66e9a009ddf0b1f","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.794605,"total_questions":5190,"correct_answers":4124,"total_cost":100.348075,"generation_time_ms":126036381,"duration_ms":7168366,"input_tokens":8029093,"output_tokens":14302323,"reasoning_tokens":13288841,"success":true,"baseline":false},{"id":"30d03ff8-777a-475f-b04d-fb73ae05785b","workflow_id":"rb-7741ebdf-ext4-fugu-max-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-18T11:55:14.193Z","resumed_from_workflow_id":null,"config_key":"631d44e09067c0e58125bd89a32ee325","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.298969,"total_questions":97,"correct_answers":29,"total_cost":57.555494,"generation_time_ms":19408156,"duration_ms":3587601,"input_tokens":104492046,"output_tokens":1491398,"reasoning_tokens":1224420,"success":true,"baseline":false},{"id":"fc4aa9e6-eaaf-4cf2-9cc7-62e21940f167","workflow_id":"rb-7741ebdf-ext4-fugu-ultra-v2-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"sakana/fugu-ultra-v2","provider_name":"auto-routing","timestamp":"2026-09-18T11:55:14.190Z","resumed_from_workflow_id":null,"config_key":"87ecead7895f766c6093ecc3a56c2f10","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.350515,"total_questions":97,"correct_answers":34,"total_cost":275.673304,"generation_time_ms":23940758,"duration_ms":3608817,"input_tokens":131713843,"output_tokens":1327595,"reasoning_tokens":802896,"success":true,"baseline":false},{"id":"672b44c4-0d7c-41b5-92e7-17c2066cbdab","workflow_id":"rb-7741ebdf-ext4-fugu-ultra-v2-mmmu-pro-vision","benchmark_type":"mmmu_pro_vision","model_permaslug":"sakana/fugu-ultra-v2","provider_name":"auto-routing","timestamp":"2026-09-18T11:55:14.184Z","resumed_from_workflow_id":null,"config_key":"a10543c341a126603b5856b3a301ba50","reasoning_effort":"high","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":3,"recorded_epochs":3,"accuracy":0.80501,"total_questions":5190,"correct_answers":4178,"total_cost":567.894964,"generation_time_ms":218281287,"duration_ms":9977928,"input_tokens":85928291,"output_tokens":5902077,"reasoning_tokens":587726,"success":true,"baseline":false},{"id":"a511c523-45a5-4e63-9e2d-6ddb22fc3fd6","workflow_id":"rb-7741ebdf-ext4-fugu-max-swe-atlas-qa","benchmark_type":"swe_atlas_qa","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-18T11:55:13.783Z","resumed_from_workflow_id":null,"config_key":"392fe58ae6feb02dc5fca40e1e66a234","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.524194,"total_questions":124,"correct_answers":65,"total_cost":193.8577,"generation_time_ms":82553717,"duration_ms":19650054,"input_tokens":530337376,"output_tokens":6213915,"reasoning_tokens":4028249,"success":true,"baseline":false},{"id":"394d6b67-5432-40bb-9b70-68be471a5ee5","workflow_id":"rb-7741ebdf-ext2-union-alpha-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"unbiased/pareto-20260917","provider_name":"auto-routing","timestamp":"2026-09-17T13:12:55.247Z","resumed_from_workflow_id":null,"config_key":"46f623096bd58feaf9465ca01b31bc73","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.371134,"total_questions":97,"correct_answers":36,"total_cost":0.010133,"generation_time_ms":16051027,"duration_ms":2849119,"input_tokens":82274590,"output_tokens":320734,"reasoning_tokens":85,"success":true,"baseline":false},{"id":"0fa6f6e3-5dc1-48cc-af42-0d6461f94cda","workflow_id":"rb-7741ebdf-ext-fugu-max-deep-swe","benchmark_type":"deep_swe","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-16T03:06:09.565Z","resumed_from_workflow_id":null,"config_key":"966cf9b20f1df4816400a69d5baf59dd","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":1,"recorded_epochs":1,"accuracy":0.383929,"total_questions":112,"correct_answers":43,"total_cost":322.464822,"generation_time_ms":166433486,"duration_ms":44515180,"input_tokens":855890840,"output_tokens":13167225,"reasoning_tokens":10921808,"success":true,"baseline":false},{"id":"82ffd18b-05b5-41f4-b0b4-29edb12978cb","workflow_id":"rb-7741ebdf-ext-fugu-max-swe-atlas-qa","benchmark_type":"swe_atlas_qa","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-16T02:46:05.137Z","resumed_from_workflow_id":null,"config_key":"392fe58ae6feb02dc5fca40e1e66a234","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.528226,"total_questions":248,"correct_answers":131,"total_cost":406.188523,"generation_time_ms":130083478,"duration_ms":22825043,"input_tokens":1045289588,"output_tokens":13086401,"reasoning_tokens":8675369,"success":true,"baseline":false},{"id":"cf5eaf24-3fa3-4ca3-9c3c-e1ee7387e810","workflow_id":"rb-7741ebdf-ext-fugu-max-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"sakana/fugu-max","provider_name":"auto-routing","timestamp":"2026-09-16T02:46:05.133Z","resumed_from_workflow_id":null,"config_key":"631d44e09067c0e58125bd89a32ee325","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.304124,"total_questions":194,"correct_answers":59,"total_cost":84.783657,"generation_time_ms":23498566,"duration_ms":3122623,"input_tokens":139764660,"output_tokens":2370109,"reasoning_tokens":1945169,"success":true,"baseline":false},{"id":"a5a585d4-4756-4390-ada5-22a5be25e79e","workflow_id":"rb-7741ebdf-ext-fugu-ultra-v2-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"sakana/fugu-ultra-v2","provider_name":"auto-routing","timestamp":"2026-09-16T02:46:05.133Z","resumed_from_workflow_id":null,"config_key":"87ecead7895f766c6093ecc3a56c2f10","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.345361,"total_questions":194,"correct_answers":67,"total_cost":628.777126,"generation_time_ms":51402317,"duration_ms":4308575,"input_tokens":303783938,"output_tokens":3071479,"reasoning_tokens":1901712,"success":true,"baseline":false},{"id":"41aeb205-db2f-4f9c-a07d-9932ac2627ad","workflow_id":"rb-7741ebdf-ext-union-alpha-tau3-banking","benchmark_type":"tau3_bench_banking","model_permaslug":"unbiased/pareto-20260917","provider_name":"auto-routing","timestamp":"2026-09-16T02:46:04.601Z","resumed_from_workflow_id":null,"config_key":"46f623096bd58feaf9465ca01b31bc73","reasoning_effort":"medium","max_tokens":null,"cost_tier":null,"route_models":null,"temperature":0,"sample_limit":null,"epochs":2,"recorded_epochs":2,"accuracy":0.350515,"total_questions":194,"correct_answers":68,"total_cost":0.005834,"generation_time_ms":17929327,"duration_ms":1692279,"input_tokens":179923246,"output_tokens":845421,"reasoning_tokens":393,"success":true,"baseline":false}],"truncated":false}}