{"title":"Qwen3.8-27B and Qwen3.6-35B-A3B on api.tokenwatt.io, thinking on vs off: 175 single-file Python tasks","post":"https://www.tokenwatt.io/blog/qwen38-qwen36-thinking-on-off-coding-benchmark.html","report":"https://www.tokenwatt.io/reports/api-qwen38-qwen36-coding-175-thinking-on-off.html","suite":"176 single-file Python tasks: Exercism Python plus Aider's polyglot hard set (Python). The model gets the instructions and the stub file and returns the whole file; the task's tests run in a rented Lambda gpu_1x_a10 sandbox; on failure, one retry with the failing test output.","basis":"Summaries are on 175 tasks, like https://www.coderouter.io/benchmark: exercism/alphametics is left out (in_175_basis: false); its records are still listed.","common_settings":{"endpoint":"https://api.tokenwatt.io/v1/chat/completions (streamed)","max_tokens":32768,"concurrency":8,"retries":1,"date":"2026-11-20"},"runs_settings":{"T-qwen36-nothink":{"label":"Qwen3.6-35B-A3B, thinking off","model":"qwen/qwen3.6-35b-a3b","thinking":"off","request_body_extra":{"reasoning_effort":"none","temperature":0},"when":"2026-11-20 ~18:13–18:16 UTC"},"T-qwen36-think":{"label":"Qwen3.6-35B-A3B, thinking on","model":"qwen/qwen3.6-35b-a3b","thinking":"on (model default)","request_body_extra":{"temperature":0.6,"top_p":0.95,"top_k":20},"when":"2026-11-20 ~18:16–18:38 UTC"},"T-qwen38-nothink":{"label":"Qwen3.8-27B, thinking off","model":"qwen/qwen3.8-27b","thinking":"off","request_body_extra":{"reasoning_effort":"none","temperature":0},"when":"2026-11-20 ~18:09–18:13 UTC"},"T-qwen38-think":{"label":"Qwen3.8-27B, thinking on","model":"qwen/qwen3.8-27b","thinking":"on (model default)","request_body_extra":{"temperature":0.6,"top_p":0.95,"top_k":20},"when":"2026-11-20 ~19:11–20:01 UTC (after MTP)"}},"prices_usd_per_million":{"qwen/qwen3.8-27b":{"input":0.06,"output":2.0,"cached_input":0.03},"qwen/qwen3.6-35b-a3b":{"input":0.1,"output":0.9,"cached_input":0.08}},"cost_note":"Cost = tokens each response reported x our list prices (what a customer pays at list), retries included. Thinking tokens are billed as output. Not our GPU cost.","fields":{"hit_token_cap":"output_tokens reached max_tokens (32768)","no_code_in_reply":"the reply had no Python code block, so no test ran","latency_s":"wall time of the request(s), retry included for a task"},"summary_175":{"T-qwen36-nothink":{"tasks":175,"passed":137,"passed_first":119,"pass_rate_pct":78.3,"first_try_pct":68.0,"cost_usd":0.1565,"cost_per_solved_task_usd":0.00114,"median_latency_s":3.2,"attempts":231,"retried_tasks":56,"solved_by_retry":18,"attempts_at_cap":0,"attempts_at_cap_pct":0.0,"attempts_at_cap_no_code":0,"attempts_no_code_below_cap":0,"first_attempts_at_cap":0,"tasks_with_cap":0,"tasks_cap_both_attempts":0,"tasks_with_cap_passed":0,"cost_of_capped_attempts_usd":0,"capped_attempt_latency_s":null,"prompt_tokens":240103,"cached_prompt_tokens":20672,"output_tokens":147623,"reasoning_tokens":0,"median_output_tokens_per_attempt":473,"errors":0},"T-qwen36-think":{"tasks":175,"passed":144,"passed_first":120,"pass_rate_pct":82.3,"first_try_pct":68.6,"cost_usd":1.2732,"cost_per_solved_task_usd":0.00884,"median_latency_s":33.04,"attempts":230,"retried_tasks":55,"solved_by_retry":24,"attempts_at_cap":2,"attempts_at_cap_pct":0.9,"attempts_at_cap_no_code":2,"attempts_no_code_below_cap":0,"first_attempts_at_cap":0,"tasks_with_cap":2,"tasks_cap_both_attempts":0,"tasks_with_cap_passed":0,"cost_of_capped_attempts_usd":0.0594,"capped_attempt_latency_s":{"median":174.805,"min":150.34,"max":199.27},"prompt_tokens":217615,"cached_prompt_tokens":53312,"output_tokens":1391628,"reasoning_tokens":1318907,"median_output_tokens_per_attempt":5367.5,"errors":0},"T-qwen38-nothink":{"tasks":175,"passed":128,"passed_first":106,"pass_rate_pct":73.1,"first_try_pct":60.6,"cost_usd":0.2706,"cost_per_solved_task_usd":0.00211,"median_latency_s":5.54,"attempts":244,"retried_tasks":69,"solved_by_retry":22,"attempts_at_cap":0,"attempts_at_cap_pct":0.0,"attempts_at_cap_no_code":0,"attempts_no_code_below_cap":0,"first_attempts_at_cap":0,"tasks_with_cap":0,"tasks_cap_both_attempts":0,"tasks_with_cap_passed":0,"cost_of_capped_attempts_usd":0,"capped_attempt_latency_s":null,"prompt_tokens":259378,"cached_prompt_tokens":49920,"output_tokens":128256,"reasoning_tokens":0,"median_output_tokens_per_attempt":389.0,"errors":0},"T-qwen38-think":{"tasks":175,"passed":138,"passed_first":103,"pass_rate_pct":78.9,"first_try_pct":58.9,"cost_usd":7.0151,"cost_per_solved_task_usd":0.05083,"median_latency_s":76.92,"attempts":247,"retried_tasks":72,"solved_by_retry":35,"attempts_at_cap":42,"attempts_at_cap_pct":17.0,"attempts_at_cap_no_code":41,"attempts_no_code_below_cap":3,"first_attempts_at_cap":27,"tasks_with_cap":30,"tasks_cap_both_attempts":12,"tasks_with_cap_passed":7,"cost_of_capped_attempts_usd":2.7547,"capped_attempt_latency_s":{"median":206.39499999999998,"min":145.1,"max":309.89},"prompt_tokens":231817,"cached_prompt_tokens":36608,"output_tokens":3501139,"reasoning_tokens":3429693,"median_output_tokens_per_attempt":11863,"errors":0}},"per_task":[{"task":"exercism/accumulate","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000126,"latency_s":1.05,"per_attempt":[{"passed":true,"prompt_tokens":245,"cached_prompt_tokens":0,"output_tokens":113,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.05,"cost_usd":0.000126,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000742,"latency_s":4.79,"per_attempt":[{"passed":true,"prompt_tokens":243,"cached_prompt_tokens":0,"output_tokens":798,"reasoning_tokens":761,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.79,"cost_usd":0.000742,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":8.5e-05,"latency_s":0.94,"per_attempt":[{"passed":true,"prompt_tokens":245,"cached_prompt_tokens":0,"output_tokens":35,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.94,"cost_usd":8.5e-05,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000895,"latency_s":3.29,"per_attempt":[{"passed":true,"prompt_tokens":281,"cached_prompt_tokens":0,"output_tokens":439,"reasoning_tokens":402,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.29,"cost_usd":0.000895,"error":null}]}}},{"task":"exercism/acronym","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000209,"latency_s":1.56,"per_attempt":[{"passed":true,"prompt_tokens":207,"cached_prompt_tokens":0,"output_tokens":209,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.56,"cost_usd":0.000209,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003161,"latency_s":22.91,"per_attempt":[{"passed":true,"prompt_tokens":205,"cached_prompt_tokens":0,"output_tokens":3490,"reasoning_tokens":3398,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":22.91,"cost_usd":0.003161,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000236,"latency_s":2.15,"per_attempt":[{"passed":true,"prompt_tokens":207,"cached_prompt_tokens":0,"output_tokens":112,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.15,"cost_usd":0.000236,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.025087,"latency_s":76.92,"per_attempt":[{"passed":true,"prompt_tokens":243,"cached_prompt_tokens":0,"output_tokens":12536,"reasoning_tokens":12472,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":76.92,"cost_usd":0.025087,"error":null}]}}},{"task":"exercism/affine-cipher","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000835,"latency_s":5.63,"per_attempt":[{"passed":true,"prompt_tokens":1203,"cached_prompt_tokens":0,"output_tokens":794,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.63,"cost_usd":0.000835,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.007188,"latency_s":49.89,"per_attempt":[{"passed":true,"prompt_tokens":1201,"cached_prompt_tokens":1088,"output_tokens":7877,"reasoning_tokens":7547,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":49.89,"cost_usd":0.007188,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000916,"latency_s":6.47,"per_attempt":[{"passed":true,"prompt_tokens":1203,"cached_prompt_tokens":0,"output_tokens":422,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.47,"cost_usd":0.000916,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.017054,"latency_s":49.37,"per_attempt":[{"passed":true,"prompt_tokens":1239,"cached_prompt_tokens":0,"output_tokens":8490,"reasoning_tokens":7974,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":49.37,"cost_usd":0.017054,"error":null}]}}},{"task":"exercism/all-your-base","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000294,"latency_s":1.98,"per_attempt":[{"passed":true,"prompt_tokens":772,"cached_prompt_tokens":0,"output_tokens":241,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.98,"cost_usd":0.000294,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004009,"latency_s":28.25,"per_attempt":[{"passed":true,"prompt_tokens":770,"cached_prompt_tokens":0,"output_tokens":4369,"reasoning_tokens":4203,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":28.25,"cost_usd":0.004009,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.001143,"latency_s":7.68,"per_attempt":[{"passed":false,"prompt_tokens":772,"cached_prompt_tokens":0,"output_tokens":260,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.09,"cost_usd":0.000566,"error":null},{"passed":true,"prompt_tokens":1381,"cached_prompt_tokens":0,"output_tokens":247,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.59,"cost_usd":0.000577,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003918,"latency_s":11.74,"per_attempt":[{"passed":true,"prompt_tokens":808,"cached_prompt_tokens":0,"output_tokens":1935,"reasoning_tokens":1765,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.74,"cost_usd":0.003918,"error":null}]}}},{"task":"exercism/allergies","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000187,"latency_s":1.32,"per_attempt":[{"passed":true,"prompt_tokens":383,"cached_prompt_tokens":0,"output_tokens":165,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.32,"cost_usd":0.000187,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002817,"latency_s":19.16,"per_attempt":[{"passed":true,"prompt_tokens":381,"cached_prompt_tokens":0,"output_tokens":3088,"reasoning_tokens":2939,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":19.16,"cost_usd":0.002817,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000577,"latency_s":4.27,"per_attempt":[{"passed":true,"prompt_tokens":383,"cached_prompt_tokens":0,"output_tokens":277,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.27,"cost_usd":0.000577,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001397,"latency_s":4.22,"per_attempt":[{"passed":true,"prompt_tokens":419,"cached_prompt_tokens":0,"output_tokens":686,"reasoning_tokens":490,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.22,"cost_usd":0.001397,"error":null}]}}},{"task":"exercism/alphametics","in_175_basis":false,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001601,"latency_s":10.9,"per_attempt":[{"passed":false,"prompt_tokens":265,"cached_prompt_tokens":0,"output_tokens":796,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.64,"cost_usd":0.000743,"error":null},{"passed":false,"prompt_tokens":1349,"cached_prompt_tokens":0,"output_tokens":804,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.26,"cost_usd":0.000858,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00631,"latency_s":44.68,"per_attempt":[{"passed":true,"prompt_tokens":263,"cached_prompt_tokens":0,"output_tokens":6982,"reasoning_tokens":6683,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":44.68,"cost_usd":0.00631,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001038,"latency_s":7.57,"per_attempt":[{"passed":true,"prompt_tokens":265,"cached_prompt_tokens":0,"output_tokens":511,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.57,"cost_usd":0.001038,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.121329,"latency_s":408.88,"per_attempt":[{"passed":false,"prompt_tokens":301,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32763,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":220.83,"cost_usd":0.065554,"error":null},{"passed":true,"prompt_tokens":346,"cached_prompt_tokens":0,"output_tokens":27877,"reasoning_tokens":27092,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":188.05,"cost_usd":0.055775,"error":null}]}}},{"task":"exercism/anagram","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000259,"latency_s":1.7,"per_attempt":[{"passed":true,"prompt_tokens":514,"cached_prompt_tokens":0,"output_tokens":231,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.7,"cost_usd":0.000259,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001792,"latency_s":11.4,"per_attempt":[{"passed":true,"prompt_tokens":512,"cached_prompt_tokens":0,"output_tokens":1934,"reasoning_tokens":1870,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.4,"cost_usd":0.001792,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000371,"latency_s":2.96,"per_attempt":[{"passed":true,"prompt_tokens":514,"cached_prompt_tokens":0,"output_tokens":170,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.96,"cost_usd":0.000371,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000773,"latency_s":2.74,"per_attempt":[{"passed":true,"prompt_tokens":550,"cached_prompt_tokens":0,"output_tokens":370,"reasoning_tokens":307,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.74,"cost_usd":0.000773,"error":null}]}}},{"task":"exercism/armstrong-numbers","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000201,"latency_s":1.47,"per_attempt":[{"passed":true,"prompt_tokens":305,"cached_prompt_tokens":0,"output_tokens":189,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.47,"cost_usd":0.000201,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000773,"latency_s":5.04,"per_attempt":[{"passed":true,"prompt_tokens":303,"cached_prompt_tokens":0,"output_tokens":825,"reasoning_tokens":777,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.04,"cost_usd":0.000773,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000114,"latency_s":1.07,"per_attempt":[{"passed":true,"prompt_tokens":305,"cached_prompt_tokens":0,"output_tokens":48,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.07,"cost_usd":0.000114,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000498,"latency_s":1.93,"per_attempt":[{"passed":true,"prompt_tokens":341,"cached_prompt_tokens":0,"output_tokens":239,"reasoning_tokens":181,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.93,"cost_usd":0.000498,"error":null}]}}},{"task":"exercism/atbash-cipher","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000523,"latency_s":3.76,"per_attempt":[{"passed":true,"prompt_tokens":390,"cached_prompt_tokens":0,"output_tokens":538,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.76,"cost_usd":0.000523,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004109,"latency_s":29.31,"per_attempt":[{"passed":true,"prompt_tokens":388,"cached_prompt_tokens":0,"output_tokens":4522,"reasoning_tokens":4382,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":29.31,"cost_usd":0.004109,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000633,"latency_s":4.47,"per_attempt":[{"passed":true,"prompt_tokens":390,"cached_prompt_tokens":0,"output_tokens":305,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.47,"cost_usd":0.000633,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005274,"latency_s":16.16,"per_attempt":[{"passed":true,"prompt_tokens":426,"cached_prompt_tokens":0,"output_tokens":2624,"reasoning_tokens":2428,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":16.16,"cost_usd":0.005274,"error":null}]}}},{"task":"exercism/bank-account","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00036,"latency_s":2.09,"per_attempt":[{"passed":true,"prompt_tokens":1145,"cached_prompt_tokens":0,"output_tokens":273,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.09,"cost_usd":0.00036,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002902,"latency_s":20.76,"per_attempt":[{"passed":true,"prompt_tokens":1143,"cached_prompt_tokens":1088,"output_tokens":3122,"reasoning_tokens":2855,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":20.76,"cost_usd":0.002902,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000615,"latency_s":4.05,"per_attempt":[{"passed":true,"prompt_tokens":1145,"cached_prompt_tokens":0,"output_tokens":273,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.05,"cost_usd":0.000615,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005173,"latency_s":14.56,"per_attempt":[{"passed":true,"prompt_tokens":1181,"cached_prompt_tokens":0,"output_tokens":2551,"reasoning_tokens":2276,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.56,"cost_usd":0.005173,"error":null}]}}},{"task":"exercism/beer-song","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000728,"latency_s":3.33,"per_attempt":[{"passed":true,"prompt_tokens":3696,"cached_prompt_tokens":3264,"output_tokens":471,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.33,"cost_usd":0.000728,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006031,"latency_s":41.07,"per_attempt":[{"passed":true,"prompt_tokens":3694,"cached_prompt_tokens":3264,"output_tokens":6363,"reasoning_tokens":6103,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":41.07,"cost_usd":0.006031,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001681,"latency_s":9.52,"per_attempt":[{"passed":false,"prompt_tokens":3696,"cached_prompt_tokens":3328,"output_tokens":369,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.21,"cost_usd":0.00086,"error":null},{"passed":false,"prompt_tokens":4891,"cached_prompt_tokens":4160,"output_tokens":326,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.31,"cost_usd":0.000821,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.011879,"latency_s":32.39,"per_attempt":[{"passed":false,"prompt_tokens":3732,"cached_prompt_tokens":3328,"output_tokens":4771,"reasoning_tokens":4488,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":26.51,"cost_usd":0.009666,"error":null},{"passed":true,"prompt_tokens":4839,"cached_prompt_tokens":3328,"output_tokens":1011,"reasoning_tokens":717,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.88,"cost_usd":0.002213,"error":null}]}}},{"task":"exercism/binary","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00025,"latency_s":1.52,"per_attempt":[{"passed":true,"prompt_tokens":646,"cached_prompt_tokens":0,"output_tokens":206,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.52,"cost_usd":0.00025,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001107,"latency_s":8.37,"per_attempt":[{"passed":true,"prompt_tokens":644,"cached_prompt_tokens":0,"output_tokens":1158,"reasoning_tokens":1090,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.37,"cost_usd":0.001107,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000419,"latency_s":2.9,"per_attempt":[{"passed":true,"prompt_tokens":646,"cached_prompt_tokens":0,"output_tokens":190,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.9,"cost_usd":0.000419,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001577,"latency_s":4.74,"per_attempt":[{"passed":true,"prompt_tokens":682,"cached_prompt_tokens":0,"output_tokens":768,"reasoning_tokens":645,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.74,"cost_usd":0.001577,"error":null}]}}},{"task":"exercism/binary-search","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000271,"latency_s":1.4,"per_attempt":[{"passed":true,"prompt_tokens":929,"cached_prompt_tokens":0,"output_tokens":198,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.4,"cost_usd":0.000271,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00149,"latency_s":9.43,"per_attempt":[{"passed":true,"prompt_tokens":927,"cached_prompt_tokens":0,"output_tokens":1553,"reasoning_tokens":1449,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.43,"cost_usd":0.00149,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000262,"latency_s":1.53,"per_attempt":[{"passed":true,"prompt_tokens":929,"cached_prompt_tokens":0,"output_tokens":103,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.53,"cost_usd":0.000262,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000968,"latency_s":2.64,"per_attempt":[{"passed":true,"prompt_tokens":965,"cached_prompt_tokens":0,"output_tokens":455,"reasoning_tokens":345,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.64,"cost_usd":0.000968,"error":null}]}}},{"task":"exercism/binary-search-tree","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000388,"latency_s":2.18,"per_attempt":[{"passed":true,"prompt_tokens":938,"cached_prompt_tokens":0,"output_tokens":327,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.18,"cost_usd":0.000388,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003422,"latency_s":22.85,"per_attempt":[{"passed":true,"prompt_tokens":936,"cached_prompt_tokens":0,"output_tokens":3698,"reasoning_tokens":3398,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":22.85,"cost_usd":0.003422,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.001465,"latency_s":9.13,"per_attempt":[{"passed":false,"prompt_tokens":938,"cached_prompt_tokens":0,"output_tokens":336,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.64,"cost_usd":0.000728,"error":null},{"passed":true,"prompt_tokens":1734,"cached_prompt_tokens":832,"output_tokens":329,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.49,"cost_usd":0.000737,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.055296,"latency_s":171.83,"per_attempt":[{"passed":false,"prompt_tokens":974,"cached_prompt_tokens":0,"output_tokens":6206,"reasoning_tokens":5838,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":36.04,"cost_usd":0.01247,"error":null},{"passed":true,"prompt_tokens":1846,"cached_prompt_tokens":832,"output_tokens":21370,"reasoning_tokens":20844,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":135.79,"cost_usd":0.042826,"error":null}]}}},{"task":"exercism/bob","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000327,"latency_s":2.33,"per_attempt":[{"passed":true,"prompt_tokens":355,"cached_prompt_tokens":0,"output_tokens":324,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.33,"cost_usd":0.000327,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002168,"latency_s":15.0,"per_attempt":[{"passed":true,"prompt_tokens":353,"cached_prompt_tokens":0,"output_tokens":2370,"reasoning_tokens":2245,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":15.0,"cost_usd":0.002168,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000463,"latency_s":3.17,"per_attempt":[{"passed":true,"prompt_tokens":355,"cached_prompt_tokens":0,"output_tokens":221,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.17,"cost_usd":0.000463,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003989,"latency_s":12.9,"per_attempt":[{"passed":true,"prompt_tokens":391,"cached_prompt_tokens":0,"output_tokens":1983,"reasoning_tokens":1811,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.9,"cost_usd":0.003989,"error":null}]}}},{"task":"exercism/book-store","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.001713,"latency_s":10.67,"per_attempt":[{"passed":false,"prompt_tokens":708,"cached_prompt_tokens":0,"output_tokens":802,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.53,"cost_usd":0.000793,"error":null},{"passed":true,"prompt_tokens":1838,"cached_prompt_tokens":0,"output_tokens":818,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.14,"cost_usd":0.00092,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.0112,"latency_s":73.87,"per_attempt":[{"passed":false,"prompt_tokens":706,"cached_prompt_tokens":0,"output_tokens":8105,"reasoning_tokens":7726,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":49.3,"cost_usd":0.007365,"error":null},{"passed":true,"prompt_tokens":1411,"cached_prompt_tokens":0,"output_tokens":4104,"reasoning_tokens":3784,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":24.57,"cost_usd":0.003835,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001978,"latency_s":12.72,"per_attempt":[{"passed":true,"prompt_tokens":708,"cached_prompt_tokens":0,"output_tokens":968,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.72,"cost_usd":0.001978,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.131164,"latency_s":408.71,"per_attempt":[{"passed":false,"prompt_tokens":744,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":205.77,"cost_usd":0.065581,"error":null},{"passed":false,"prompt_tokens":789,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":202.94,"cost_usd":0.065583,"error":null}]}}},{"task":"exercism/bottle-song","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.00099,"latency_s":6.07,"per_attempt":[{"passed":false,"prompt_tokens":516,"cached_prompt_tokens":0,"output_tokens":431,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.11,"cost_usd":0.00044,"error":null},{"passed":false,"prompt_tokens":1629,"cached_prompt_tokens":0,"output_tokens":431,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.96,"cost_usd":0.000551,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004759,"latency_s":31.86,"per_attempt":[{"passed":true,"prompt_tokens":514,"cached_prompt_tokens":0,"output_tokens":5231,"reasoning_tokens":4977,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":31.86,"cost_usd":0.004759,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001861,"latency_s":11.5,"per_attempt":[{"passed":false,"prompt_tokens":516,"cached_prompt_tokens":0,"output_tokens":433,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.7,"cost_usd":0.000897,"error":null},{"passed":false,"prompt_tokens":1635,"cached_prompt_tokens":0,"output_tokens":433,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.8,"cost_usd":0.000964,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.016513,"latency_s":47.97,"per_attempt":[{"passed":true,"prompt_tokens":552,"cached_prompt_tokens":0,"output_tokens":8240,"reasoning_tokens":7947,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":47.97,"cost_usd":0.016513,"error":null}]}}},{"task":"exercism/bowling","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.00403,"latency_s":25.67,"per_attempt":[{"passed":false,"prompt_tokens":961,"cached_prompt_tokens":0,"output_tokens":1979,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.85,"cost_usd":0.001877,"error":null},{"passed":false,"prompt_tokens":3707,"cached_prompt_tokens":0,"output_tokens":1980,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.82,"cost_usd":0.002153,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.02821,"latency_s":190.15,"per_attempt":[{"passed":false,"prompt_tokens":959,"cached_prompt_tokens":0,"output_tokens":16398,"reasoning_tokens":15849,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":101.04,"cost_usd":0.014854,"error":null},{"passed":false,"prompt_tokens":1902,"cached_prompt_tokens":0,"output_tokens":14629,"reasoning_tokens":13795,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":89.11,"cost_usd":0.013356,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.004349,"latency_s":27.08,"per_attempt":[{"passed":false,"prompt_tokens":961,"cached_prompt_tokens":0,"output_tokens":1036,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.48,"cost_usd":0.00213,"error":null},{"passed":false,"prompt_tokens":2572,"cached_prompt_tokens":832,"output_tokens":1045,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.6,"cost_usd":0.002219,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.128882,"latency_s":379.79,"per_attempt":[{"passed":false,"prompt_tokens":997,"cached_prompt_tokens":0,"output_tokens":31599,"reasoning_tokens":30914,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":187.7,"cost_usd":0.063258,"error":null},{"passed":false,"prompt_tokens":1887,"cached_prompt_tokens":832,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":192.09,"cost_usd":0.065624,"error":null}]}}},{"task":"exercism/camicia","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.004444,"latency_s":27.16,"per_attempt":[{"passed":false,"prompt_tokens":1992,"cached_prompt_tokens":0,"output_tokens":2591,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":16.58,"cost_usd":0.002531,"error":null},{"passed":false,"prompt_tokens":5111,"cached_prompt_tokens":1088,"output_tokens":1582,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.58,"cost_usd":0.001913,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.0445,"latency_s":298.82,"per_attempt":[{"passed":false,"prompt_tokens":1990,"cached_prompt_tokens":1088,"output_tokens":16142,"reasoning_tokens":15403,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":99.55,"cost_usd":0.014705,"error":null},{"passed":false,"prompt_tokens":3255,"cached_prompt_tokens":1088,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":199.27,"cost_usd":0.029795,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.008936,"latency_s":66.21,"per_attempt":[{"passed":false,"prompt_tokens":1992,"cached_prompt_tokens":1664,"output_tokens":2607,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":43.76,"cost_usd":0.005284,"error":null},{"passed":false,"prompt_tokens":5131,"cached_prompt_tokens":1664,"output_tokens":1697,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":22.45,"cost_usd":0.003652,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.131268,"latency_s":442.94,"per_attempt":[{"passed":false,"prompt_tokens":2028,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":217.24,"cost_usd":0.065658,"error":null},{"passed":false,"prompt_tokens":2073,"cached_prompt_tokens":1664,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":225.7,"cost_usd":0.06561,"error":null}]}}},{"task":"exercism/change","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000508,"latency_s":3.33,"per_attempt":[{"passed":true,"prompt_tokens":823,"cached_prompt_tokens":0,"output_tokens":473,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.33,"cost_usd":0.000508,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006682,"latency_s":43.95,"per_attempt":[{"passed":true,"prompt_tokens":821,"cached_prompt_tokens":0,"output_tokens":7333,"reasoning_tokens":7110,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":43.95,"cost_usd":0.006682,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000811,"latency_s":4.96,"per_attempt":[{"passed":true,"prompt_tokens":823,"cached_prompt_tokens":0,"output_tokens":381,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.96,"cost_usd":0.000811,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.035051,"latency_s":106.66,"per_attempt":[{"passed":false,"prompt_tokens":859,"cached_prompt_tokens":0,"output_tokens":15494,"reasoning_tokens":15129,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":93.38,"cost_usd":0.03104,"error":null},{"passed":true,"prompt_tokens":1476,"cached_prompt_tokens":832,"output_tokens":1974,"reasoning_tokens":1609,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.28,"cost_usd":0.004012,"error":null}]}}},{"task":"exercism/circular-buffer","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000496,"latency_s":2.87,"per_attempt":[{"passed":true,"prompt_tokens":1140,"cached_prompt_tokens":0,"output_tokens":424,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.87,"cost_usd":0.000496,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004122,"latency_s":26.91,"per_attempt":[{"passed":true,"prompt_tokens":1138,"cached_prompt_tokens":1088,"output_tokens":4478,"reasoning_tokens":4060,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":26.91,"cost_usd":0.004122,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00089,"latency_s":5.44,"per_attempt":[{"passed":true,"prompt_tokens":1140,"cached_prompt_tokens":0,"output_tokens":411,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.44,"cost_usd":0.00089,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.047039,"latency_s":149.72,"per_attempt":[{"passed":false,"prompt_tokens":1176,"cached_prompt_tokens":0,"output_tokens":19879,"reasoning_tokens":19376,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":125.8,"cost_usd":0.039829,"error":null},{"passed":true,"prompt_tokens":1954,"cached_prompt_tokens":832,"output_tokens":3559,"reasoning_tokens":3070,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":23.92,"cost_usd":0.00721,"error":null}]}}},{"task":"exercism/clock","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000401,"latency_s":1.94,"per_attempt":[{"passed":true,"prompt_tokens":1324,"cached_prompt_tokens":0,"output_tokens":298,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.94,"cost_usd":0.000401,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003648,"latency_s":23.22,"per_attempt":[{"passed":true,"prompt_tokens":1322,"cached_prompt_tokens":1088,"output_tokens":3931,"reasoning_tokens":3735,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":23.22,"cost_usd":0.003648,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000501,"latency_s":2.91,"per_attempt":[{"passed":true,"prompt_tokens":1324,"cached_prompt_tokens":0,"output_tokens":211,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.91,"cost_usd":0.000501,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.014356,"latency_s":44.17,"per_attempt":[{"passed":true,"prompt_tokens":1360,"cached_prompt_tokens":0,"output_tokens":7137,"reasoning_tokens":6916,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":44.17,"cost_usd":0.014356,"error":null}]}}},{"task":"exercism/collatz-conjecture","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000235,"latency_s":1.38,"per_attempt":[{"passed":true,"prompt_tokens":742,"cached_prompt_tokens":0,"output_tokens":179,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.38,"cost_usd":0.000235,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001201,"latency_s":7.45,"per_attempt":[{"passed":true,"prompt_tokens":740,"cached_prompt_tokens":0,"output_tokens":1252,"reasoning_tokens":1164,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.45,"cost_usd":0.001201,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000371,"latency_s":2.2,"per_attempt":[{"passed":true,"prompt_tokens":742,"cached_prompt_tokens":0,"output_tokens":163,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.2,"cost_usd":0.000371,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.008861,"latency_s":30.43,"per_attempt":[{"passed":true,"prompt_tokens":778,"cached_prompt_tokens":0,"output_tokens":4407,"reasoning_tokens":4320,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":30.43,"cost_usd":0.008861,"error":null}]}}},{"task":"exercism/complex-numbers","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000851,"latency_s":5.12,"per_attempt":[{"passed":true,"prompt_tokens":1014,"cached_prompt_tokens":0,"output_tokens":833,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.12,"cost_usd":0.000851,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005811,"latency_s":38.21,"per_attempt":[{"passed":true,"prompt_tokens":1012,"cached_prompt_tokens":0,"output_tokens":6344,"reasoning_tokens":5766,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":38.21,"cost_usd":0.005811,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001683,"latency_s":10.61,"per_attempt":[{"passed":true,"prompt_tokens":1014,"cached_prompt_tokens":0,"output_tokens":811,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.61,"cost_usd":0.001683,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.042315,"latency_s":126.59,"per_attempt":[{"passed":true,"prompt_tokens":1050,"cached_prompt_tokens":0,"output_tokens":21126,"reasoning_tokens":20232,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":126.59,"cost_usd":0.042315,"error":null}]}}},{"task":"exercism/connect","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002025,"latency_s":12.78,"per_attempt":[{"passed":false,"prompt_tokens":393,"cached_prompt_tokens":0,"output_tokens":1088,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.86,"cost_usd":0.001019,"error":null},{"passed":false,"prompt_tokens":1729,"cached_prompt_tokens":0,"output_tokens":926,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.92,"cost_usd":0.001006,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.026191,"latency_s":177.69,"per_attempt":[{"passed":false,"prompt_tokens":391,"cached_prompt_tokens":0,"output_tokens":11557,"reasoning_tokens":11082,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":72.14,"cost_usd":0.01044,"error":null},{"passed":false,"prompt_tokens":1102,"cached_prompt_tokens":0,"output_tokens":17378,"reasoning_tokens":16904,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":105.55,"cost_usd":0.01575,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.006312,"latency_s":39.98,"per_attempt":[{"passed":false,"prompt_tokens":393,"cached_prompt_tokens":0,"output_tokens":1518,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":19.65,"cost_usd":0.00306,"error":null},{"passed":false,"prompt_tokens":2333,"cached_prompt_tokens":0,"output_tokens":1556,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":20.33,"cost_usd":0.003252,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.06295,"latency_s":194.9,"per_attempt":[{"passed":true,"prompt_tokens":429,"cached_prompt_tokens":0,"output_tokens":31462,"reasoning_tokens":30739,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":194.9,"cost_usd":0.06295,"error":null}]}}},{"task":"exercism/crypto-square","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000572,"latency_s":3.44,"per_attempt":[{"passed":true,"prompt_tokens":624,"cached_prompt_tokens":0,"output_tokens":566,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.44,"cost_usd":0.000572,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005057,"latency_s":33.76,"per_attempt":[{"passed":true,"prompt_tokens":622,"cached_prompt_tokens":0,"output_tokens":5550,"reasoning_tokens":5372,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":33.76,"cost_usd":0.005057,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.007473,"latency_s":46.93,"per_attempt":[{"passed":false,"prompt_tokens":624,"cached_prompt_tokens":0,"output_tokens":3189,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":41.27,"cost_usd":0.006415,"error":null},{"passed":false,"prompt_tokens":4254,"cached_prompt_tokens":0,"output_tokens":401,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.66,"cost_usd":0.001057,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.020384,"latency_s":66.22,"per_attempt":[{"passed":true,"prompt_tokens":660,"cached_prompt_tokens":0,"output_tokens":10172,"reasoning_tokens":10010,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":66.22,"cost_usd":0.020384,"error":null}]}}},{"task":"exercism/custom-set","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000521,"latency_s":3.29,"per_attempt":[{"passed":true,"prompt_tokens":255,"cached_prompt_tokens":0,"output_tokens":550,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.29,"cost_usd":0.000521,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005536,"latency_s":38.73,"per_attempt":[{"passed":true,"prompt_tokens":253,"cached_prompt_tokens":0,"output_tokens":6123,"reasoning_tokens":5756,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":38.73,"cost_usd":0.005536,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000651,"latency_s":4.35,"per_attempt":[{"passed":true,"prompt_tokens":255,"cached_prompt_tokens":0,"output_tokens":318,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.35,"cost_usd":0.000651,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.124696,"latency_s":362.26,"per_attempt":[{"passed":false,"prompt_tokens":291,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":187.11,"cost_usd":0.065553,"error":null},{"passed":true,"prompt_tokens":336,"cached_prompt_tokens":0,"output_tokens":29561,"reasoning_tokens":28949,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":175.15,"cost_usd":0.059142,"error":null}]}}},{"task":"exercism/darts","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000251,"latency_s":1.4,"per_attempt":[{"passed":true,"prompt_tokens":564,"cached_prompt_tokens":0,"output_tokens":216,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.4,"cost_usd":0.000251,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00074,"latency_s":4.72,"per_attempt":[{"passed":true,"prompt_tokens":562,"cached_prompt_tokens":0,"output_tokens":760,"reasoning_tokens":679,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.72,"cost_usd":0.00074,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000352,"latency_s":2.16,"per_attempt":[{"passed":true,"prompt_tokens":564,"cached_prompt_tokens":0,"output_tokens":159,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.16,"cost_usd":0.000352,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000364,"latency_s":1.07,"per_attempt":[{"passed":true,"prompt_tokens":600,"cached_prompt_tokens":0,"output_tokens":164,"reasoning_tokens":87,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.07,"cost_usd":0.000364,"error":null}]}}},{"task":"exercism/diamond","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000323,"latency_s":2.08,"per_attempt":[{"passed":true,"prompt_tokens":401,"cached_prompt_tokens":0,"output_tokens":314,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.08,"cost_usd":0.000323,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002867,"latency_s":21.14,"per_attempt":[{"passed":true,"prompt_tokens":399,"cached_prompt_tokens":0,"output_tokens":3141,"reasoning_tokens":3003,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":21.14,"cost_usd":0.002867,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001056,"latency_s":6.77,"per_attempt":[{"passed":true,"prompt_tokens":401,"cached_prompt_tokens":0,"output_tokens":516,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.77,"cost_usd":0.001056,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004016,"latency_s":12.27,"per_attempt":[{"passed":true,"prompt_tokens":437,"cached_prompt_tokens":0,"output_tokens":1995,"reasoning_tokens":1815,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.27,"cost_usd":0.004016,"error":null}]}}},{"task":"exercism/difference-of-squares","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000249,"latency_s":1.66,"per_attempt":[{"passed":true,"prompt_tokens":287,"cached_prompt_tokens":0,"output_tokens":245,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.66,"cost_usd":0.000249,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000895,"latency_s":5.95,"per_attempt":[{"passed":true,"prompt_tokens":285,"cached_prompt_tokens":0,"output_tokens":963,"reasoning_tokens":873,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.95,"cost_usd":0.000895,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000187,"latency_s":1.21,"per_attempt":[{"passed":true,"prompt_tokens":287,"cached_prompt_tokens":0,"output_tokens":85,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.21,"cost_usd":0.000187,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001385,"latency_s":4.49,"per_attempt":[{"passed":true,"prompt_tokens":323,"cached_prompt_tokens":0,"output_tokens":683,"reasoning_tokens":594,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.49,"cost_usd":0.001385,"error":null}]}}},{"task":"exercism/diffie-hellman","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000186,"latency_s":1.11,"per_attempt":[{"passed":true,"prompt_tokens":606,"cached_prompt_tokens":0,"output_tokens":139,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.11,"cost_usd":0.000186,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001833,"latency_s":14.08,"per_attempt":[{"passed":true,"prompt_tokens":604,"cached_prompt_tokens":0,"output_tokens":1970,"reasoning_tokens":1898,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.08,"cost_usd":0.001833,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000282,"latency_s":1.73,"per_attempt":[{"passed":true,"prompt_tokens":606,"cached_prompt_tokens":0,"output_tokens":123,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.73,"cost_usd":0.000282,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003413,"latency_s":11.15,"per_attempt":[{"passed":true,"prompt_tokens":642,"cached_prompt_tokens":0,"output_tokens":1687,"reasoning_tokens":1597,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.15,"cost_usd":0.003413,"error":null}]}}},{"task":"exercism/dnd-character","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000259,"latency_s":1.53,"per_attempt":[{"passed":true,"prompt_tokens":832,"cached_prompt_tokens":0,"output_tokens":195,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.53,"cost_usd":0.000259,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00219,"latency_s":16.27,"per_attempt":[{"passed":true,"prompt_tokens":830,"cached_prompt_tokens":0,"output_tokens":2341,"reasoning_tokens":2189,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":16.27,"cost_usd":0.00219,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000358,"latency_s":2.12,"per_attempt":[{"passed":true,"prompt_tokens":832,"cached_prompt_tokens":0,"output_tokens":154,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.12,"cost_usd":0.000358,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.00371,"latency_s":11.28,"per_attempt":[{"passed":false,"prompt_tokens":868,"cached_prompt_tokens":0,"output_tokens":1377,"reasoning_tokens":1211,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.48,"cost_usd":0.002806,"error":null},{"passed":true,"prompt_tokens":1222,"cached_prompt_tokens":832,"output_tokens":428,"reasoning_tokens":273,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.8,"cost_usd":0.000904,"error":null}]}}},{"task":"exercism/dominoes","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001063,"latency_s":7.32,"per_attempt":[{"passed":true,"prompt_tokens":505,"cached_prompt_tokens":0,"output_tokens":1125,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.32,"cost_usd":0.001063,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.008175,"latency_s":57.0,"per_attempt":[{"passed":true,"prompt_tokens":503,"cached_prompt_tokens":0,"output_tokens":9027,"reasoning_tokens":8672,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":57.0,"cost_usd":0.008175,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.005812,"latency_s":36.72,"per_attempt":[{"passed":false,"prompt_tokens":505,"cached_prompt_tokens":0,"output_tokens":1710,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":21.93,"cost_usd":0.00345,"error":null},{"passed":true,"prompt_tokens":2532,"cached_prompt_tokens":0,"output_tokens":1105,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.79,"cost_usd":0.002362,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.075632,"latency_s":214.27,"per_attempt":[{"passed":false,"prompt_tokens":541,"cached_prompt_tokens":0,"output_tokens":4995,"reasoning_tokens":4746,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":33.08,"cost_usd":0.010022,"error":null},{"passed":false,"prompt_tokens":1219,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":181.19,"cost_usd":0.065609,"error":null}]}}},{"task":"exercism/dot-dsl","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.00112,"latency_s":6.53,"per_attempt":[{"passed":false,"prompt_tokens":949,"cached_prompt_tokens":0,"output_tokens":478,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.54,"cost_usd":0.000525,"error":null},{"passed":false,"prompt_tokens":1622,"cached_prompt_tokens":0,"output_tokens":481,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.99,"cost_usd":0.000595,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.01468,"latency_s":97.76,"per_attempt":[{"passed":false,"prompt_tokens":947,"cached_prompt_tokens":0,"output_tokens":9226,"reasoning_tokens":8689,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":56.31,"cost_usd":0.008398,"error":null},{"passed":false,"prompt_tokens":1668,"cached_prompt_tokens":1088,"output_tokens":6819,"reasoning_tokens":6194,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":41.45,"cost_usd":0.006282,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002517,"latency_s":15.58,"per_attempt":[{"passed":false,"prompt_tokens":949,"cached_prompt_tokens":0,"output_tokens":577,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.57,"cost_usd":0.001211,"error":null},{"passed":false,"prompt_tokens":2145,"cached_prompt_tokens":832,"output_tokens":601,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.01,"cost_usd":0.001306,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.076456,"latency_s":240.69,"per_attempt":[{"passed":false,"prompt_tokens":985,"cached_prompt_tokens":0,"output_tokens":25824,"reasoning_tokens":25229,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":164.51,"cost_usd":0.051707,"error":null},{"passed":false,"prompt_tokens":2227,"cached_prompt_tokens":832,"output_tokens":12320,"reasoning_tokens":11772,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":76.18,"cost_usd":0.024749,"error":null}]}}},{"task":"exercism/eliuds-eggs","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000188,"latency_s":1.26,"per_attempt":[{"passed":true,"prompt_tokens":596,"cached_prompt_tokens":0,"output_tokens":143,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.26,"cost_usd":0.000188,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001211,"latency_s":7.87,"per_attempt":[{"passed":true,"prompt_tokens":594,"cached_prompt_tokens":0,"output_tokens":1279,"reasoning_tokens":1230,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.87,"cost_usd":0.001211,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000288,"latency_s":1.75,"per_attempt":[{"passed":true,"prompt_tokens":596,"cached_prompt_tokens":0,"output_tokens":126,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.75,"cost_usd":0.000288,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000966,"latency_s":3.16,"per_attempt":[{"passed":true,"prompt_tokens":632,"cached_prompt_tokens":0,"output_tokens":464,"reasoning_tokens":418,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.16,"cost_usd":0.000966,"error":null}]}}},{"task":"exercism/error-handling","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.00055,"latency_s":3.52,"per_attempt":[{"passed":false,"prompt_tokens":299,"cached_prompt_tokens":0,"output_tokens":249,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.79,"cost_usd":0.000254,"error":null},{"passed":true,"prompt_tokens":742,"cached_prompt_tokens":0,"output_tokens":246,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.73,"cost_usd":0.000296,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003148,"latency_s":21.44,"per_attempt":[{"passed":true,"prompt_tokens":297,"cached_prompt_tokens":0,"output_tokens":3465,"reasoning_tokens":3345,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":21.44,"cost_usd":0.003148,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000588,"latency_s":3.52,"per_attempt":[{"passed":false,"prompt_tokens":299,"cached_prompt_tokens":0,"output_tokens":133,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.73,"cost_usd":0.000284,"error":null},{"passed":true,"prompt_tokens":634,"cached_prompt_tokens":0,"output_tokens":133,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.79,"cost_usd":0.000304,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.014132,"latency_s":45.76,"per_attempt":[{"passed":false,"prompt_tokens":335,"cached_prompt_tokens":0,"output_tokens":6577,"reasoning_tokens":6452,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":42.93,"cost_usd":0.013174,"error":null},{"passed":true,"prompt_tokens":660,"cached_prompt_tokens":0,"output_tokens":459,"reasoning_tokens":333,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.83,"cost_usd":0.000958,"error":null}]}}},{"task":"exercism/etl","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00017,"latency_s":1.03,"per_attempt":[{"passed":true,"prompt_tokens":569,"cached_prompt_tokens":0,"output_tokens":126,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.03,"cost_usd":0.00017,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00099,"latency_s":6.35,"per_attempt":[{"passed":true,"prompt_tokens":567,"cached_prompt_tokens":0,"output_tokens":1037,"reasoning_tokens":1001,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.35,"cost_usd":0.00099,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000312,"latency_s":1.93,"per_attempt":[{"passed":true,"prompt_tokens":569,"cached_prompt_tokens":0,"output_tokens":139,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.93,"cost_usd":0.000312,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000508,"latency_s":1.82,"per_attempt":[{"passed":true,"prompt_tokens":605,"cached_prompt_tokens":0,"output_tokens":236,"reasoning_tokens":192,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.82,"cost_usd":0.000508,"error":null}]}}},{"task":"exercism/flatten-array","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000147,"latency_s":1.04,"per_attempt":[{"passed":true,"prompt_tokens":304,"cached_prompt_tokens":0,"output_tokens":130,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.04,"cost_usd":0.000147,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002044,"latency_s":14.0,"per_attempt":[{"passed":true,"prompt_tokens":302,"cached_prompt_tokens":0,"output_tokens":2237,"reasoning_tokens":2175,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.0,"cost_usd":0.002044,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000222,"latency_s":1.47,"per_attempt":[{"passed":true,"prompt_tokens":304,"cached_prompt_tokens":0,"output_tokens":102,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.47,"cost_usd":0.000222,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.02031,"latency_s":64.1,"per_attempt":[{"passed":true,"prompt_tokens":340,"cached_prompt_tokens":0,"output_tokens":10145,"reasoning_tokens":9978,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":64.1,"cost_usd":0.02031,"error":null}]}}},{"task":"exercism/flower-field","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000558,"latency_s":3.7,"per_attempt":[{"passed":true,"prompt_tokens":653,"cached_prompt_tokens":0,"output_tokens":547,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.7,"cost_usd":0.000558,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004319,"latency_s":29.22,"per_attempt":[{"passed":true,"prompt_tokens":651,"cached_prompt_tokens":0,"output_tokens":4727,"reasoning_tokens":4446,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":29.22,"cost_usd":0.004319,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000701,"latency_s":9.32,"per_attempt":[{"passed":true,"prompt_tokens":653,"cached_prompt_tokens":0,"output_tokens":331,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.32,"cost_usd":0.000701,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004121,"latency_s":13.19,"per_attempt":[{"passed":true,"prompt_tokens":689,"cached_prompt_tokens":0,"output_tokens":2040,"reasoning_tokens":1708,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.19,"cost_usd":0.004121,"error":null}]}}},{"task":"exercism/food-chain","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.001559,"latency_s":9.01,"per_attempt":[{"passed":false,"prompt_tokens":727,"cached_prompt_tokens":0,"output_tokens":695,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.21,"cost_usd":0.000698,"error":null},{"passed":true,"prompt_tokens":2139,"cached_prompt_tokens":0,"output_tokens":719,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.8,"cost_usd":0.000861,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.007474,"latency_s":49.6,"per_attempt":[{"passed":true,"prompt_tokens":725,"cached_prompt_tokens":0,"output_tokens":8224,"reasoning_tokens":7817,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":49.6,"cost_usd":0.007474,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002001,"latency_s":12.34,"per_attempt":[{"passed":false,"prompt_tokens":727,"cached_prompt_tokens":0,"output_tokens":473,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.27,"cost_usd":0.00099,"error":null},{"passed":false,"prompt_tokens":1497,"cached_prompt_tokens":0,"output_tokens":461,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.07,"cost_usd":0.001012,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.008888,"latency_s":24.92,"per_attempt":[{"passed":true,"prompt_tokens":763,"cached_prompt_tokens":0,"output_tokens":4421,"reasoning_tokens":3988,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":24.92,"cost_usd":0.008888,"error":null}]}}},{"task":"exercism/forth","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.002575,"latency_s":16.49,"per_attempt":[{"passed":false,"prompt_tokens":788,"cached_prompt_tokens":0,"output_tokens":1256,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.04,"cost_usd":0.001209,"error":null},{"passed":true,"prompt_tokens":2286,"cached_prompt_tokens":0,"output_tokens":1264,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.45,"cost_usd":0.001366,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.019672,"latency_s":131.27,"per_attempt":[{"passed":false,"prompt_tokens":786,"cached_prompt_tokens":0,"output_tokens":9805,"reasoning_tokens":9228,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":59.43,"cost_usd":0.008903,"error":null},{"passed":false,"prompt_tokens":1704,"cached_prompt_tokens":0,"output_tokens":11776,"reasoning_tokens":11182,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":71.84,"cost_usd":0.010769,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.004018,"latency_s":25.13,"per_attempt":[{"passed":false,"prompt_tokens":788,"cached_prompt_tokens":0,"output_tokens":955,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.41,"cost_usd":0.001957,"error":null},{"passed":false,"prompt_tokens":2506,"cached_prompt_tokens":0,"output_tokens":955,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.72,"cost_usd":0.00206,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.118232,"latency_s":359.22,"per_attempt":[{"passed":false,"prompt_tokens":824,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":194.85,"cost_usd":0.065585,"error":null},{"passed":false,"prompt_tokens":869,"cached_prompt_tokens":0,"output_tokens":26297,"reasoning_tokens":25481,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":164.37,"cost_usd":0.052646,"error":null}]}}},{"task":"exercism/game-of-life","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000375,"latency_s":2.51,"per_attempt":[{"passed":true,"prompt_tokens":295,"cached_prompt_tokens":0,"output_tokens":384,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.51,"cost_usd":0.000375,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002764,"latency_s":18.32,"per_attempt":[{"passed":true,"prompt_tokens":293,"cached_prompt_tokens":0,"output_tokens":3038,"reasoning_tokens":2817,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":18.32,"cost_usd":0.002764,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000874,"latency_s":5.47,"per_attempt":[{"passed":true,"prompt_tokens":295,"cached_prompt_tokens":0,"output_tokens":428,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.47,"cost_usd":0.000874,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001502,"latency_s":4.41,"per_attempt":[{"passed":true,"prompt_tokens":331,"cached_prompt_tokens":0,"output_tokens":741,"reasoning_tokens":510,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.41,"cost_usd":0.001502,"error":null}]}}},{"task":"exercism/gigasecond","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000192,"latency_s":0.98,"per_attempt":[{"passed":true,"prompt_tokens":1019,"cached_prompt_tokens":0,"output_tokens":100,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.98,"cost_usd":0.000192,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000752,"latency_s":4.47,"per_attempt":[{"passed":true,"prompt_tokens":1017,"cached_prompt_tokens":0,"output_tokens":722,"reasoning_tokens":683,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.47,"cost_usd":0.000752,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000135,"latency_s":0.68,"per_attempt":[{"passed":true,"prompt_tokens":1019,"cached_prompt_tokens":0,"output_tokens":37,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.68,"cost_usd":0.000135,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000707,"latency_s":2.22,"per_attempt":[{"passed":true,"prompt_tokens":1055,"cached_prompt_tokens":0,"output_tokens":322,"reasoning_tokens":283,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.22,"cost_usd":0.000707,"error":null}]}}},{"task":"exercism/go-counting","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001713,"latency_s":10.85,"per_attempt":[{"passed":false,"prompt_tokens":845,"cached_prompt_tokens":0,"output_tokens":800,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.51,"cost_usd":0.000804,"error":null},{"passed":false,"prompt_tokens":1890,"cached_prompt_tokens":0,"output_tokens":800,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.34,"cost_usd":0.000909,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.012208,"latency_s":80.9,"per_attempt":[{"passed":false,"prompt_tokens":843,"cached_prompt_tokens":0,"output_tokens":8010,"reasoning_tokens":7341,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":48.86,"cost_usd":0.007293,"error":null},{"passed":true,"prompt_tokens":1755,"cached_prompt_tokens":0,"output_tokens":5266,"reasoning_tokens":4586,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":32.04,"cost_usd":0.004915,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.003981,"latency_s":25.25,"per_attempt":[{"passed":false,"prompt_tokens":845,"cached_prompt_tokens":0,"output_tokens":952,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.33,"cost_usd":0.001955,"error":null},{"passed":false,"prompt_tokens":2048,"cached_prompt_tokens":832,"output_tokens":964,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.92,"cost_usd":0.002026,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.131155,"latency_s":397.72,"per_attempt":[{"passed":false,"prompt_tokens":881,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":190.3,"cost_usd":0.065589,"error":null},{"passed":false,"prompt_tokens":926,"cached_prompt_tokens":832,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":207.42,"cost_usd":0.065567,"error":null}]}}},{"task":"exercism/grade-school","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000836,"latency_s":5.16,"per_attempt":[{"passed":false,"prompt_tokens":499,"cached_prompt_tokens":0,"output_tokens":347,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.31,"cost_usd":0.000362,"error":null},{"passed":true,"prompt_tokens":1046,"cached_prompt_tokens":0,"output_tokens":410,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.85,"cost_usd":0.000474,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.006934,"latency_s":46.26,"per_attempt":[{"passed":false,"prompt_tokens":497,"cached_prompt_tokens":0,"output_tokens":4453,"reasoning_tokens":4297,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":27.16,"cost_usd":0.004057,"error":null},{"passed":true,"prompt_tokens":851,"cached_prompt_tokens":0,"output_tokens":3102,"reasoning_tokens":2920,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":19.1,"cost_usd":0.002877,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.000924,"latency_s":5.72,"per_attempt":[{"passed":false,"prompt_tokens":499,"cached_prompt_tokens":0,"output_tokens":207,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.96,"cost_usd":0.000444,"error":null},{"passed":false,"prompt_tokens":906,"cached_prompt_tokens":0,"output_tokens":213,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.76,"cost_usd":0.00048,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.043061,"latency_s":129.06,"per_attempt":[{"passed":false,"prompt_tokens":535,"cached_prompt_tokens":0,"output_tokens":19615,"reasoning_tokens":19441,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":116.63,"cost_usd":0.039262,"error":null},{"passed":true,"prompt_tokens":955,"cached_prompt_tokens":0,"output_tokens":1871,"reasoning_tokens":1667,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.43,"cost_usd":0.003799,"error":null}]}}},{"task":"exercism/grains","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000265,"latency_s":1.85,"per_attempt":[{"passed":true,"prompt_tokens":495,"cached_prompt_tokens":0,"output_tokens":239,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.85,"cost_usd":0.000265,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000938,"latency_s":6.03,"per_attempt":[{"passed":true,"prompt_tokens":493,"cached_prompt_tokens":0,"output_tokens":988,"reasoning_tokens":919,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.03,"cost_usd":0.000938,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000166,"latency_s":1.15,"per_attempt":[{"passed":true,"prompt_tokens":495,"cached_prompt_tokens":0,"output_tokens":68,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.15,"cost_usd":0.000166,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000454,"latency_s":1.45,"per_attempt":[{"passed":true,"prompt_tokens":531,"cached_prompt_tokens":0,"output_tokens":211,"reasoning_tokens":141,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.45,"cost_usd":0.000454,"error":null}]}}},{"task":"exercism/grep","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001233,"latency_s":7.92,"per_attempt":[{"passed":false,"prompt_tokens":460,"cached_prompt_tokens":0,"output_tokens":561,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.85,"cost_usd":0.000551,"error":null},{"passed":false,"prompt_tokens":1324,"cached_prompt_tokens":0,"output_tokens":611,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.07,"cost_usd":0.000682,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.011623,"latency_s":78.17,"per_attempt":[{"passed":false,"prompt_tokens":458,"cached_prompt_tokens":0,"output_tokens":7130,"reasoning_tokens":6842,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":43.8,"cost_usd":0.006463,"error":null},{"passed":false,"prompt_tokens":1047,"cached_prompt_tokens":0,"output_tokens":5617,"reasoning_tokens":5307,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":34.37,"cost_usd":0.00516,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.00225,"latency_s":14.45,"per_attempt":[{"passed":false,"prompt_tokens":460,"cached_prompt_tokens":0,"output_tokens":536,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.16,"cost_usd":0.0011,"error":null},{"passed":false,"prompt_tokens":1303,"cached_prompt_tokens":0,"output_tokens":536,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.29,"cost_usd":0.00115,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.083022,"latency_s":260.43,"per_attempt":[{"passed":false,"prompt_tokens":496,"cached_prompt_tokens":0,"output_tokens":19338,"reasoning_tokens":18932,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":121.53,"cost_usd":0.038706,"error":null},{"passed":false,"prompt_tokens":1206,"cached_prompt_tokens":0,"output_tokens":22122,"reasoning_tokens":21501,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":138.9,"cost_usd":0.044316,"error":null}]}}},{"task":"exercism/hamming","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00012,"latency_s":0.58,"per_attempt":[{"passed":true,"prompt_tokens":600,"cached_prompt_tokens":0,"output_tokens":67,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.58,"cost_usd":0.00012,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000584,"latency_s":3.71,"per_attempt":[{"passed":true,"prompt_tokens":598,"cached_prompt_tokens":0,"output_tokens":583,"reasoning_tokens":515,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.71,"cost_usd":0.000584,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00016,"latency_s":0.93,"per_attempt":[{"passed":true,"prompt_tokens":600,"cached_prompt_tokens":0,"output_tokens":62,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.93,"cost_usd":0.00016,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000576,"latency_s":1.8,"per_attempt":[{"passed":true,"prompt_tokens":636,"cached_prompt_tokens":0,"output_tokens":269,"reasoning_tokens":205,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.8,"cost_usd":0.000576,"error":null}]}}},{"task":"exercism/hangman","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00037,"latency_s":2.38,"per_attempt":[{"passed":true,"prompt_tokens":629,"cached_prompt_tokens":0,"output_tokens":341,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.38,"cost_usd":0.00037,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004809,"latency_s":32.45,"per_attempt":[{"passed":true,"prompt_tokens":627,"cached_prompt_tokens":0,"output_tokens":5274,"reasoning_tokens":5012,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":32.45,"cost_usd":0.004809,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001202,"latency_s":7.73,"per_attempt":[{"passed":false,"prompt_tokens":629,"cached_prompt_tokens":0,"output_tokens":274,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.94,"cost_usd":0.000586,"error":null},{"passed":false,"prompt_tokens":1144,"cached_prompt_tokens":0,"output_tokens":274,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.79,"cost_usd":0.000617,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.058742,"latency_s":184.16,"per_attempt":[{"passed":false,"prompt_tokens":665,"cached_prompt_tokens":0,"output_tokens":17451,"reasoning_tokens":17203,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":113.49,"cost_usd":0.034942,"error":null},{"passed":false,"prompt_tokens":1237,"cached_prompt_tokens":0,"output_tokens":11863,"reasoning_tokens":11550,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":70.67,"cost_usd":0.0238,"error":null}]}}},{"task":"exercism/hello-world","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":3.6e-05,"latency_s":0.23,"per_attempt":[{"passed":true,"prompt_tokens":209,"cached_prompt_tokens":0,"output_tokens":17,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.23,"cost_usd":3.6e-05,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000594,"latency_s":3.95,"per_attempt":[{"passed":true,"prompt_tokens":207,"cached_prompt_tokens":0,"output_tokens":637,"reasoning_tokens":618,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.95,"cost_usd":0.000594,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":4.7e-05,"latency_s":0.35,"per_attempt":[{"passed":true,"prompt_tokens":209,"cached_prompt_tokens":0,"output_tokens":17,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.35,"cost_usd":4.7e-05,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000153,"latency_s":0.6,"per_attempt":[{"passed":true,"prompt_tokens":245,"cached_prompt_tokens":0,"output_tokens":69,"reasoning_tokens":50,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.6,"cost_usd":0.000153,"error":null}]}}},{"task":"exercism/hexadecimal","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000326,"latency_s":2.4,"per_attempt":[{"passed":true,"prompt_tokens":181,"cached_prompt_tokens":0,"output_tokens":342,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.4,"cost_usd":0.000326,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.005207,"latency_s":35.0,"per_attempt":[{"passed":false,"prompt_tokens":179,"cached_prompt_tokens":0,"output_tokens":2717,"reasoning_tokens":2585,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":16.58,"cost_usd":0.002463,"error":null},{"passed":true,"prompt_tokens":471,"cached_prompt_tokens":0,"output_tokens":2996,"reasoning_tokens":2860,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":18.42,"cost_usd":0.002743,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000597,"latency_s":4.1,"per_attempt":[{"passed":true,"prompt_tokens":181,"cached_prompt_tokens":0,"output_tokens":293,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.1,"cost_usd":0.000597,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.007401,"latency_s":23.48,"per_attempt":[{"passed":false,"prompt_tokens":217,"cached_prompt_tokens":0,"output_tokens":2364,"reasoning_tokens":2110,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.65,"cost_usd":0.004741,"error":null},{"passed":true,"prompt_tokens":635,"cached_prompt_tokens":0,"output_tokens":1311,"reasoning_tokens":1037,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.83,"cost_usd":0.00266,"error":null}]}}},{"task":"exercism/high-scores","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.0001,"latency_s":0.75,"per_attempt":[{"passed":true,"prompt_tokens":299,"cached_prompt_tokens":0,"output_tokens":78,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.75,"cost_usd":0.0001,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001681,"latency_s":11.08,"per_attempt":[{"passed":true,"prompt_tokens":297,"cached_prompt_tokens":0,"output_tokens":1835,"reasoning_tokens":1755,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.08,"cost_usd":0.001681,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000389,"latency_s":2.51,"per_attempt":[{"passed":false,"prompt_tokens":299,"cached_prompt_tokens":0,"output_tokens":90,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.29,"cost_usd":0.000198,"error":null},{"passed":true,"prompt_tokens":589,"cached_prompt_tokens":0,"output_tokens":78,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.22,"cost_usd":0.000191,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002176,"latency_s":6.88,"per_attempt":[{"passed":false,"prompt_tokens":335,"cached_prompt_tokens":0,"output_tokens":265,"reasoning_tokens":187,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.82,"cost_usd":0.00055,"error":null},{"passed":false,"prompt_tokens":634,"cached_prompt_tokens":0,"output_tokens":794,"reasoning_tokens":685,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.06,"cost_usd":0.001626,"error":null}]}}},{"task":"exercism/house","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001398,"latency_s":8.26,"per_attempt":[{"passed":false,"prompt_tokens":830,"cached_prompt_tokens":0,"output_tokens":612,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.99,"cost_usd":0.000634,"error":null},{"passed":false,"prompt_tokens":2179,"cached_prompt_tokens":0,"output_tokens":607,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.27,"cost_usd":0.000764,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005197,"latency_s":34.81,"per_attempt":[{"passed":true,"prompt_tokens":828,"cached_prompt_tokens":0,"output_tokens":5682,"reasoning_tokens":5371,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":34.81,"cost_usd":0.005197,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.005953,"latency_s":37.25,"per_attempt":[{"passed":false,"prompt_tokens":830,"cached_prompt_tokens":0,"output_tokens":1585,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":20.58,"cost_usd":0.00322,"error":null},{"passed":false,"prompt_tokens":3154,"cached_prompt_tokens":0,"output_tokens":1272,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":16.67,"cost_usd":0.002733,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.027318,"latency_s":78.33,"per_attempt":[{"passed":false,"prompt_tokens":866,"cached_prompt_tokens":0,"output_tokens":11669,"reasoning_tokens":11227,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":67.16,"cost_usd":0.02339,"error":null},{"passed":true,"prompt_tokens":2046,"cached_prompt_tokens":832,"output_tokens":1915,"reasoning_tokens":1528,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.17,"cost_usd":0.003928,"error":null}]}}},{"task":"exercism/isbn-verifier","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000396,"latency_s":2.44,"per_attempt":[{"passed":true,"prompt_tokens":596,"cached_prompt_tokens":0,"output_tokens":374,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.44,"cost_usd":0.000396,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00418,"latency_s":27.99,"per_attempt":[{"passed":true,"prompt_tokens":594,"cached_prompt_tokens":0,"output_tokens":4578,"reasoning_tokens":4452,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":27.99,"cost_usd":0.00418,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000398,"latency_s":2.59,"per_attempt":[{"passed":true,"prompt_tokens":596,"cached_prompt_tokens":0,"output_tokens":181,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.59,"cost_usd":0.000398,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.020472,"latency_s":63.46,"per_attempt":[{"passed":true,"prompt_tokens":632,"cached_prompt_tokens":0,"output_tokens":10217,"reasoning_tokens":10045,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":63.46,"cost_usd":0.020472,"error":null}]}}},{"task":"exercism/isogram","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000178,"latency_s":1.29,"per_attempt":[{"passed":true,"prompt_tokens":173,"cached_prompt_tokens":0,"output_tokens":178,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.29,"cost_usd":0.000178,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001984,"latency_s":13.52,"per_attempt":[{"passed":true,"prompt_tokens":171,"cached_prompt_tokens":0,"output_tokens":2185,"reasoning_tokens":2140,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.52,"cost_usd":0.001984,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000142,"latency_s":1.03,"per_attempt":[{"passed":true,"prompt_tokens":173,"cached_prompt_tokens":0,"output_tokens":66,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.03,"cost_usd":0.000142,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003887,"latency_s":12.79,"per_attempt":[{"passed":true,"prompt_tokens":209,"cached_prompt_tokens":0,"output_tokens":1937,"reasoning_tokens":1880,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.79,"cost_usd":0.003887,"error":null}]}}},{"task":"exercism/killer-sudoku-helper","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000866,"latency_s":4.22,"per_attempt":[{"passed":false,"prompt_tokens":1367,"cached_prompt_tokens":0,"output_tokens":296,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.04,"cost_usd":0.000403,"error":null},{"passed":true,"prompt_tokens":2126,"cached_prompt_tokens":1088,"output_tokens":302,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.18,"cost_usd":0.000463,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002858,"latency_s":18.69,"per_attempt":[{"passed":true,"prompt_tokens":1365,"cached_prompt_tokens":1088,"output_tokens":3048,"reasoning_tokens":2981,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":18.69,"cost_usd":0.002858,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000882,"latency_s":5.05,"per_attempt":[{"passed":false,"prompt_tokens":1367,"cached_prompt_tokens":0,"output_tokens":187,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.59,"cost_usd":0.000456,"error":null},{"passed":true,"prompt_tokens":1787,"cached_prompt_tokens":832,"output_tokens":172,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.46,"cost_usd":0.000426,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.011213,"latency_s":37.32,"per_attempt":[{"passed":false,"prompt_tokens":1403,"cached_prompt_tokens":0,"output_tokens":5065,"reasoning_tokens":4964,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":33.89,"cost_usd":0.010214,"error":null},{"passed":true,"prompt_tokens":1735,"cached_prompt_tokens":832,"output_tokens":460,"reasoning_tokens":362,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.43,"cost_usd":0.000999,"error":null}]}}},{"task":"exercism/kindergarten-garden","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000539,"latency_s":3.12,"per_attempt":[{"passed":true,"prompt_tokens":1246,"cached_prompt_tokens":0,"output_tokens":460,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.12,"cost_usd":0.000539,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003941,"latency_s":25.81,"per_attempt":[{"passed":true,"prompt_tokens":1244,"cached_prompt_tokens":1088,"output_tokens":4265,"reasoning_tokens":4018,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":25.81,"cost_usd":0.003941,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000669,"latency_s":4.07,"per_attempt":[{"passed":true,"prompt_tokens":1246,"cached_prompt_tokens":0,"output_tokens":297,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.07,"cost_usd":0.000669,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.015257,"latency_s":46.63,"per_attempt":[{"passed":true,"prompt_tokens":1282,"cached_prompt_tokens":0,"output_tokens":7590,"reasoning_tokens":7335,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":46.63,"cost_usd":0.015257,"error":null}]}}},{"task":"exercism/knapsack","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000263,"latency_s":1.8,"per_attempt":[{"passed":true,"prompt_tokens":450,"cached_prompt_tokens":0,"output_tokens":242,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.8,"cost_usd":0.000263,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002631,"latency_s":17.4,"per_attempt":[{"passed":true,"prompt_tokens":448,"cached_prompt_tokens":0,"output_tokens":2874,"reasoning_tokens":2775,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":17.4,"cost_usd":0.002631,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000503,"latency_s":3.13,"per_attempt":[{"passed":true,"prompt_tokens":450,"cached_prompt_tokens":0,"output_tokens":238,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.13,"cost_usd":0.000503,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.069075,"latency_s":216.49,"per_attempt":[{"passed":false,"prompt_tokens":486,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":205.33,"cost_usd":0.065565,"error":null},{"passed":true,"prompt_tokens":531,"cached_prompt_tokens":0,"output_tokens":1739,"reasoning_tokens":1580,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.16,"cost_usd":0.00351,"error":null}]}}},{"task":"exercism/largest-series-product","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000372,"latency_s":2.3,"per_attempt":[{"passed":true,"prompt_tokens":793,"cached_prompt_tokens":0,"output_tokens":325,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.3,"cost_usd":0.000372,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005109,"latency_s":33.84,"per_attempt":[{"passed":true,"prompt_tokens":791,"cached_prompt_tokens":0,"output_tokens":5589,"reasoning_tokens":5434,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":33.84,"cost_usd":0.005109,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.001566,"latency_s":9.59,"per_attempt":[{"passed":false,"prompt_tokens":793,"cached_prompt_tokens":0,"output_tokens":353,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.84,"cost_usd":0.000754,"error":null},{"passed":true,"prompt_tokens":1412,"cached_prompt_tokens":0,"output_tokens":364,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.75,"cost_usd":0.000813,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00833,"latency_s":26.09,"per_attempt":[{"passed":true,"prompt_tokens":829,"cached_prompt_tokens":0,"output_tokens":4140,"reasoning_tokens":3963,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":26.09,"cost_usd":0.00833,"error":null}]}}},{"task":"exercism/leap","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000175,"latency_s":1.33,"per_attempt":[{"passed":true,"prompt_tokens":267,"cached_prompt_tokens":0,"output_tokens":165,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.33,"cost_usd":0.000175,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000719,"latency_s":4.76,"per_attempt":[{"passed":true,"prompt_tokens":265,"cached_prompt_tokens":0,"output_tokens":769,"reasoning_tokens":722,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.76,"cost_usd":0.000719,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000144,"latency_s":1.0,"per_attempt":[{"passed":true,"prompt_tokens":267,"cached_prompt_tokens":0,"output_tokens":64,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.0,"cost_usd":0.000144,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000354,"latency_s":1.03,"per_attempt":[{"passed":true,"prompt_tokens":303,"cached_prompt_tokens":0,"output_tokens":168,"reasoning_tokens":123,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.03,"cost_usd":0.000354,"error":null}]}}},{"task":"exercism/ledger","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.003041,"latency_s":17.51,"per_attempt":[{"passed":false,"prompt_tokens":2515,"cached_prompt_tokens":0,"output_tokens":1306,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.43,"cost_usd":0.001427,"error":null},{"passed":false,"prompt_tokens":4257,"cached_prompt_tokens":2176,"output_tokens":1369,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.08,"cost_usd":0.001614,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.018413,"latency_s":120.28,"per_attempt":[{"passed":false,"prompt_tokens":2513,"cached_prompt_tokens":2176,"output_tokens":9128,"reasoning_tokens":8321,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":55.77,"cost_usd":0.008423,"error":null},{"passed":true,"prompt_tokens":3767,"cached_prompt_tokens":2176,"output_tokens":10730,"reasoning_tokens":9844,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":64.51,"cost_usd":0.00999,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.00527,"latency_s":32.24,"per_attempt":[{"passed":false,"prompt_tokens":2515,"cached_prompt_tokens":0,"output_tokens":1236,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":16.16,"cost_usd":0.002623,"error":null},{"passed":false,"prompt_tokens":4173,"cached_prompt_tokens":2496,"output_tokens":1236,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":16.08,"cost_usd":0.002648,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.054899,"latency_s":171.33,"per_attempt":[{"passed":true,"prompt_tokens":2551,"cached_prompt_tokens":0,"output_tokens":27373,"reasoning_tokens":26621,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":171.33,"cost_usd":0.054899,"error":null}]}}},{"task":"exercism/line-up","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000258,"latency_s":1.65,"per_attempt":[{"passed":true,"prompt_tokens":419,"cached_prompt_tokens":0,"output_tokens":240,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.65,"cost_usd":0.000258,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002145,"latency_s":14.2,"per_attempt":[{"passed":true,"prompt_tokens":417,"cached_prompt_tokens":0,"output_tokens":2337,"reasoning_tokens":2203,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.2,"cost_usd":0.002145,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000305,"latency_s":2.07,"per_attempt":[{"passed":true,"prompt_tokens":419,"cached_prompt_tokens":0,"output_tokens":140,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.07,"cost_usd":0.000305,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000681,"latency_s":1.8,"per_attempt":[{"passed":true,"prompt_tokens":455,"cached_prompt_tokens":0,"output_tokens":327,"reasoning_tokens":189,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.8,"cost_usd":0.000681,"error":null}]}}},{"task":"exercism/linked-list","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000788,"latency_s":4.81,"per_attempt":[{"passed":true,"prompt_tokens":1307,"cached_prompt_tokens":0,"output_tokens":730,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.81,"cost_usd":0.000788,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005032,"latency_s":33.04,"per_attempt":[{"passed":true,"prompt_tokens":1305,"cached_prompt_tokens":1088,"output_tokens":5470,"reasoning_tokens":4919,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":33.04,"cost_usd":0.005032,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001264,"latency_s":7.75,"per_attempt":[{"passed":true,"prompt_tokens":1307,"cached_prompt_tokens":0,"output_tokens":593,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.75,"cost_usd":0.001264,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.007485,"latency_s":20.09,"per_attempt":[{"passed":true,"prompt_tokens":1343,"cached_prompt_tokens":0,"output_tokens":3702,"reasoning_tokens":3124,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":20.09,"cost_usd":0.007485,"error":null}]}}},{"task":"exercism/list-ops","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000473,"latency_s":3.25,"per_attempt":[{"passed":true,"prompt_tokens":471,"cached_prompt_tokens":0,"output_tokens":473,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.25,"cost_usd":0.000473,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.007761,"latency_s":51.73,"per_attempt":[{"passed":false,"prompt_tokens":469,"cached_prompt_tokens":0,"output_tokens":5312,"reasoning_tokens":5008,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":32.2,"cost_usd":0.004828,"error":null},{"passed":true,"prompt_tokens":999,"cached_prompt_tokens":0,"output_tokens":3148,"reasoning_tokens":2844,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":19.53,"cost_usd":0.002933,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001159,"latency_s":7.19,"per_attempt":[{"passed":false,"prompt_tokens":471,"cached_prompt_tokens":0,"output_tokens":268,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.62,"cost_usd":0.000564,"error":null},{"passed":false,"prompt_tokens":971,"cached_prompt_tokens":0,"output_tokens":268,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.57,"cost_usd":0.000594,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.026964,"latency_s":84.77,"per_attempt":[{"passed":false,"prompt_tokens":507,"cached_prompt_tokens":0,"output_tokens":11892,"reasoning_tokens":11507,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":75.21,"cost_usd":0.023814,"error":null},{"passed":true,"prompt_tokens":1122,"cached_prompt_tokens":0,"output_tokens":1541,"reasoning_tokens":1165,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.56,"cost_usd":0.003149,"error":null}]}}},{"task":"exercism/luhn","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000288,"latency_s":1.57,"per_attempt":[{"passed":true,"prompt_tokens":816,"cached_prompt_tokens":0,"output_tokens":229,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.57,"cost_usd":0.000288,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004411,"latency_s":29.33,"per_attempt":[{"passed":true,"prompt_tokens":814,"cached_prompt_tokens":0,"output_tokens":4811,"reasoning_tokens":4661,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":29.33,"cost_usd":0.004411,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000551,"latency_s":3.39,"per_attempt":[{"passed":true,"prompt_tokens":816,"cached_prompt_tokens":0,"output_tokens":251,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.39,"cost_usd":0.000551,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001943,"latency_s":5.71,"per_attempt":[{"passed":true,"prompt_tokens":852,"cached_prompt_tokens":0,"output_tokens":946,"reasoning_tokens":781,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.71,"cost_usd":0.001943,"error":null}]}}},{"task":"exercism/markdown","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001486,"latency_s":8.53,"per_attempt":[{"passed":false,"prompt_tokens":1075,"cached_prompt_tokens":0,"output_tokens":654,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.23,"cost_usd":0.000696,"error":null},{"passed":false,"prompt_tokens":2017,"cached_prompt_tokens":0,"output_tokens":654,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.3,"cost_usd":0.00079,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.013733,"latency_s":90.75,"per_attempt":[{"passed":false,"prompt_tokens":1073,"cached_prompt_tokens":0,"output_tokens":7179,"reasoning_tokens":6691,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":44.16,"cost_usd":0.006568,"error":null},{"passed":true,"prompt_tokens":1882,"cached_prompt_tokens":0,"output_tokens":7751,"reasoning_tokens":7235,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":46.59,"cost_usd":0.007164,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001143,"latency_s":7.13,"per_attempt":[{"passed":true,"prompt_tokens":1075,"cached_prompt_tokens":0,"output_tokens":539,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.13,"cost_usd":0.001143,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.064947,"latency_s":211.92,"per_attempt":[{"passed":true,"prompt_tokens":1111,"cached_prompt_tokens":0,"output_tokens":32440,"reasoning_tokens":32036,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":211.92,"cost_usd":0.064947,"error":null}]}}},{"task":"exercism/matching-brackets","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000261,"latency_s":1.73,"per_attempt":[{"passed":true,"prompt_tokens":281,"cached_prompt_tokens":0,"output_tokens":259,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.73,"cost_usd":0.000261,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001513,"latency_s":9.94,"per_attempt":[{"passed":true,"prompt_tokens":279,"cached_prompt_tokens":0,"output_tokens":1650,"reasoning_tokens":1559,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.94,"cost_usd":0.001513,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000265,"latency_s":1.71,"per_attempt":[{"passed":true,"prompt_tokens":281,"cached_prompt_tokens":0,"output_tokens":124,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.71,"cost_usd":0.000265,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000927,"latency_s":2.92,"per_attempt":[{"passed":true,"prompt_tokens":317,"cached_prompt_tokens":0,"output_tokens":454,"reasoning_tokens":351,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.92,"cost_usd":0.000927,"error":null}]}}},{"task":"exercism/matrix","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000192,"latency_s":1.1,"per_attempt":[{"passed":true,"prompt_tokens":588,"cached_prompt_tokens":0,"output_tokens":148,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.1,"cost_usd":0.000192,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00317,"latency_s":21.54,"per_attempt":[{"passed":true,"prompt_tokens":586,"cached_prompt_tokens":0,"output_tokens":3457,"reasoning_tokens":3366,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":21.54,"cost_usd":0.00317,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000239,"latency_s":1.46,"per_attempt":[{"passed":true,"prompt_tokens":588,"cached_prompt_tokens":0,"output_tokens":102,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.46,"cost_usd":0.000239,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002291,"latency_s":7.65,"per_attempt":[{"passed":true,"prompt_tokens":624,"cached_prompt_tokens":0,"output_tokens":1127,"reasoning_tokens":1026,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.65,"cost_usd":0.002291,"error":null}]}}},{"task":"exercism/meetup","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00096,"latency_s":5.73,"per_attempt":[{"passed":true,"prompt_tokens":1395,"cached_prompt_tokens":0,"output_tokens":912,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.73,"cost_usd":0.00096,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006213,"latency_s":41.79,"per_attempt":[{"passed":true,"prompt_tokens":1393,"cached_prompt_tokens":1088,"output_tokens":6773,"reasoning_tokens":6373,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":41.79,"cost_usd":0.006213,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00146,"latency_s":9.13,"per_attempt":[{"passed":true,"prompt_tokens":1395,"cached_prompt_tokens":0,"output_tokens":688,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.13,"cost_usd":0.00146,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.016244,"latency_s":44.15,"per_attempt":[{"passed":true,"prompt_tokens":1431,"cached_prompt_tokens":0,"output_tokens":8079,"reasoning_tokens":7594,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":44.15,"cost_usd":0.016244,"error":null}]}}},{"task":"exercism/minesweeper","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000557,"latency_s":3.66,"per_attempt":[{"passed":true,"prompt_tokens":588,"cached_prompt_tokens":0,"output_tokens":553,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.66,"cost_usd":0.000557,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004066,"latency_s":27.8,"per_attempt":[{"passed":true,"prompt_tokens":586,"cached_prompt_tokens":0,"output_tokens":4453,"reasoning_tokens":4165,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":27.8,"cost_usd":0.004066,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000889,"latency_s":5.44,"per_attempt":[{"passed":true,"prompt_tokens":588,"cached_prompt_tokens":0,"output_tokens":427,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.44,"cost_usd":0.000889,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006171,"latency_s":19.29,"per_attempt":[{"passed":true,"prompt_tokens":624,"cached_prompt_tokens":0,"output_tokens":3067,"reasoning_tokens":2746,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":19.29,"cost_usd":0.006171,"error":null}]}}},{"task":"exercism/nth-prime","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000243,"latency_s":1.51,"per_attempt":[{"passed":true,"prompt_tokens":429,"cached_prompt_tokens":0,"output_tokens":222,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.51,"cost_usd":0.000243,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002517,"latency_s":17.08,"per_attempt":[{"passed":true,"prompt_tokens":427,"cached_prompt_tokens":0,"output_tokens":2749,"reasoning_tokens":2612,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":17.08,"cost_usd":0.002517,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000408,"latency_s":2.5,"per_attempt":[{"passed":true,"prompt_tokens":429,"cached_prompt_tokens":0,"output_tokens":191,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.5,"cost_usd":0.000408,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.042614,"latency_s":133.85,"per_attempt":[{"passed":true,"prompt_tokens":465,"cached_prompt_tokens":0,"output_tokens":21293,"reasoning_tokens":21112,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":133.85,"cost_usd":0.042614,"error":null}]}}},{"task":"exercism/nucleotide-count","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000439,"latency_s":2.77,"per_attempt":[{"passed":true,"prompt_tokens":397,"cached_prompt_tokens":0,"output_tokens":444,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.77,"cost_usd":0.000439,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003046,"latency_s":20.55,"per_attempt":[{"passed":true,"prompt_tokens":395,"cached_prompt_tokens":0,"output_tokens":3340,"reasoning_tokens":3171,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":20.55,"cost_usd":0.003046,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000826,"latency_s":5.07,"per_attempt":[{"passed":true,"prompt_tokens":397,"cached_prompt_tokens":0,"output_tokens":401,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.07,"cost_usd":0.000826,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001208,"latency_s":3.49,"per_attempt":[{"passed":true,"prompt_tokens":433,"cached_prompt_tokens":0,"output_tokens":591,"reasoning_tokens":423,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.49,"cost_usd":0.001208,"error":null}]}}},{"task":"exercism/ocr-numbers","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000688,"latency_s":4.16,"per_attempt":[{"passed":true,"prompt_tokens":864,"cached_prompt_tokens":0,"output_tokens":669,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.16,"cost_usd":0.000688,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.008823,"latency_s":58.57,"per_attempt":[{"passed":true,"prompt_tokens":862,"cached_prompt_tokens":0,"output_tokens":9707,"reasoning_tokens":9376,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":58.57,"cost_usd":0.008823,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001582,"latency_s":10.05,"per_attempt":[{"passed":true,"prompt_tokens":864,"cached_prompt_tokens":0,"output_tokens":765,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.05,"cost_usd":0.001582,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.031382,"latency_s":104.85,"per_attempt":[{"passed":true,"prompt_tokens":900,"cached_prompt_tokens":0,"output_tokens":15664,"reasoning_tokens":15198,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":104.85,"cost_usd":0.031382,"error":null}]}}},{"task":"exercism/octal","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000525,"latency_s":2.81,"per_attempt":[{"passed":false,"prompt_tokens":465,"cached_prompt_tokens":0,"output_tokens":219,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.42,"cost_usd":0.000244,"error":null},{"passed":true,"prompt_tokens":870,"cached_prompt_tokens":0,"output_tokens":216,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.39,"cost_usd":0.000281,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.004372,"latency_s":28.55,"per_attempt":[{"passed":false,"prompt_tokens":463,"cached_prompt_tokens":0,"output_tokens":2942,"reasoning_tokens":2878,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":18.03,"cost_usd":0.002694,"error":null},{"passed":true,"prompt_tokens":711,"cached_prompt_tokens":0,"output_tokens":1785,"reasoning_tokens":1716,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.52,"cost_usd":0.001678,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000844,"latency_s":4.95,"per_attempt":[{"passed":false,"prompt_tokens":465,"cached_prompt_tokens":0,"output_tokens":181,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.29,"cost_usd":0.00039,"error":null},{"passed":true,"prompt_tokens":836,"cached_prompt_tokens":0,"output_tokens":202,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.66,"cost_usd":0.000454,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.052053,"latency_s":125.01,"per_attempt":[{"passed":false,"prompt_tokens":501,"cached_prompt_tokens":0,"output_tokens":1220,"reasoning_tokens":1129,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.11,"cost_usd":0.00247,"error":null},{"passed":false,"prompt_tokens":780,"cached_prompt_tokens":0,"output_tokens":24768,"reasoning_tokens":24766,"hit_token_cap":false,"no_code_in_reply":true,"latency_s":116.9,"cost_usd":0.049583,"error":null}]}}},{"task":"exercism/paasio","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001344,"latency_s":8.43,"per_attempt":[{"passed":false,"prompt_tokens":478,"cached_prompt_tokens":0,"output_tokens":626,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.08,"cost_usd":0.000611,"error":null},{"passed":false,"prompt_tokens":1491,"cached_prompt_tokens":0,"output_tokens":648,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.35,"cost_usd":0.000732,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.013639,"latency_s":89.39,"per_attempt":[{"passed":false,"prompt_tokens":476,"cached_prompt_tokens":0,"output_tokens":7233,"reasoning_tokens":6655,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":43.38,"cost_usd":0.006557,"error":null},{"passed":false,"prompt_tokens":1439,"cached_prompt_tokens":0,"output_tokens":7709,"reasoning_tokens":7133,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":46.01,"cost_usd":0.007082,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002513,"latency_s":15.77,"per_attempt":[{"passed":false,"prompt_tokens":478,"cached_prompt_tokens":0,"output_tokens":593,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.98,"cost_usd":0.001215,"error":null},{"passed":false,"prompt_tokens":1464,"cached_prompt_tokens":0,"output_tokens":605,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.79,"cost_usd":0.001298,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.116949,"latency_s":381.43,"per_attempt":[{"passed":false,"prompt_tokens":514,"cached_prompt_tokens":0,"output_tokens":26063,"reasoning_tokens":25298,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":160.3,"cost_usd":0.052157,"error":null},{"passed":false,"prompt_tokens":1670,"cached_prompt_tokens":0,"output_tokens":32346,"reasoning_tokens":31497,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":221.13,"cost_usd":0.064792,"error":null}]}}},{"task":"exercism/palindrome-products","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000821,"latency_s":5.23,"per_attempt":[{"passed":true,"prompt_tokens":1078,"cached_prompt_tokens":0,"output_tokens":792,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.23,"cost_usd":0.000821,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.008637,"latency_s":56.71,"per_attempt":[{"passed":true,"prompt_tokens":1076,"cached_prompt_tokens":0,"output_tokens":9477,"reasoning_tokens":8852,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":56.71,"cost_usd":0.008637,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001167,"latency_s":7.57,"per_attempt":[{"passed":true,"prompt_tokens":1078,"cached_prompt_tokens":0,"output_tokens":551,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.57,"cost_usd":0.001167,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.131183,"latency_s":423.01,"per_attempt":[{"passed":false,"prompt_tokens":1114,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":206.32,"cost_usd":0.065603,"error":null},{"passed":false,"prompt_tokens":1159,"cached_prompt_tokens":832,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":216.69,"cost_usd":0.065581,"error":null}]}}},{"task":"exercism/pangram","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000184,"latency_s":1.07,"per_attempt":[{"passed":true,"prompt_tokens":318,"cached_prompt_tokens":0,"output_tokens":169,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.07,"cost_usd":0.000184,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001092,"latency_s":7.05,"per_attempt":[{"passed":true,"prompt_tokens":316,"cached_prompt_tokens":0,"output_tokens":1178,"reasoning_tokens":1149,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.05,"cost_usd":0.001092,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":8.5e-05,"latency_s":0.53,"per_attempt":[{"passed":true,"prompt_tokens":318,"cached_prompt_tokens":0,"output_tokens":33,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.53,"cost_usd":8.5e-05,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000369,"latency_s":1.38,"per_attempt":[{"passed":true,"prompt_tokens":354,"cached_prompt_tokens":0,"output_tokens":174,"reasoning_tokens":139,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.38,"cost_usd":0.000369,"error":null}]}}},{"task":"exercism/parallel-letter-frequency","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000344,"latency_s":2.55,"per_attempt":[{"passed":true,"prompt_tokens":142,"cached_prompt_tokens":0,"output_tokens":366,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.55,"cost_usd":0.000344,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003195,"latency_s":20.91,"per_attempt":[{"passed":true,"prompt_tokens":140,"cached_prompt_tokens":0,"output_tokens":3534,"reasoning_tokens":3420,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":20.91,"cost_usd":0.003195,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000459,"latency_s":3.19,"per_attempt":[{"passed":true,"prompt_tokens":142,"cached_prompt_tokens":0,"output_tokens":225,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.19,"cost_usd":0.000459,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.073896,"latency_s":404.96,"per_attempt":[{"passed":false,"prompt_tokens":178,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":214.15,"cost_usd":0.065547,"error":null},{"passed":false,"prompt_tokens":223,"cached_prompt_tokens":0,"output_tokens":4168,"reasoning_tokens":4167,"hit_token_cap":false,"no_code_in_reply":true,"latency_s":190.81,"cost_usd":0.008349,"error":null}]}}},{"task":"exercism/pascals-triangle","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000427,"latency_s":2.32,"per_attempt":[{"passed":true,"prompt_tokens":1353,"cached_prompt_tokens":0,"output_tokens":324,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.32,"cost_usd":0.000427,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003223,"latency_s":20.36,"per_attempt":[{"passed":true,"prompt_tokens":1351,"cached_prompt_tokens":1088,"output_tokens":3455,"reasoning_tokens":3333,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":20.36,"cost_usd":0.003223,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000399,"latency_s":2.21,"per_attempt":[{"passed":true,"prompt_tokens":1353,"cached_prompt_tokens":0,"output_tokens":159,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.21,"cost_usd":0.000399,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.051079,"latency_s":164.81,"per_attempt":[{"passed":false,"prompt_tokens":1389,"cached_prompt_tokens":0,"output_tokens":11198,"reasoning_tokens":10896,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":69.32,"cost_usd":0.022479,"error":null},{"passed":true,"prompt_tokens":1906,"cached_prompt_tokens":832,"output_tokens":14255,"reasoning_tokens":14114,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":95.49,"cost_usd":0.028599,"error":null}]}}},{"task":"exercism/perfect-numbers","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00028,"latency_s":1.6,"per_attempt":[{"passed":true,"prompt_tokens":844,"cached_prompt_tokens":0,"output_tokens":217,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.6,"cost_usd":0.00028,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001573,"latency_s":9.94,"per_attempt":[{"passed":true,"prompt_tokens":842,"cached_prompt_tokens":0,"output_tokens":1654,"reasoning_tokens":1511,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.94,"cost_usd":0.001573,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000493,"latency_s":3.13,"per_attempt":[{"passed":true,"prompt_tokens":844,"cached_prompt_tokens":0,"output_tokens":221,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.13,"cost_usd":0.000493,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002161,"latency_s":6.73,"per_attempt":[{"passed":true,"prompt_tokens":880,"cached_prompt_tokens":0,"output_tokens":1054,"reasoning_tokens":845,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.73,"cost_usd":0.002161,"error":null}]}}},{"task":"exercism/phone-number","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001131,"latency_s":6.5,"per_attempt":[{"passed":false,"prompt_tokens":1094,"cached_prompt_tokens":0,"output_tokens":470,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.09,"cost_usd":0.000532,"error":null},{"passed":false,"prompt_tokens":1750,"cached_prompt_tokens":1088,"output_tokens":495,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.41,"cost_usd":0.000599,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.010433,"latency_s":67.94,"per_attempt":[{"passed":false,"prompt_tokens":1092,"cached_prompt_tokens":1088,"output_tokens":7080,"reasoning_tokens":6795,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":42.59,"cost_usd":0.006459,"error":null},{"passed":false,"prompt_tokens":1561,"cached_prompt_tokens":1088,"output_tokens":4266,"reasoning_tokens":3920,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":25.35,"cost_usd":0.003974,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002226,"latency_s":13.33,"per_attempt":[{"passed":false,"prompt_tokens":1094,"cached_prompt_tokens":0,"output_tokens":527,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.81,"cost_usd":0.00112,"error":null},{"passed":false,"prompt_tokens":1830,"cached_prompt_tokens":832,"output_tokens":511,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.52,"cost_usd":0.001107,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.100025,"latency_s":319.98,"per_attempt":[{"passed":false,"prompt_tokens":1130,"cached_prompt_tokens":0,"output_tokens":30510,"reasoning_tokens":30069,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":186.46,"cost_usd":0.061088,"error":null},{"passed":true,"prompt_tokens":1776,"cached_prompt_tokens":832,"output_tokens":19428,"reasoning_tokens":18950,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":133.52,"cost_usd":0.038938,"error":null}]}}},{"task":"exercism/pig-latin","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000534,"latency_s":3.43,"per_attempt":[{"passed":true,"prompt_tokens":727,"cached_prompt_tokens":0,"output_tokens":513,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.43,"cost_usd":0.000534,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005878,"latency_s":38.84,"per_attempt":[{"passed":true,"prompt_tokens":725,"cached_prompt_tokens":0,"output_tokens":6451,"reasoning_tokens":6207,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":38.84,"cost_usd":0.005878,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000894,"latency_s":5.54,"per_attempt":[{"passed":true,"prompt_tokens":727,"cached_prompt_tokens":0,"output_tokens":425,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.54,"cost_usd":0.000894,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.03602,"latency_s":111.47,"per_attempt":[{"passed":true,"prompt_tokens":763,"cached_prompt_tokens":0,"output_tokens":17987,"reasoning_tokens":17729,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":111.47,"cost_usd":0.03602,"error":null}]}}},{"task":"exercism/point-mutations","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000489,"latency_s":2.63,"per_attempt":[{"passed":false,"prompt_tokens":394,"cached_prompt_tokens":0,"output_tokens":192,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.28,"cost_usd":0.000212,"error":null},{"passed":true,"prompt_tokens":1070,"cached_prompt_tokens":0,"output_tokens":189,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.35,"cost_usd":0.000277,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001046,"latency_s":7.23,"per_attempt":[{"passed":true,"prompt_tokens":392,"cached_prompt_tokens":0,"output_tokens":1119,"reasoning_tokens":1060,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.23,"cost_usd":0.001046,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001129,"latency_s":6.75,"per_attempt":[{"passed":false,"prompt_tokens":394,"cached_prompt_tokens":0,"output_tokens":194,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.6,"cost_usd":0.000412,"error":null},{"passed":false,"prompt_tokens":1097,"cached_prompt_tokens":0,"output_tokens":326,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.15,"cost_usd":0.000718,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.010411,"latency_s":30.22,"per_attempt":[{"passed":false,"prompt_tokens":430,"cached_prompt_tokens":0,"output_tokens":780,"reasoning_tokens":688,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.79,"cost_usd":0.001586,"error":null},{"passed":true,"prompt_tokens":921,"cached_prompt_tokens":0,"output_tokens":4385,"reasoning_tokens":4326,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":25.43,"cost_usd":0.008825,"error":null}]}}},{"task":"exercism/poker","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.003376,"latency_s":22.26,"per_attempt":[{"passed":false,"prompt_tokens":131,"cached_prompt_tokens":0,"output_tokens":2105,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.29,"cost_usd":0.001908,"error":null},{"passed":false,"prompt_tokens":2558,"cached_prompt_tokens":0,"output_tokens":1347,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.97,"cost_usd":0.001468,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.022196,"latency_s":146.57,"per_attempt":[{"passed":false,"prompt_tokens":129,"cached_prompt_tokens":0,"output_tokens":11010,"reasoning_tokens":10329,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":65.82,"cost_usd":0.009922,"error":null},{"passed":false,"prompt_tokens":1132,"cached_prompt_tokens":0,"output_tokens":13512,"reasoning_tokens":12815,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":80.75,"cost_usd":0.012274,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001476,"latency_s":9.29,"per_attempt":[{"passed":true,"prompt_tokens":131,"cached_prompt_tokens":0,"output_tokens":734,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.29,"cost_usd":0.001476,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.037822,"latency_s":114.43,"per_attempt":[{"passed":true,"prompt_tokens":167,"cached_prompt_tokens":0,"output_tokens":18906,"reasoning_tokens":18067,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":114.43,"cost_usd":0.037822,"error":null}]}}},{"task":"exercism/pov","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.004686,"latency_s":29.71,"per_attempt":[{"passed":false,"prompt_tokens":888,"cached_prompt_tokens":0,"output_tokens":3062,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":19.43,"cost_usd":0.002845,"error":null},{"passed":false,"prompt_tokens":4245,"cached_prompt_tokens":0,"output_tokens":1574,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.28,"cost_usd":0.001841,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.017532,"latency_s":114.17,"per_attempt":[{"passed":false,"prompt_tokens":886,"cached_prompt_tokens":0,"output_tokens":11237,"reasoning_tokens":10698,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":67.12,"cost_usd":0.010202,"error":null},{"passed":false,"prompt_tokens":1898,"cached_prompt_tokens":0,"output_tokens":7934,"reasoning_tokens":7414,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":47.05,"cost_usd":0.00733,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.005967,"latency_s":36.87,"per_attempt":[{"passed":false,"prompt_tokens":888,"cached_prompt_tokens":0,"output_tokens":1484,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":18.92,"cost_usd":0.003021,"error":null},{"passed":false,"prompt_tokens":3147,"cached_prompt_tokens":832,"output_tokens":1391,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":17.95,"cost_usd":0.002946,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.049733,"latency_s":156.51,"per_attempt":[{"passed":false,"prompt_tokens":924,"cached_prompt_tokens":0,"output_tokens":22222,"reasoning_tokens":21575,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":141.03,"cost_usd":0.044499,"error":null},{"passed":false,"prompt_tokens":1874,"cached_prompt_tokens":832,"output_tokens":2573,"reasoning_tokens":1931,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":15.48,"cost_usd":0.005233,"error":null}]}}},{"task":"exercism/prime-factors","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000171,"latency_s":1.0,"per_attempt":[{"passed":true,"prompt_tokens":356,"cached_prompt_tokens":0,"output_tokens":151,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.0,"cost_usd":0.000171,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001618,"latency_s":10.55,"per_attempt":[{"passed":true,"prompt_tokens":354,"cached_prompt_tokens":0,"output_tokens":1758,"reasoning_tokens":1681,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.55,"cost_usd":0.001618,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000347,"latency_s":2.14,"per_attempt":[{"passed":true,"prompt_tokens":356,"cached_prompt_tokens":0,"output_tokens":163,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.14,"cost_usd":0.000347,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000884,"latency_s":2.87,"per_attempt":[{"passed":true,"prompt_tokens":392,"cached_prompt_tokens":0,"output_tokens":430,"reasoning_tokens":348,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.87,"cost_usd":0.000884,"error":null}]}}},{"task":"exercism/protein-translation","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000427,"latency_s":2.86,"per_attempt":[{"passed":true,"prompt_tokens":509,"cached_prompt_tokens":0,"output_tokens":418,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.86,"cost_usd":0.000427,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003302,"latency_s":21.51,"per_attempt":[{"passed":true,"prompt_tokens":507,"cached_prompt_tokens":0,"output_tokens":3612,"reasoning_tokens":3361,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":21.51,"cost_usd":0.003302,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000657,"latency_s":4.13,"per_attempt":[{"passed":true,"prompt_tokens":509,"cached_prompt_tokens":0,"output_tokens":313,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.13,"cost_usd":0.000657,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005481,"latency_s":17.49,"per_attempt":[{"passed":true,"prompt_tokens":545,"cached_prompt_tokens":0,"output_tokens":2724,"reasoning_tokens":2392,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":17.49,"cost_usd":0.005481,"error":null}]}}},{"task":"exercism/proverb","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000501,"latency_s":2.97,"per_attempt":[{"passed":false,"prompt_tokens":408,"cached_prompt_tokens":0,"output_tokens":209,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.46,"cost_usd":0.000229,"error":null},{"passed":true,"prompt_tokens":851,"cached_prompt_tokens":0,"output_tokens":208,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.51,"cost_usd":0.000272,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003707,"latency_s":24.25,"per_attempt":[{"passed":true,"prompt_tokens":406,"cached_prompt_tokens":0,"output_tokens":4074,"reasoning_tokens":3972,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":24.25,"cost_usd":0.003707,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000466,"latency_s":2.89,"per_attempt":[{"passed":true,"prompt_tokens":408,"cached_prompt_tokens":0,"output_tokens":221,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.89,"cost_usd":0.000466,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006633,"latency_s":19.77,"per_attempt":[{"passed":true,"prompt_tokens":444,"cached_prompt_tokens":0,"output_tokens":3303,"reasoning_tokens":3138,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":19.77,"cost_usd":0.006633,"error":null}]}}},{"task":"exercism/pythagorean-triplet","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00069,"latency_s":4.49,"per_attempt":[{"passed":true,"prompt_tokens":712,"cached_prompt_tokens":0,"output_tokens":687,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.49,"cost_usd":0.00069,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004496,"latency_s":29.32,"per_attempt":[{"passed":true,"prompt_tokens":710,"cached_prompt_tokens":0,"output_tokens":4917,"reasoning_tokens":4803,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":29.32,"cost_usd":0.004496,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001215,"latency_s":7.62,"per_attempt":[{"passed":true,"prompt_tokens":712,"cached_prompt_tokens":0,"output_tokens":586,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.62,"cost_usd":0.001215,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.056813,"latency_s":217.0,"per_attempt":[{"passed":true,"prompt_tokens":748,"cached_prompt_tokens":0,"output_tokens":28384,"reasoning_tokens":27762,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":217.0,"cost_usd":0.056813,"error":null}]}}},{"task":"exercism/queen-attack","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00034,"latency_s":2.07,"per_attempt":[{"passed":true,"prompt_tokens":775,"cached_prompt_tokens":0,"output_tokens":292,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.07,"cost_usd":0.00034,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002672,"latency_s":17.32,"per_attempt":[{"passed":true,"prompt_tokens":773,"cached_prompt_tokens":0,"output_tokens":2883,"reasoning_tokens":2684,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":17.32,"cost_usd":0.002672,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00045,"latency_s":2.87,"per_attempt":[{"passed":true,"prompt_tokens":775,"cached_prompt_tokens":0,"output_tokens":202,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.87,"cost_usd":0.00045,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002071,"latency_s":6.21,"per_attempt":[{"passed":true,"prompt_tokens":811,"cached_prompt_tokens":0,"output_tokens":1011,"reasoning_tokens":812,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.21,"cost_usd":0.002071,"error":null}]}}},{"task":"exercism/rail-fence-cipher","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000499,"latency_s":3.07,"per_attempt":[{"passed":true,"prompt_tokens":742,"cached_prompt_tokens":0,"output_tokens":472,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.07,"cost_usd":0.000499,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006145,"latency_s":40.48,"per_attempt":[{"passed":true,"prompt_tokens":740,"cached_prompt_tokens":0,"output_tokens":6746,"reasoning_tokens":6416,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":40.48,"cost_usd":0.006145,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001035,"latency_s":6.52,"per_attempt":[{"passed":true,"prompt_tokens":742,"cached_prompt_tokens":0,"output_tokens":495,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.52,"cost_usd":0.001035,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.051817,"latency_s":177.4,"per_attempt":[{"passed":true,"prompt_tokens":778,"cached_prompt_tokens":0,"output_tokens":25885,"reasoning_tokens":25510,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":177.4,"cost_usd":0.051817,"error":null}]}}},{"task":"exercism/raindrops","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000148,"latency_s":0.79,"per_attempt":[{"passed":true,"prompt_tokens":683,"cached_prompt_tokens":0,"output_tokens":89,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.79,"cost_usd":0.000148,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000522,"latency_s":3.16,"per_attempt":[{"passed":true,"prompt_tokens":681,"cached_prompt_tokens":0,"output_tokens":504,"reasoning_tokens":420,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.16,"cost_usd":0.000522,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000217,"latency_s":1.44,"per_attempt":[{"passed":true,"prompt_tokens":683,"cached_prompt_tokens":0,"output_tokens":88,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.44,"cost_usd":0.000217,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000601,"latency_s":1.86,"per_attempt":[{"passed":true,"prompt_tokens":719,"cached_prompt_tokens":0,"output_tokens":279,"reasoning_tokens":191,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.86,"cost_usd":0.000601,"error":null}]}}},{"task":"exercism/rational-numbers","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000744,"latency_s":4.56,"per_attempt":[{"passed":true,"prompt_tokens":1105,"cached_prompt_tokens":0,"output_tokens":704,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.56,"cost_usd":0.000744,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006721,"latency_s":44.13,"per_attempt":[{"passed":true,"prompt_tokens":1103,"cached_prompt_tokens":1088,"output_tokens":7369,"reasoning_tokens":6951,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":44.13,"cost_usd":0.006721,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001474,"latency_s":9.1,"per_attempt":[{"passed":true,"prompt_tokens":1105,"cached_prompt_tokens":0,"output_tokens":704,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.1,"cost_usd":0.001474,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.131187,"latency_s":416.11,"per_attempt":[{"passed":false,"prompt_tokens":1141,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":215.86,"cost_usd":0.065604,"error":null},{"passed":false,"prompt_tokens":1186,"cached_prompt_tokens":832,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":200.25,"cost_usd":0.065582,"error":null}]}}},{"task":"exercism/react","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001017,"latency_s":6.11,"per_attempt":[{"passed":false,"prompt_tokens":277,"cached_prompt_tokens":0,"output_tokens":464,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.87,"cost_usd":0.000445,"error":null},{"passed":false,"prompt_tokens":1345,"cached_prompt_tokens":0,"output_tokens":486,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.24,"cost_usd":0.000572,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.013356,"latency_s":87.95,"per_attempt":[{"passed":false,"prompt_tokens":275,"cached_prompt_tokens":0,"output_tokens":9655,"reasoning_tokens":9182,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":57.48,"cost_usd":0.008717,"error":null},{"passed":false,"prompt_tokens":1233,"cached_prompt_tokens":0,"output_tokens":5017,"reasoning_tokens":4501,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":30.47,"cost_usd":0.004639,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001453,"latency_s":8.92,"per_attempt":[{"passed":false,"prompt_tokens":277,"cached_prompt_tokens":0,"output_tokens":344,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.4,"cost_usd":0.000705,"error":null},{"passed":false,"prompt_tokens":1007,"cached_prompt_tokens":0,"output_tokens":344,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.52,"cost_usd":0.000748,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.131112,"latency_s":456.56,"per_attempt":[{"passed":false,"prompt_tokens":313,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":233.47,"cost_usd":0.065555,"error":null},{"passed":false,"prompt_tokens":358,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":223.09,"cost_usd":0.065557,"error":null}]}}},{"task":"exercism/rectangles","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.00159,"latency_s":10.16,"per_attempt":[{"passed":false,"prompt_tokens":265,"cached_prompt_tokens":0,"output_tokens":799,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.0,"cost_usd":0.000746,"error":null},{"passed":false,"prompt_tokens":1253,"cached_prompt_tokens":0,"output_tokens":799,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.16,"cost_usd":0.000844,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005339,"latency_s":35.39,"per_attempt":[{"passed":true,"prompt_tokens":263,"cached_prompt_tokens":0,"output_tokens":5903,"reasoning_tokens":5648,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":35.39,"cost_usd":0.005339,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001036,"latency_s":6.65,"per_attempt":[{"passed":true,"prompt_tokens":265,"cached_prompt_tokens":0,"output_tokens":510,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.65,"cost_usd":0.001036,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.032284,"latency_s":240.55,"per_attempt":[{"passed":true,"prompt_tokens":301,"cached_prompt_tokens":0,"output_tokens":16133,"reasoning_tokens":15889,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":240.55,"cost_usd":0.032284,"error":null}]}}},{"task":"exercism/relative-distance","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000594,"latency_s":3.19,"per_attempt":[{"passed":true,"prompt_tokens":1543,"cached_prompt_tokens":0,"output_tokens":489,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.19,"cost_usd":0.000594,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006039,"latency_s":39.35,"per_attempt":[{"passed":true,"prompt_tokens":1541,"cached_prompt_tokens":1088,"output_tokens":6563,"reasoning_tokens":6207,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":39.35,"cost_usd":0.006039,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001986,"latency_s":11.37,"per_attempt":[{"passed":false,"prompt_tokens":1543,"cached_prompt_tokens":0,"output_tokens":438,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.65,"cost_usd":0.000969,"error":null},{"passed":false,"prompt_tokens":2766,"cached_prompt_tokens":832,"output_tokens":438,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.72,"cost_usd":0.001017,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.063619,"latency_s":209.16,"per_attempt":[{"passed":true,"prompt_tokens":1579,"cached_prompt_tokens":0,"output_tokens":31762,"reasoning_tokens":31290,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":209.16,"cost_usd":0.063619,"error":null}]}}},{"task":"exercism/resistor-color","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000201,"latency_s":1.11,"per_attempt":[{"passed":true,"prompt_tokens":427,"cached_prompt_tokens":0,"output_tokens":176,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.11,"cost_usd":0.000201,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001851,"latency_s":12.49,"per_attempt":[{"passed":true,"prompt_tokens":425,"cached_prompt_tokens":0,"output_tokens":2009,"reasoning_tokens":1923,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.49,"cost_usd":0.001851,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000418,"latency_s":2.48,"per_attempt":[{"passed":true,"prompt_tokens":427,"cached_prompt_tokens":0,"output_tokens":196,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.48,"cost_usd":0.000418,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005058,"latency_s":18.01,"per_attempt":[{"passed":true,"prompt_tokens":463,"cached_prompt_tokens":0,"output_tokens":2515,"reasoning_tokens":2381,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":18.01,"cost_usd":0.005058,"error":null}]}}},{"task":"exercism/resistor-color-duo","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000252,"latency_s":1.6,"per_attempt":[{"passed":true,"prompt_tokens":398,"cached_prompt_tokens":0,"output_tokens":236,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.6,"cost_usd":0.000252,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001633,"latency_s":10.49,"per_attempt":[{"passed":true,"prompt_tokens":396,"cached_prompt_tokens":0,"output_tokens":1770,"reasoning_tokens":1647,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.49,"cost_usd":0.001633,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000364,"latency_s":2.36,"per_attempt":[{"passed":true,"prompt_tokens":398,"cached_prompt_tokens":0,"output_tokens":170,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.36,"cost_usd":0.000364,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00302,"latency_s":11.08,"per_attempt":[{"passed":true,"prompt_tokens":434,"cached_prompt_tokens":0,"output_tokens":1497,"reasoning_tokens":1338,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.08,"cost_usd":0.00302,"error":null}]}}},{"task":"exercism/resistor-color-expert","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000828,"latency_s":5.12,"per_attempt":[{"passed":true,"prompt_tokens":1124,"cached_prompt_tokens":0,"output_tokens":795,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.12,"cost_usd":0.000828,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.007557,"latency_s":49.68,"per_attempt":[{"passed":true,"prompt_tokens":1122,"cached_prompt_tokens":1088,"output_tokens":8296,"reasoning_tokens":7805,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":49.68,"cost_usd":0.007557,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002003,"latency_s":12.49,"per_attempt":[{"passed":true,"prompt_tokens":1124,"cached_prompt_tokens":0,"output_tokens":968,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.49,"cost_usd":0.002003,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.035908,"latency_s":107.0,"per_attempt":[{"passed":true,"prompt_tokens":1160,"cached_prompt_tokens":0,"output_tokens":17919,"reasoning_tokens":17374,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":107.0,"cost_usd":0.035908,"error":null}]}}},{"task":"exercism/resistor-color-trio","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00054,"latency_s":3.3,"per_attempt":[{"passed":true,"prompt_tokens":704,"cached_prompt_tokens":0,"output_tokens":522,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.3,"cost_usd":0.00054,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004171,"latency_s":27.02,"per_attempt":[{"passed":true,"prompt_tokens":702,"cached_prompt_tokens":0,"output_tokens":4556,"reasoning_tokens":4256,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":27.02,"cost_usd":0.004171,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000794,"latency_s":4.97,"per_attempt":[{"passed":true,"prompt_tokens":704,"cached_prompt_tokens":0,"output_tokens":376,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.97,"cost_usd":0.000794,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.008048,"latency_s":25.18,"per_attempt":[{"passed":true,"prompt_tokens":740,"cached_prompt_tokens":0,"output_tokens":4002,"reasoning_tokens":3767,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":25.18,"cost_usd":0.008048,"error":null}]}}},{"task":"exercism/rest-api","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.002478,"latency_s":15.77,"per_attempt":[{"passed":false,"prompt_tokens":660,"cached_prompt_tokens":0,"output_tokens":1220,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.77,"cost_usd":0.001164,"error":null},{"passed":true,"prompt_tokens":2213,"cached_prompt_tokens":0,"output_tokens":1214,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.0,"cost_usd":0.001314,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.016383,"latency_s":110.03,"per_attempt":[{"passed":false,"prompt_tokens":658,"cached_prompt_tokens":0,"output_tokens":10599,"reasoning_tokens":9912,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":63.48,"cost_usd":0.009605,"error":null},{"passed":true,"prompt_tokens":1663,"cached_prompt_tokens":0,"output_tokens":7346,"reasoning_tokens":6629,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":46.55,"cost_usd":0.006778,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.005246,"latency_s":32.37,"per_attempt":[{"passed":false,"prompt_tokens":660,"cached_prompt_tokens":0,"output_tokens":1477,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":18.98,"cost_usd":0.002994,"error":null},{"passed":false,"prompt_tokens":2471,"cached_prompt_tokens":0,"output_tokens":1052,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.39,"cost_usd":0.002252,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.114956,"latency_s":377.43,"per_attempt":[{"passed":false,"prompt_tokens":696,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":215.78,"cost_usd":0.065578,"error":null},{"passed":false,"prompt_tokens":741,"cached_prompt_tokens":0,"output_tokens":24667,"reasoning_tokens":23641,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":161.65,"cost_usd":0.049378,"error":null}]}}},{"task":"exercism/reverse-string","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":3.5e-05,"latency_s":0.22,"per_attempt":[{"passed":true,"prompt_tokens":201,"cached_prompt_tokens":0,"output_tokens":17,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.22,"cost_usd":3.5e-05,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000242,"latency_s":1.57,"per_attempt":[{"passed":true,"prompt_tokens":199,"cached_prompt_tokens":0,"output_tokens":247,"reasoning_tokens":228,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.57,"cost_usd":0.000242,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":4.6e-05,"latency_s":0.35,"per_attempt":[{"passed":true,"prompt_tokens":201,"cached_prompt_tokens":0,"output_tokens":17,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.35,"cost_usd":4.6e-05,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000196,"latency_s":0.79,"per_attempt":[{"passed":true,"prompt_tokens":237,"cached_prompt_tokens":0,"output_tokens":91,"reasoning_tokens":72,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.79,"cost_usd":0.000196,"error":null}]}}},{"task":"exercism/rna-transcription","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000175,"latency_s":1.02,"per_attempt":[{"passed":true,"prompt_tokens":418,"cached_prompt_tokens":0,"output_tokens":148,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.02,"cost_usd":0.000175,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001374,"latency_s":8.74,"per_attempt":[{"passed":true,"prompt_tokens":416,"cached_prompt_tokens":0,"output_tokens":1480,"reasoning_tokens":1417,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.74,"cost_usd":0.001374,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000333,"latency_s":2.1,"per_attempt":[{"passed":true,"prompt_tokens":418,"cached_prompt_tokens":0,"output_tokens":154,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.1,"cost_usd":0.000333,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002885,"latency_s":9.06,"per_attempt":[{"passed":true,"prompt_tokens":454,"cached_prompt_tokens":0,"output_tokens":1429,"reasoning_tokens":1290,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.06,"cost_usd":0.002885,"error":null}]}}},{"task":"exercism/robot-name","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.00051,"latency_s":3.2,"per_attempt":[{"passed":false,"prompt_tokens":226,"cached_prompt_tokens":0,"output_tokens":231,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.61,"cost_usd":0.00023,"error":null},{"passed":false,"prompt_tokens":719,"cached_prompt_tokens":0,"output_tokens":231,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.59,"cost_usd":0.00028,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006666,"latency_s":44.29,"per_attempt":[{"passed":true,"prompt_tokens":224,"cached_prompt_tokens":0,"output_tokens":7382,"reasoning_tokens":7256,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":44.29,"cost_usd":0.006666,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000748,"latency_s":4.73,"per_attempt":[{"passed":false,"prompt_tokens":226,"cached_prompt_tokens":0,"output_tokens":149,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.18,"cost_usd":0.000312,"error":null},{"passed":true,"prompt_tokens":641,"cached_prompt_tokens":0,"output_tokens":199,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.55,"cost_usd":0.000436,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.019424,"latency_s":66.97,"per_attempt":[{"passed":true,"prompt_tokens":262,"cached_prompt_tokens":0,"output_tokens":9704,"reasoning_tokens":9535,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":66.97,"cost_usd":0.019424,"error":null}]}}},{"task":"exercism/robot-simulator","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.00067,"latency_s":4.1,"per_attempt":[{"passed":false,"prompt_tokens":334,"cached_prompt_tokens":0,"output_tokens":303,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.05,"cost_usd":0.000306,"error":null},{"passed":true,"prompt_tokens":913,"cached_prompt_tokens":0,"output_tokens":303,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.05,"cost_usd":0.000364,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004134,"latency_s":27.5,"per_attempt":[{"passed":true,"prompt_tokens":332,"cached_prompt_tokens":0,"output_tokens":4556,"reasoning_tokens":4300,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":27.5,"cost_usd":0.004134,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.001527,"latency_s":9.39,"per_attempt":[{"passed":false,"prompt_tokens":334,"cached_prompt_tokens":0,"output_tokens":345,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.48,"cost_usd":0.00071,"error":null},{"passed":true,"prompt_tokens":890,"cached_prompt_tokens":0,"output_tokens":382,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.91,"cost_usd":0.000817,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.029907,"latency_s":91.57,"per_attempt":[{"passed":false,"prompt_tokens":370,"cached_prompt_tokens":0,"output_tokens":9605,"reasoning_tokens":9210,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":59.18,"cost_usd":0.019232,"error":null},{"passed":true,"prompt_tokens":974,"cached_prompt_tokens":0,"output_tokens":5308,"reasoning_tokens":4812,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":32.39,"cost_usd":0.010674,"error":null}]}}},{"task":"exercism/roman-numerals","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000332,"latency_s":2.03,"per_attempt":[{"passed":true,"prompt_tokens":770,"cached_prompt_tokens":0,"output_tokens":283,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.03,"cost_usd":0.000332,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002212,"latency_s":14.57,"per_attempt":[{"passed":true,"prompt_tokens":768,"cached_prompt_tokens":0,"output_tokens":2372,"reasoning_tokens":2192,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.57,"cost_usd":0.002212,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00063,"latency_s":3.71,"per_attempt":[{"passed":true,"prompt_tokens":770,"cached_prompt_tokens":0,"output_tokens":292,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.71,"cost_usd":0.00063,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003372,"latency_s":11.16,"per_attempt":[{"passed":true,"prompt_tokens":806,"cached_prompt_tokens":0,"output_tokens":1662,"reasoning_tokens":1449,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.16,"cost_usd":0.003372,"error":null}]}}},{"task":"exercism/rotational-cipher","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000218,"latency_s":1.31,"per_attempt":[{"passed":true,"prompt_tokens":394,"cached_prompt_tokens":0,"output_tokens":199,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.31,"cost_usd":0.000218,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00208,"latency_s":14.08,"per_attempt":[{"passed":true,"prompt_tokens":392,"cached_prompt_tokens":0,"output_tokens":2268,"reasoning_tokens":2149,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.08,"cost_usd":0.00208,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000208,"latency_s":1.27,"per_attempt":[{"passed":true,"prompt_tokens":394,"cached_prompt_tokens":0,"output_tokens":92,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.27,"cost_usd":0.000208,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000886,"latency_s":2.74,"per_attempt":[{"passed":true,"prompt_tokens":430,"cached_prompt_tokens":0,"output_tokens":430,"reasoning_tokens":304,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.74,"cost_usd":0.000886,"error":null}]}}},{"task":"exercism/run-length-encoding","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000314,"latency_s":2.2,"per_attempt":[{"passed":true,"prompt_tokens":315,"cached_prompt_tokens":0,"output_tokens":314,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.2,"cost_usd":0.000314,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004879,"latency_s":32.34,"per_attempt":[{"passed":true,"prompt_tokens":313,"cached_prompt_tokens":0,"output_tokens":5386,"reasoning_tokens":5184,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":32.34,"cost_usd":0.004879,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000619,"latency_s":3.89,"per_attempt":[{"passed":true,"prompt_tokens":315,"cached_prompt_tokens":0,"output_tokens":300,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.89,"cost_usd":0.000619,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.009447,"latency_s":32.17,"per_attempt":[{"passed":true,"prompt_tokens":351,"cached_prompt_tokens":0,"output_tokens":4713,"reasoning_tokens":4565,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":32.17,"cost_usd":0.009447,"error":null}]}}},{"task":"exercism/saddle-points","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000402,"latency_s":2.59,"per_attempt":[{"passed":true,"prompt_tokens":715,"cached_prompt_tokens":0,"output_tokens":367,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.59,"cost_usd":0.000402,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004025,"latency_s":26.45,"per_attempt":[{"passed":true,"prompt_tokens":713,"cached_prompt_tokens":0,"output_tokens":4393,"reasoning_tokens":4235,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":26.45,"cost_usd":0.004025,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000613,"latency_s":3.8,"per_attempt":[{"passed":true,"prompt_tokens":715,"cached_prompt_tokens":0,"output_tokens":285,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.8,"cost_usd":0.000613,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.010857,"latency_s":34.45,"per_attempt":[{"passed":false,"prompt_tokens":751,"cached_prompt_tokens":0,"output_tokens":4379,"reasoning_tokens":4193,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":28.24,"cost_usd":0.008803,"error":null},{"passed":true,"prompt_tokens":1461,"cached_prompt_tokens":0,"output_tokens":983,"reasoning_tokens":812,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.21,"cost_usd":0.002054,"error":null}]}}},{"task":"exercism/satellite","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000886,"latency_s":5.09,"per_attempt":[{"passed":false,"prompt_tokens":636,"cached_prompt_tokens":0,"output_tokens":371,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.59,"cost_usd":0.000398,"error":null},{"passed":true,"prompt_tokens":1547,"cached_prompt_tokens":0,"output_tokens":371,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.5,"cost_usd":0.000489,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.006928,"latency_s":44.62,"per_attempt":[{"passed":false,"prompt_tokens":634,"cached_prompt_tokens":0,"output_tokens":4893,"reasoning_tokens":4698,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":29.29,"cost_usd":0.004467,"error":null},{"passed":true,"prompt_tokens":1362,"cached_prompt_tokens":0,"output_tokens":2583,"reasoning_tokens":2388,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":15.33,"cost_usd":0.002461,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.002163,"latency_s":13.39,"per_attempt":[{"passed":false,"prompt_tokens":636,"cached_prompt_tokens":0,"output_tokens":500,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.47,"cost_usd":0.001038,"error":null},{"passed":true,"prompt_tokens":1477,"cached_prompt_tokens":0,"output_tokens":518,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.92,"cost_usd":0.001125,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.068478,"latency_s":222.31,"per_attempt":[{"passed":false,"prompt_tokens":672,"cached_prompt_tokens":0,"output_tokens":26926,"reasoning_tokens":26576,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":174.72,"cost_usd":0.053892,"error":null},{"passed":true,"prompt_tokens":1359,"cached_prompt_tokens":0,"output_tokens":7252,"reasoning_tokens":6872,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":47.59,"cost_usd":0.014586,"error":null}]}}},{"task":"exercism/say","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000665,"latency_s":4.55,"per_attempt":[{"passed":true,"prompt_tokens":553,"cached_prompt_tokens":0,"output_tokens":677,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.55,"cost_usd":0.000665,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006737,"latency_s":44.3,"per_attempt":[{"passed":true,"prompt_tokens":551,"cached_prompt_tokens":0,"output_tokens":7424,"reasoning_tokens":6985,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":44.3,"cost_usd":0.006737,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001345,"latency_s":8.38,"per_attempt":[{"passed":true,"prompt_tokens":553,"cached_prompt_tokens":0,"output_tokens":656,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.38,"cost_usd":0.001345,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.009567,"latency_s":28.16,"per_attempt":[{"passed":true,"prompt_tokens":589,"cached_prompt_tokens":0,"output_tokens":4766,"reasoning_tokens":4228,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":28.16,"cost_usd":0.009567,"error":null}]}}},{"task":"exercism/scale-generator","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002134,"latency_s":12.85,"per_attempt":[{"passed":false,"prompt_tokens":1071,"cached_prompt_tokens":0,"output_tokens":1221,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.99,"cost_usd":0.001206,"error":null},{"passed":false,"prompt_tokens":2562,"cached_prompt_tokens":0,"output_tokens":746,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.86,"cost_usd":0.000928,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.015344,"latency_s":103.06,"per_attempt":[{"passed":false,"prompt_tokens":1069,"cached_prompt_tokens":0,"output_tokens":6742,"reasoning_tokens":6465,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":40.73,"cost_usd":0.006175,"error":null},{"passed":true,"prompt_tokens":1773,"cached_prompt_tokens":1088,"output_tokens":10015,"reasoning_tokens":9681,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":62.33,"cost_usd":0.009169,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.005258,"latency_s":58.3,"per_attempt":[{"passed":false,"prompt_tokens":1071,"cached_prompt_tokens":832,"output_tokens":1500,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":44.76,"cost_usd":0.003039,"error":null},{"passed":false,"prompt_tokens":2923,"cached_prompt_tokens":832,"output_tokens":1034,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.54,"cost_usd":0.002218,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.130609,"latency_s":414.18,"per_attempt":[{"passed":false,"prompt_tokens":1107,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32767,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":210.33,"cost_usd":0.065602,"error":null},{"passed":true,"prompt_tokens":1152,"cached_prompt_tokens":832,"output_tokens":32481,"reasoning_tokens":31788,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":203.85,"cost_usd":0.065006,"error":null}]}}},{"task":"exercism/scrabble-score","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000391,"latency_s":2.67,"per_attempt":[{"passed":true,"prompt_tokens":351,"cached_prompt_tokens":0,"output_tokens":395,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.67,"cost_usd":0.000391,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002353,"latency_s":15.39,"per_attempt":[{"passed":true,"prompt_tokens":349,"cached_prompt_tokens":0,"output_tokens":2576,"reasoning_tokens":2372,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":15.39,"cost_usd":0.002353,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000527,"latency_s":3.27,"per_attempt":[{"passed":true,"prompt_tokens":351,"cached_prompt_tokens":0,"output_tokens":253,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.27,"cost_usd":0.000527,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.007117,"latency_s":19.01,"per_attempt":[{"passed":true,"prompt_tokens":387,"cached_prompt_tokens":0,"output_tokens":3547,"reasoning_tokens":3320,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":19.01,"cost_usd":0.007117,"error":null}]}}},{"task":"exercism/secret-handshake","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00042,"latency_s":2.66,"per_attempt":[{"passed":true,"prompt_tokens":624,"cached_prompt_tokens":0,"output_tokens":397,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.66,"cost_usd":0.00042,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004434,"latency_s":29.57,"per_attempt":[{"passed":true,"prompt_tokens":622,"cached_prompt_tokens":0,"output_tokens":4858,"reasoning_tokens":4768,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":29.57,"cost_usd":0.004434,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001115,"latency_s":6.96,"per_attempt":[{"passed":true,"prompt_tokens":624,"cached_prompt_tokens":0,"output_tokens":539,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.96,"cost_usd":0.001115,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004796,"latency_s":15.4,"per_attempt":[{"passed":true,"prompt_tokens":660,"cached_prompt_tokens":0,"output_tokens":2378,"reasoning_tokens":2237,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":15.4,"cost_usd":0.004796,"error":null}]}}},{"task":"exercism/series","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00019,"latency_s":1.3,"per_attempt":[{"passed":true,"prompt_tokens":548,"cached_prompt_tokens":0,"output_tokens":150,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.3,"cost_usd":0.00019,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002754,"latency_s":17.69,"per_attempt":[{"passed":true,"prompt_tokens":546,"cached_prompt_tokens":0,"output_tokens":2999,"reasoning_tokens":2885,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":17.69,"cost_usd":0.002754,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000279,"latency_s":1.85,"per_attempt":[{"passed":true,"prompt_tokens":548,"cached_prompt_tokens":0,"output_tokens":123,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.85,"cost_usd":0.000279,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001817,"latency_s":5.71,"per_attempt":[{"passed":true,"prompt_tokens":584,"cached_prompt_tokens":0,"output_tokens":891,"reasoning_tokens":778,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.71,"cost_usd":0.001817,"error":null}]}}},{"task":"exercism/sgf-parsing","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002699,"latency_s":16.28,"per_attempt":[{"passed":false,"prompt_tokens":1510,"cached_prompt_tokens":0,"output_tokens":1225,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.95,"cost_usd":0.001254,"error":null},{"passed":false,"prompt_tokens":3020,"cached_prompt_tokens":1088,"output_tokens":1295,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.33,"cost_usd":0.001446,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.022705,"latency_s":151.57,"per_attempt":[{"passed":false,"prompt_tokens":1508,"cached_prompt_tokens":1088,"output_tokens":14822,"reasoning_tokens":13900,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":91.63,"cost_usd":0.013469,"error":null},{"passed":false,"prompt_tokens":2837,"cached_prompt_tokens":1088,"output_tokens":9971,"reasoning_tokens":9087,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":59.94,"cost_usd":0.009236,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.005134,"latency_s":56.51,"per_attempt":[{"passed":false,"prompt_tokens":1510,"cached_prompt_tokens":832,"output_tokens":1229,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":40.34,"cost_usd":0.002524,"error":null},{"passed":false,"prompt_tokens":2981,"cached_prompt_tokens":832,"output_tokens":1228,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":16.17,"cost_usd":0.00261,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.123031,"latency_s":406.9,"per_attempt":[{"passed":false,"prompt_tokens":1546,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":214.15,"cost_usd":0.065629,"error":null},{"passed":true,"prompt_tokens":1591,"cached_prompt_tokens":832,"output_tokens":28666,"reasoning_tokens":27713,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":192.75,"cost_usd":0.057403,"error":null}]}}},{"task":"exercism/sieve","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000395,"latency_s":2.2,"per_attempt":[{"passed":true,"prompt_tokens":1128,"cached_prompt_tokens":0,"output_tokens":314,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.2,"cost_usd":0.000395,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002245,"latency_s":14.06,"per_attempt":[{"passed":true,"prompt_tokens":1126,"cached_prompt_tokens":1088,"output_tokens":2394,"reasoning_tokens":2271,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.06,"cost_usd":0.002245,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000556,"latency_s":3.28,"per_attempt":[{"passed":true,"prompt_tokens":1128,"cached_prompt_tokens":0,"output_tokens":244,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.28,"cost_usd":0.000556,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001972,"latency_s":6.21,"per_attempt":[{"passed":true,"prompt_tokens":1164,"cached_prompt_tokens":0,"output_tokens":951,"reasoning_tokens":802,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.21,"cost_usd":0.001972,"error":null}]}}},{"task":"exercism/simple-cipher","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000525,"latency_s":3.1,"per_attempt":[{"passed":true,"prompt_tokens":974,"cached_prompt_tokens":0,"output_tokens":475,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.1,"cost_usd":0.000525,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003737,"latency_s":27.24,"per_attempt":[{"passed":true,"prompt_tokens":972,"cached_prompt_tokens":0,"output_tokens":4044,"reasoning_tokens":3848,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":27.24,"cost_usd":0.003737,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00068,"latency_s":4.2,"per_attempt":[{"passed":true,"prompt_tokens":974,"cached_prompt_tokens":0,"output_tokens":311,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.2,"cost_usd":0.00068,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.026989,"latency_s":88.99,"per_attempt":[{"passed":true,"prompt_tokens":1010,"cached_prompt_tokens":0,"output_tokens":13464,"reasoning_tokens":13111,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":88.99,"cost_usd":0.026989,"error":null}]}}},{"task":"exercism/simple-linked-list","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000541,"latency_s":2.3,"per_attempt":[{"passed":true,"prompt_tokens":2190,"cached_prompt_tokens":0,"output_tokens":358,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.3,"cost_usd":0.000541,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.010334,"latency_s":69.14,"per_attempt":[{"passed":false,"prompt_tokens":2188,"cached_prompt_tokens":2176,"output_tokens":6494,"reasoning_tokens":6149,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":41.81,"cost_usd":0.00602,"error":null},{"passed":true,"prompt_tokens":2748,"cached_prompt_tokens":2176,"output_tokens":4537,"reasoning_tokens":4160,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":27.33,"cost_usd":0.004315,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000831,"latency_s":4.78,"per_attempt":[{"passed":true,"prompt_tokens":2190,"cached_prompt_tokens":0,"output_tokens":350,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.78,"cost_usd":0.000831,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.05744,"latency_s":180.65,"per_attempt":[{"passed":true,"prompt_tokens":2226,"cached_prompt_tokens":0,"output_tokens":28653,"reasoning_tokens":28204,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":180.65,"cost_usd":0.05744,"error":null}]}}},{"task":"exercism/space-age","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000489,"latency_s":2.79,"per_attempt":[{"passed":true,"prompt_tokens":1108,"cached_prompt_tokens":0,"output_tokens":420,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.79,"cost_usd":0.000489,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003181,"latency_s":23.59,"per_attempt":[{"passed":true,"prompt_tokens":1106,"cached_prompt_tokens":1088,"output_tokens":3436,"reasoning_tokens":3057,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":23.59,"cost_usd":0.003181,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000648,"latency_s":3.97,"per_attempt":[{"passed":true,"prompt_tokens":1108,"cached_prompt_tokens":0,"output_tokens":291,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.97,"cost_usd":0.000648,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002193,"latency_s":6.16,"per_attempt":[{"passed":true,"prompt_tokens":1144,"cached_prompt_tokens":0,"output_tokens":1062,"reasoning_tokens":663,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.16,"cost_usd":0.002193,"error":null}]}}},{"task":"exercism/spiral-matrix","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000376,"latency_s":2.5,"per_attempt":[{"passed":true,"prompt_tokens":363,"cached_prompt_tokens":0,"output_tokens":378,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.5,"cost_usd":0.000376,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003344,"latency_s":24.87,"per_attempt":[{"passed":true,"prompt_tokens":361,"cached_prompt_tokens":0,"output_tokens":3676,"reasoning_tokens":3444,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":24.87,"cost_usd":0.003344,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000996,"latency_s":6.36,"per_attempt":[{"passed":true,"prompt_tokens":363,"cached_prompt_tokens":0,"output_tokens":487,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.36,"cost_usd":0.000996,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001804,"latency_s":5.45,"per_attempt":[{"passed":true,"prompt_tokens":399,"cached_prompt_tokens":0,"output_tokens":890,"reasoning_tokens":656,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.45,"cost_usd":0.001804,"error":null}]}}},{"task":"exercism/square-root","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000263,"latency_s":1.62,"per_attempt":[{"passed":true,"prompt_tokens":621,"cached_prompt_tokens":0,"output_tokens":223,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.62,"cost_usd":0.000263,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002421,"latency_s":17.87,"per_attempt":[{"passed":true,"prompt_tokens":619,"cached_prompt_tokens":0,"output_tokens":2621,"reasoning_tokens":2533,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":17.87,"cost_usd":0.002421,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000667,"latency_s":4.17,"per_attempt":[{"passed":true,"prompt_tokens":621,"cached_prompt_tokens":0,"output_tokens":315,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.17,"cost_usd":0.000667,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004065,"latency_s":12.04,"per_attempt":[{"passed":true,"prompt_tokens":657,"cached_prompt_tokens":0,"output_tokens":2013,"reasoning_tokens":1925,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.04,"cost_usd":0.004065,"error":null}]}}},{"task":"exercism/state-of-tic-tac-toe","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00077,"latency_s":4.82,"per_attempt":[{"passed":true,"prompt_tokens":1034,"cached_prompt_tokens":0,"output_tokens":741,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.82,"cost_usd":0.00077,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005765,"latency_s":37.7,"per_attempt":[{"passed":true,"prompt_tokens":1032,"cached_prompt_tokens":0,"output_tokens":6291,"reasoning_tokens":5889,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":37.7,"cost_usd":0.005765,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001314,"latency_s":8.0,"per_attempt":[{"passed":true,"prompt_tokens":1034,"cached_prompt_tokens":0,"output_tokens":626,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.0,"cost_usd":0.001314,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.031468,"latency_s":103.26,"per_attempt":[{"passed":true,"prompt_tokens":1070,"cached_prompt_tokens":0,"output_tokens":15702,"reasoning_tokens":15309,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":103.26,"cost_usd":0.031468,"error":null}]}}},{"task":"exercism/strain","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000137,"latency_s":1.03,"per_attempt":[{"passed":true,"prompt_tokens":304,"cached_prompt_tokens":0,"output_tokens":118,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.03,"cost_usd":0.000137,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001469,"latency_s":9.67,"per_attempt":[{"passed":true,"prompt_tokens":302,"cached_prompt_tokens":0,"output_tokens":1599,"reasoning_tokens":1521,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.67,"cost_usd":0.001469,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00017,"latency_s":1.09,"per_attempt":[{"passed":true,"prompt_tokens":304,"cached_prompt_tokens":0,"output_tokens":76,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.09,"cost_usd":0.00017,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001022,"latency_s":3.35,"per_attempt":[{"passed":true,"prompt_tokens":340,"cached_prompt_tokens":0,"output_tokens":501,"reasoning_tokens":423,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.35,"cost_usd":0.001022,"error":null}]}}},{"task":"exercism/sublist","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000452,"latency_s":2.79,"per_attempt":[{"passed":true,"prompt_tokens":812,"cached_prompt_tokens":0,"output_tokens":412,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.79,"cost_usd":0.000452,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003027,"latency_s":19.69,"per_attempt":[{"passed":true,"prompt_tokens":810,"cached_prompt_tokens":0,"output_tokens":3273,"reasoning_tokens":2966,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":19.69,"cost_usd":0.003027,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.001171,"latency_s":6.85,"per_attempt":[{"passed":false,"prompt_tokens":812,"cached_prompt_tokens":0,"output_tokens":211,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.8,"cost_usd":0.000471,"error":null},{"passed":true,"prompt_tokens":1241,"cached_prompt_tokens":0,"output_tokens":313,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.05,"cost_usd":0.0007,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005705,"latency_s":18.16,"per_attempt":[{"passed":true,"prompt_tokens":848,"cached_prompt_tokens":0,"output_tokens":2827,"reasoning_tokens":2652,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":18.16,"cost_usd":0.005705,"error":null}]}}},{"task":"exercism/sum-of-multiples","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000265,"latency_s":1.74,"per_attempt":[{"passed":true,"prompt_tokens":590,"cached_prompt_tokens":0,"output_tokens":229,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.74,"cost_usd":0.000265,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002772,"latency_s":18.07,"per_attempt":[{"passed":true,"prompt_tokens":588,"cached_prompt_tokens":0,"output_tokens":3015,"reasoning_tokens":2951,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":18.07,"cost_usd":0.002772,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000489,"latency_s":2.98,"per_attempt":[{"passed":true,"prompt_tokens":590,"cached_prompt_tokens":0,"output_tokens":227,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.98,"cost_usd":0.000489,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.101906,"latency_s":331.13,"per_attempt":[{"passed":false,"prompt_tokens":626,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":206.47,"cost_usd":0.065574,"error":null},{"passed":true,"prompt_tokens":671,"cached_prompt_tokens":0,"output_tokens":18146,"reasoning_tokens":18085,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":124.66,"cost_usd":0.036332,"error":null}]}}},{"task":"exercism/swift-scheduling","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002651,"latency_s":15.95,"per_attempt":[{"passed":false,"prompt_tokens":721,"cached_prompt_tokens":0,"output_tokens":1261,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.95,"cost_usd":0.001207,"error":null},{"passed":false,"prompt_tokens":2813,"cached_prompt_tokens":0,"output_tokens":1292,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.0,"cost_usd":0.001444,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.014724,"latency_s":96.43,"per_attempt":[{"passed":false,"prompt_tokens":719,"cached_prompt_tokens":0,"output_tokens":8495,"reasoning_tokens":8024,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":51.19,"cost_usd":0.007717,"error":null},{"passed":true,"prompt_tokens":1838,"cached_prompt_tokens":0,"output_tokens":7581,"reasoning_tokens":7069,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":45.24,"cost_usd":0.007007,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.006588,"latency_s":42.41,"per_attempt":[{"passed":false,"prompt_tokens":721,"cached_prompt_tokens":0,"output_tokens":1565,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":20.22,"cost_usd":0.003173,"error":null},{"passed":true,"prompt_tokens":2771,"cached_prompt_tokens":0,"output_tokens":1624,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":22.19,"cost_usd":0.003414,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.068422,"latency_s":211.09,"per_attempt":[{"passed":false,"prompt_tokens":757,"cached_prompt_tokens":0,"output_tokens":12267,"reasoning_tokens":11593,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":77.47,"cost_usd":0.024579,"error":null},{"passed":true,"prompt_tokens":1939,"cached_prompt_tokens":0,"output_tokens":21863,"reasoning_tokens":20645,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":133.62,"cost_usd":0.043842,"error":null}]}}},{"task":"exercism/tournament","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000651,"latency_s":4.12,"per_attempt":[{"passed":true,"prompt_tokens":576,"cached_prompt_tokens":0,"output_tokens":659,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.12,"cost_usd":0.000651,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.007844,"latency_s":52.27,"per_attempt":[{"passed":true,"prompt_tokens":574,"cached_prompt_tokens":0,"output_tokens":8652,"reasoning_tokens":8183,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":52.27,"cost_usd":0.007844,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001007,"latency_s":6.16,"per_attempt":[{"passed":true,"prompt_tokens":576,"cached_prompt_tokens":0,"output_tokens":486,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.16,"cost_usd":0.001007,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005851,"latency_s":18.15,"per_attempt":[{"passed":true,"prompt_tokens":612,"cached_prompt_tokens":0,"output_tokens":2907,"reasoning_tokens":2467,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":18.15,"cost_usd":0.005851,"error":null}]}}},{"task":"exercism/transpose","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00055,"latency_s":3.81,"per_attempt":[{"passed":true,"prompt_tokens":303,"cached_prompt_tokens":0,"output_tokens":577,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.81,"cost_usd":0.00055,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00701,"latency_s":46.88,"per_attempt":[{"passed":true,"prompt_tokens":301,"cached_prompt_tokens":0,"output_tokens":7755,"reasoning_tokens":7566,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":46.88,"cost_usd":0.00701,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.004206,"latency_s":27.88,"per_attempt":[{"passed":false,"prompt_tokens":303,"cached_prompt_tokens":0,"output_tokens":549,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.02,"cost_usd":0.001116,"error":null},{"passed":false,"prompt_tokens":1230,"cached_prompt_tokens":0,"output_tokens":1508,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":20.86,"cost_usd":0.00309,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.025274,"latency_s":80.9,"per_attempt":[{"passed":true,"prompt_tokens":339,"cached_prompt_tokens":0,"output_tokens":12627,"reasoning_tokens":12478,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":80.9,"cost_usd":0.025274,"error":null}]}}},{"task":"exercism/tree-building","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.000986,"latency_s":5.32,"per_attempt":[{"passed":false,"prompt_tokens":953,"cached_prompt_tokens":0,"output_tokens":389,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.64,"cost_usd":0.000445,"error":null},{"passed":false,"prompt_tokens":1674,"cached_prompt_tokens":0,"output_tokens":415,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.68,"cost_usd":0.000541,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.008632,"latency_s":56.15,"per_attempt":[{"passed":false,"prompt_tokens":951,"cached_prompt_tokens":0,"output_tokens":6260,"reasoning_tokens":5959,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":37.94,"cost_usd":0.005729,"error":null},{"passed":false,"prompt_tokens":1582,"cached_prompt_tokens":0,"output_tokens":3050,"reasoning_tokens":2776,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":18.21,"cost_usd":0.002903,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001923,"latency_s":11.89,"per_attempt":[{"passed":false,"prompt_tokens":953,"cached_prompt_tokens":0,"output_tokens":449,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.96,"cost_usd":0.000955,"error":null},{"passed":false,"prompt_tokens":1748,"cached_prompt_tokens":832,"output_tokens":444,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.93,"cost_usd":0.000968,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.112053,"latency_s":326.14,"per_attempt":[{"passed":false,"prompt_tokens":989,"cached_prompt_tokens":0,"output_tokens":27669,"reasoning_tokens":27275,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":151.16,"cost_usd":0.055397,"error":null},{"passed":false,"prompt_tokens":1711,"cached_prompt_tokens":832,"output_tokens":28289,"reasoning_tokens":27838,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":174.98,"cost_usd":0.056656,"error":null}]}}},{"task":"exercism/triangle","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00047,"latency_s":3.22,"per_attempt":[{"passed":true,"prompt_tokens":412,"cached_prompt_tokens":0,"output_tokens":477,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.22,"cost_usd":0.00047,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00331,"latency_s":22.31,"per_attempt":[{"passed":true,"prompt_tokens":410,"cached_prompt_tokens":0,"output_tokens":3632,"reasoning_tokens":3500,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":22.31,"cost_usd":0.00331,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000489,"latency_s":2.99,"per_attempt":[{"passed":true,"prompt_tokens":412,"cached_prompt_tokens":0,"output_tokens":232,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.99,"cost_usd":0.000489,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.007301,"latency_s":22.33,"per_attempt":[{"passed":true,"prompt_tokens":448,"cached_prompt_tokens":0,"output_tokens":3637,"reasoning_tokens":3345,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":22.33,"cost_usd":0.007301,"error":null}]}}},{"task":"exercism/trinary","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00016,"latency_s":0.99,"per_attempt":[{"passed":true,"prompt_tokens":328,"cached_prompt_tokens":0,"output_tokens":141,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.99,"cost_usd":0.00016,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00145,"latency_s":9.68,"per_attempt":[{"passed":true,"prompt_tokens":326,"cached_prompt_tokens":0,"output_tokens":1575,"reasoning_tokens":1511,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.68,"cost_usd":0.00145,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000426,"latency_s":2.9,"per_attempt":[{"passed":true,"prompt_tokens":328,"cached_prompt_tokens":0,"output_tokens":203,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.9,"cost_usd":0.000426,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001962,"latency_s":6.01,"per_attempt":[{"passed":true,"prompt_tokens":364,"cached_prompt_tokens":0,"output_tokens":970,"reasoning_tokens":891,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.01,"cost_usd":0.001962,"error":null}]}}},{"task":"exercism/twelve-days","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000665,"latency_s":4.06,"per_attempt":[{"passed":true,"prompt_tokens":971,"cached_prompt_tokens":0,"output_tokens":631,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.06,"cost_usd":0.000665,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.008643,"latency_s":55.79,"per_attempt":[{"passed":false,"prompt_tokens":969,"cached_prompt_tokens":0,"output_tokens":5130,"reasoning_tokens":4818,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":30.79,"cost_usd":0.004714,"error":null},{"passed":true,"prompt_tokens":1750,"cached_prompt_tokens":0,"output_tokens":4171,"reasoning_tokens":3856,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":25.0,"cost_usd":0.003929,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000852,"latency_s":5.49,"per_attempt":[{"passed":true,"prompt_tokens":971,"cached_prompt_tokens":0,"output_tokens":397,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.49,"cost_usd":0.000852,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003524,"latency_s":9.8,"per_attempt":[{"passed":true,"prompt_tokens":1007,"cached_prompt_tokens":0,"output_tokens":1732,"reasoning_tokens":1374,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.8,"cost_usd":0.003524,"error":null}]}}},{"task":"exercism/two-bucket","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000875,"latency_s":5.56,"per_attempt":[{"passed":true,"prompt_tokens":881,"cached_prompt_tokens":0,"output_tokens":874,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.56,"cost_usd":0.000875,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.0069,"latency_s":45.16,"per_attempt":[{"passed":true,"prompt_tokens":879,"cached_prompt_tokens":0,"output_tokens":7569,"reasoning_tokens":7221,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":45.16,"cost_usd":0.0069,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002509,"latency_s":16.48,"per_attempt":[{"passed":true,"prompt_tokens":881,"cached_prompt_tokens":0,"output_tokens":1228,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":16.48,"cost_usd":0.002509,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.120813,"latency_s":488.66,"per_attempt":[{"passed":false,"prompt_tokens":917,"cached_prompt_tokens":832,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":309.89,"cost_usd":0.065566,"error":null},{"passed":true,"prompt_tokens":962,"cached_prompt_tokens":832,"output_tokens":27607,"reasoning_tokens":27032,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":178.77,"cost_usd":0.055247,"error":null}]}}},{"task":"exercism/two-fer","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":6.1e-05,"latency_s":0.28,"per_attempt":[{"passed":true,"prompt_tokens":371,"cached_prompt_tokens":0,"output_tokens":27,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.28,"cost_usd":6.1e-05,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001738,"latency_s":11.38,"per_attempt":[{"passed":true,"prompt_tokens":369,"cached_prompt_tokens":0,"output_tokens":1890,"reasoning_tokens":1861,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.38,"cost_usd":0.001738,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":7.6e-05,"latency_s":0.51,"per_attempt":[{"passed":true,"prompt_tokens":371,"cached_prompt_tokens":0,"output_tokens":27,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.51,"cost_usd":7.6e-05,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00094,"latency_s":3.04,"per_attempt":[{"passed":true,"prompt_tokens":407,"cached_prompt_tokens":0,"output_tokens":458,"reasoning_tokens":416,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.04,"cost_usd":0.00094,"error":null}]}}},{"task":"exercism/variable-length-quantity","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000447,"latency_s":2.65,"per_attempt":[{"passed":true,"prompt_tokens":775,"cached_prompt_tokens":0,"output_tokens":410,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.65,"cost_usd":0.000447,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005586,"latency_s":36.3,"per_attempt":[{"passed":true,"prompt_tokens":773,"cached_prompt_tokens":0,"output_tokens":6121,"reasoning_tokens":5889,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":36.3,"cost_usd":0.005586,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000739,"latency_s":4.63,"per_attempt":[{"passed":true,"prompt_tokens":775,"cached_prompt_tokens":0,"output_tokens":346,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.63,"cost_usd":0.000739,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.034017,"latency_s":100.53,"per_attempt":[{"passed":true,"prompt_tokens":811,"cached_prompt_tokens":0,"output_tokens":16984,"reasoning_tokens":16740,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":100.53,"cost_usd":0.034017,"error":null}]}}},{"task":"exercism/word-count","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001082,"latency_s":7.11,"per_attempt":[{"passed":true,"prompt_tokens":513,"cached_prompt_tokens":0,"output_tokens":1145,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.11,"cost_usd":0.001082,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003732,"latency_s":24.23,"per_attempt":[{"passed":true,"prompt_tokens":511,"cached_prompt_tokens":0,"output_tokens":4090,"reasoning_tokens":4034,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":24.23,"cost_usd":0.003732,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000823,"latency_s":5.19,"per_attempt":[{"passed":true,"prompt_tokens":513,"cached_prompt_tokens":0,"output_tokens":396,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.19,"cost_usd":0.000823,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.030863,"latency_s":100.7,"per_attempt":[{"passed":true,"prompt_tokens":549,"cached_prompt_tokens":0,"output_tokens":15415,"reasoning_tokens":15339,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":100.7,"cost_usd":0.030863,"error":null}]}}},{"task":"exercism/word-search","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000514,"latency_s":3.34,"per_attempt":[{"passed":true,"prompt_tokens":294,"cached_prompt_tokens":0,"output_tokens":539,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.34,"cost_usd":0.000514,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.011585,"latency_s":76.15,"per_attempt":[{"passed":false,"prompt_tokens":292,"cached_prompt_tokens":0,"output_tokens":5043,"reasoning_tokens":4707,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":30.12,"cost_usd":0.004568,"error":null},{"passed":false,"prompt_tokens":1217,"cached_prompt_tokens":0,"output_tokens":7661,"reasoning_tokens":7312,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":46.03,"cost_usd":0.007017,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002076,"latency_s":14.22,"per_attempt":[{"passed":false,"prompt_tokens":294,"cached_prompt_tokens":0,"output_tokens":494,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.69,"cost_usd":0.001006,"error":null},{"passed":false,"prompt_tokens":1376,"cached_prompt_tokens":0,"output_tokens":494,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.53,"cost_usd":0.001071,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.084755,"latency_s":273.98,"per_attempt":[{"passed":false,"prompt_tokens":330,"cached_prompt_tokens":0,"output_tokens":32049,"reasoning_tokens":31526,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":209.46,"cost_usd":0.064118,"error":null},{"passed":true,"prompt_tokens":1461,"cached_prompt_tokens":0,"output_tokens":10275,"reasoning_tokens":9743,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":64.52,"cost_usd":0.020638,"error":null}]}}},{"task":"exercism/wordy","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001352,"latency_s":7.6,"per_attempt":[{"passed":false,"prompt_tokens":959,"cached_prompt_tokens":0,"output_tokens":607,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.86,"cost_usd":0.000642,"error":null},{"passed":false,"prompt_tokens":1792,"cached_prompt_tokens":0,"output_tokens":589,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.74,"cost_usd":0.000709,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.018252,"latency_s":120.12,"per_attempt":[{"passed":false,"prompt_tokens":957,"cached_prompt_tokens":0,"output_tokens":10245,"reasoning_tokens":9722,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":61.46,"cost_usd":0.009316,"error":null},{"passed":true,"prompt_tokens":1704,"cached_prompt_tokens":0,"output_tokens":9739,"reasoning_tokens":9216,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":58.66,"cost_usd":0.008936,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.002902,"latency_s":18.98,"per_attempt":[{"passed":false,"prompt_tokens":959,"cached_prompt_tokens":0,"output_tokens":633,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.72,"cost_usd":0.001324,"error":null},{"passed":true,"prompt_tokens":2360,"cached_prompt_tokens":832,"output_tokens":731,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.26,"cost_usd":0.001579,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.090145,"latency_s":228.98,"per_attempt":[{"passed":false,"prompt_tokens":995,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":166.49,"cost_usd":0.065596,"error":null},{"passed":false,"prompt_tokens":1040,"cached_prompt_tokens":832,"output_tokens":12256,"reasoning_tokens":12253,"hit_token_cap":false,"no_code_in_reply":true,"latency_s":62.49,"cost_usd":0.024549,"error":null}]}}},{"task":"exercism/yacht","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000724,"latency_s":4.47,"per_attempt":[{"passed":true,"prompt_tokens":864,"cached_prompt_tokens":0,"output_tokens":708,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.47,"cost_usd":0.000724,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005581,"latency_s":36.91,"per_attempt":[{"passed":true,"prompt_tokens":862,"cached_prompt_tokens":0,"output_tokens":6105,"reasoning_tokens":5678,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":36.91,"cost_usd":0.005581,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001086,"latency_s":7.5,"per_attempt":[{"passed":true,"prompt_tokens":864,"cached_prompt_tokens":0,"output_tokens":517,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.5,"cost_usd":0.001086,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004718,"latency_s":16.11,"per_attempt":[{"passed":true,"prompt_tokens":900,"cached_prompt_tokens":0,"output_tokens":2332,"reasoning_tokens":1899,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":16.11,"cost_usd":0.004718,"error":null}]}}},{"task":"exercism/zebra-puzzle","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003932,"latency_s":26.51,"per_attempt":[{"passed":true,"prompt_tokens":595,"cached_prompt_tokens":0,"output_tokens":4303,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":26.51,"cost_usd":0.003932,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004331,"latency_s":28.68,"per_attempt":[{"passed":true,"prompt_tokens":593,"cached_prompt_tokens":0,"output_tokens":4746,"reasoning_tokens":4714,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":28.68,"cost_usd":0.004331,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.00021,"latency_s":1.24,"per_attempt":[{"passed":false,"prompt_tokens":595,"cached_prompt_tokens":0,"output_tokens":32,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.62,"cost_usd":0.0001,"error":null},{"passed":true,"prompt_tokens":838,"cached_prompt_tokens":0,"output_tokens":30,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.62,"cost_usd":0.00011,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004008,"latency_s":13.6,"per_attempt":[{"passed":true,"prompt_tokens":631,"cached_prompt_tokens":0,"output_tokens":1985,"reasoning_tokens":1925,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.6,"cost_usd":0.004008,"error":null}]}}},{"task":"exercism/zipper","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000513,"latency_s":3.32,"per_attempt":[{"passed":true,"prompt_tokens":526,"cached_prompt_tokens":0,"output_tokens":512,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.32,"cost_usd":0.000513,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.015628,"latency_s":103.38,"per_attempt":[{"passed":false,"prompt_tokens":524,"cached_prompt_tokens":0,"output_tokens":8564,"reasoning_tokens":8080,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":51.54,"cost_usd":0.00776,"error":null},{"passed":false,"prompt_tokens":1270,"cached_prompt_tokens":0,"output_tokens":8601,"reasoning_tokens":8097,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":51.84,"cost_usd":0.007868,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.002072,"latency_s":20.9,"per_attempt":[{"passed":false,"prompt_tokens":526,"cached_prompt_tokens":0,"output_tokens":456,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.93,"cost_usd":0.000944,"error":null},{"passed":true,"prompt_tokens":1341,"cached_prompt_tokens":0,"output_tokens":524,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.97,"cost_usd":0.001128,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.097143,"latency_s":317.02,"per_attempt":[{"passed":false,"prompt_tokens":562,"cached_prompt_tokens":0,"output_tokens":32515,"reasoning_tokens":31733,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":214.21,"cost_usd":0.065064,"error":null},{"passed":true,"prompt_tokens":1819,"cached_prompt_tokens":0,"output_tokens":15985,"reasoning_tokens":15114,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":102.81,"cost_usd":0.032079,"error":null}]}}},{"task":"polyglot/affine-cipher","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000794,"latency_s":4.78,"per_attempt":[{"passed":true,"prompt_tokens":1203,"cached_prompt_tokens":1088,"output_tokens":773,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.78,"cost_usd":0.000794,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006016,"latency_s":39.59,"per_attempt":[{"passed":true,"prompt_tokens":1201,"cached_prompt_tokens":1088,"output_tokens":6575,"reasoning_tokens":6220,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":39.59,"cost_usd":0.006016,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000755,"latency_s":5.2,"per_attempt":[{"passed":true,"prompt_tokens":1203,"cached_prompt_tokens":832,"output_tokens":354,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.2,"cost_usd":0.000755,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.031924,"latency_s":103.25,"per_attempt":[{"passed":true,"prompt_tokens":1239,"cached_prompt_tokens":0,"output_tokens":15925,"reasoning_tokens":15530,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":103.25,"cost_usd":0.031924,"error":null}]}}},{"task":"polyglot/beer-song","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000708,"latency_s":3.0,"per_attempt":[{"passed":true,"prompt_tokens":3696,"cached_prompt_tokens":3264,"output_tokens":448,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.0,"cost_usd":0.000708,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.008339,"latency_s":51.1,"per_attempt":[{"passed":false,"prompt_tokens":3694,"cached_prompt_tokens":3264,"output_tokens":4984,"reasoning_tokens":4703,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":30.12,"cost_usd":0.00479,"error":null},{"passed":true,"prompt_tokens":4780,"cached_prompt_tokens":3264,"output_tokens":3485,"reasoning_tokens":3200,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":20.98,"cost_usd":0.003549,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.001813,"latency_s":9.82,"per_attempt":[{"passed":false,"prompt_tokens":3696,"cached_prompt_tokens":3328,"output_tokens":368,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.79,"cost_usd":0.000858,"error":null},{"passed":true,"prompt_tokens":4890,"cached_prompt_tokens":3328,"output_tokens":381,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.03,"cost_usd":0.000956,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.033396,"latency_s":105.73,"per_attempt":[{"passed":false,"prompt_tokens":3732,"cached_prompt_tokens":0,"output_tokens":10612,"reasoning_tokens":10339,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":67.75,"cost_usd":0.021448,"error":null},{"passed":true,"prompt_tokens":4829,"cached_prompt_tokens":3328,"output_tokens":5879,"reasoning_tokens":5544,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":37.98,"cost_usd":0.011948,"error":null}]}}},{"task":"polyglot/book-store","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.00203,"latency_s":12.16,"per_attempt":[{"passed":false,"prompt_tokens":708,"cached_prompt_tokens":0,"output_tokens":1028,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.61,"cost_usd":0.000996,"error":null},{"passed":true,"prompt_tokens":2064,"cached_prompt_tokens":0,"output_tokens":920,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.55,"cost_usd":0.001034,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.012253,"latency_s":80.39,"per_attempt":[{"passed":false,"prompt_tokens":706,"cached_prompt_tokens":0,"output_tokens":6466,"reasoning_tokens":6206,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":38.59,"cost_usd":0.00589,"error":null},{"passed":true,"prompt_tokens":1292,"cached_prompt_tokens":0,"output_tokens":6926,"reasoning_tokens":6683,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":41.8,"cost_usd":0.006363,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.0055,"latency_s":49.39,"per_attempt":[{"passed":false,"prompt_tokens":708,"cached_prompt_tokens":0,"output_tokens":1851,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":38.96,"cost_usd":0.003744,"error":null},{"passed":true,"prompt_tokens":2891,"cached_prompt_tokens":0,"output_tokens":791,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.43,"cost_usd":0.001755,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.131164,"latency_s":419.5,"per_attempt":[{"passed":false,"prompt_tokens":744,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":213.58,"cost_usd":0.065581,"error":null},{"passed":false,"prompt_tokens":789,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":205.92,"cost_usd":0.065583,"error":null}]}}},{"task":"polyglot/bottle-song","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001026,"latency_s":5.99,"per_attempt":[{"passed":false,"prompt_tokens":516,"cached_prompt_tokens":0,"output_tokens":450,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.81,"cost_usd":0.000457,"error":null},{"passed":false,"prompt_tokens":1648,"cached_prompt_tokens":0,"output_tokens":450,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.18,"cost_usd":0.00057,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.004529,"latency_s":29.94,"per_attempt":[{"passed":true,"prompt_tokens":514,"cached_prompt_tokens":0,"output_tokens":4975,"reasoning_tokens":4733,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":29.94,"cost_usd":0.004529,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000953,"latency_s":10.21,"per_attempt":[{"passed":true,"prompt_tokens":516,"cached_prompt_tokens":0,"output_tokens":461,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.21,"cost_usd":0.000953,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.016588,"latency_s":52.06,"per_attempt":[{"passed":false,"prompt_tokens":552,"cached_prompt_tokens":0,"output_tokens":6912,"reasoning_tokens":6642,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":43.8,"cost_usd":0.013857,"error":null},{"passed":true,"prompt_tokens":1512,"cached_prompt_tokens":0,"output_tokens":1320,"reasoning_tokens":1038,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.26,"cost_usd":0.002731,"error":null}]}}},{"task":"polyglot/bowling","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.004314,"latency_s":26.36,"per_attempt":[{"passed":false,"prompt_tokens":961,"cached_prompt_tokens":0,"output_tokens":2291,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.24,"cost_usd":0.002158,"error":null},{"passed":false,"prompt_tokens":3942,"cached_prompt_tokens":0,"output_tokens":1958,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.12,"cost_usd":0.002156,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.025709,"latency_s":172.14,"per_attempt":[{"passed":false,"prompt_tokens":959,"cached_prompt_tokens":0,"output_tokens":14734,"reasoning_tokens":13998,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":88.35,"cost_usd":0.013357,"error":null},{"passed":false,"prompt_tokens":2078,"cached_prompt_tokens":0,"output_tokens":13494,"reasoning_tokens":12719,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":83.79,"cost_usd":0.012352,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002185,"latency_s":13.8,"per_attempt":[{"passed":true,"prompt_tokens":961,"cached_prompt_tokens":832,"output_tokens":1076,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.8,"cost_usd":0.002185,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.090999,"latency_s":267.34,"per_attempt":[{"passed":false,"prompt_tokens":997,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":190.71,"cost_usd":0.065596,"error":null},{"passed":true,"prompt_tokens":1042,"cached_prompt_tokens":832,"output_tokens":12683,"reasoning_tokens":11858,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":76.63,"cost_usd":0.025404,"error":null}]}}},{"task":"polyglot/connect","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002906,"latency_s":18.02,"per_attempt":[{"passed":false,"prompt_tokens":393,"cached_prompt_tokens":0,"output_tokens":1444,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.03,"cost_usd":0.001339,"error":null},{"passed":false,"prompt_tokens":2635,"cached_prompt_tokens":0,"output_tokens":1448,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.99,"cost_usd":0.001567,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.018965,"latency_s":197.79,"per_attempt":[{"passed":false,"prompt_tokens":391,"cached_prompt_tokens":0,"output_tokens":11167,"reasoning_tokens":10720,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":67.15,"cost_usd":0.010089,"error":null},{"passed":false,"prompt_tokens":1083,"cached_prompt_tokens":0,"output_tokens":9741,"reasoning_tokens":9289,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":130.64,"cost_usd":0.008875,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.004535,"latency_s":28.18,"per_attempt":[{"passed":false,"prompt_tokens":393,"cached_prompt_tokens":0,"output_tokens":1183,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":15.17,"cost_usd":0.00239,"error":null},{"passed":false,"prompt_tokens":2316,"cached_prompt_tokens":0,"output_tokens":1003,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.01,"cost_usd":0.002145,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.131126,"latency_s":420.3,"per_attempt":[{"passed":false,"prompt_tokens":429,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":210.06,"cost_usd":0.065562,"error":null},{"passed":false,"prompt_tokens":474,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":210.24,"cost_usd":0.065564,"error":null}]}}},{"task":"polyglot/dominoes","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.002444,"latency_s":15.07,"per_attempt":[{"passed":false,"prompt_tokens":336,"cached_prompt_tokens":0,"output_tokens":1754,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.02,"cost_usd":0.001612,"error":null},{"passed":true,"prompt_tokens":2519,"cached_prompt_tokens":0,"output_tokens":644,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.05,"cost_usd":0.000831,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.005754,"latency_s":38.17,"per_attempt":[{"passed":true,"prompt_tokens":334,"cached_prompt_tokens":0,"output_tokens":6356,"reasoning_tokens":6062,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":38.17,"cost_usd":0.005754,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000838,"latency_s":5.42,"per_attempt":[{"passed":true,"prompt_tokens":336,"cached_prompt_tokens":0,"output_tokens":409,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.42,"cost_usd":0.000838,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.037039,"latency_s":118.35,"per_attempt":[{"passed":false,"prompt_tokens":372,"cached_prompt_tokens":0,"output_tokens":5448,"reasoning_tokens":5234,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":34.23,"cost_usd":0.010918,"error":null},{"passed":false,"prompt_tokens":1015,"cached_prompt_tokens":0,"output_tokens":13030,"reasoning_tokens":12522,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":84.12,"cost_usd":0.026121,"error":null}]}}},{"task":"polyglot/dot-dsl","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001213,"latency_s":6.93,"per_attempt":[{"passed":false,"prompt_tokens":953,"cached_prompt_tokens":0,"output_tokens":526,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.71,"cost_usd":0.000569,"error":null},{"passed":false,"prompt_tokens":1667,"cached_prompt_tokens":0,"output_tokens":531,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.22,"cost_usd":0.000645,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.009759,"latency_s":63.46,"per_attempt":[{"passed":false,"prompt_tokens":951,"cached_prompt_tokens":0,"output_tokens":5912,"reasoning_tokens":5416,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":35.43,"cost_usd":0.005416,"error":null},{"passed":false,"prompt_tokens":1624,"cached_prompt_tokens":0,"output_tokens":4645,"reasoning_tokens":4054,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":28.03,"cost_usd":0.004343,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002698,"latency_s":16.78,"per_attempt":[{"passed":false,"prompt_tokens":953,"cached_prompt_tokens":0,"output_tokens":689,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.75,"cost_usd":0.001435,"error":null},{"passed":false,"prompt_tokens":1832,"cached_prompt_tokens":832,"output_tokens":589,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.03,"cost_usd":0.001263,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.096726,"latency_s":294.69,"per_attempt":[{"passed":false,"prompt_tokens":989,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":199.41,"cost_usd":0.065595,"error":null},{"passed":false,"prompt_tokens":1034,"cached_prompt_tokens":832,"output_tokens":15547,"reasoning_tokens":14986,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":95.28,"cost_usd":0.031131,"error":null}]}}},{"task":"polyglot/food-chain","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000709,"latency_s":4.63,"per_attempt":[{"passed":true,"prompt_tokens":727,"cached_prompt_tokens":0,"output_tokens":707,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.63,"cost_usd":0.000709,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.012981,"latency_s":85.37,"per_attempt":[{"passed":false,"prompt_tokens":725,"cached_prompt_tokens":0,"output_tokens":6237,"reasoning_tokens":5831,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":37.58,"cost_usd":0.005686,"error":null},{"passed":false,"prompt_tokens":1712,"cached_prompt_tokens":0,"output_tokens":7916,"reasoning_tokens":7513,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":47.79,"cost_usd":0.007296,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001976,"latency_s":12.14,"per_attempt":[{"passed":false,"prompt_tokens":727,"cached_prompt_tokens":0,"output_tokens":473,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.1,"cost_usd":0.00099,"error":null},{"passed":false,"prompt_tokens":1497,"cached_prompt_tokens":832,"output_tokens":461,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.04,"cost_usd":0.000987,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.019124,"latency_s":56.61,"per_attempt":[{"passed":false,"prompt_tokens":763,"cached_prompt_tokens":0,"output_tokens":6479,"reasoning_tokens":6086,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":39.13,"cost_usd":0.013004,"error":null},{"passed":false,"prompt_tokens":1635,"cached_prompt_tokens":0,"output_tokens":3011,"reasoning_tokens":2598,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":17.48,"cost_usd":0.00612,"error":null}]}}},{"task":"polyglot/forth","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002277,"latency_s":13.74,"per_attempt":[{"passed":false,"prompt_tokens":788,"cached_prompt_tokens":0,"output_tokens":1101,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.78,"cost_usd":0.00107,"error":null},{"passed":false,"prompt_tokens":2077,"cached_prompt_tokens":0,"output_tokens":1111,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.96,"cost_usd":0.001208,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.020362,"latency_s":310.87,"per_attempt":[{"passed":false,"prompt_tokens":786,"cached_prompt_tokens":0,"output_tokens":11223,"reasoning_tokens":10485,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":67.93,"cost_usd":0.010179,"error":null},{"passed":false,"prompt_tokens":1875,"cached_prompt_tokens":1088,"output_tokens":11130,"reasoning_tokens":10352,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":242.94,"cost_usd":0.010183,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.003337,"latency_s":20.5,"per_attempt":[{"passed":false,"prompt_tokens":788,"cached_prompt_tokens":0,"output_tokens":729,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.21,"cost_usd":0.001505,"error":null},{"passed":false,"prompt_tokens":2196,"cached_prompt_tokens":0,"output_tokens":850,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.29,"cost_usd":0.001832,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.129586,"latency_s":379.53,"per_attempt":[{"passed":false,"prompt_tokens":824,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":181.15,"cost_usd":0.065585,"error":null},{"passed":false,"prompt_tokens":869,"cached_prompt_tokens":0,"output_tokens":31974,"reasoning_tokens":31165,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":198.38,"cost_usd":0.064,"error":null}]}}},{"task":"polyglot/go-counting","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001717,"latency_s":9.83,"per_attempt":[{"passed":false,"prompt_tokens":845,"cached_prompt_tokens":0,"output_tokens":802,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.74,"cost_usd":0.000806,"error":null},{"passed":false,"prompt_tokens":1890,"cached_prompt_tokens":0,"output_tokens":802,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.09,"cost_usd":0.000911,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.010469,"latency_s":68.45,"per_attempt":[{"passed":false,"prompt_tokens":843,"cached_prompt_tokens":0,"output_tokens":6936,"reasoning_tokens":6257,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":42.05,"cost_usd":0.006327,"error":null},{"passed":false,"prompt_tokens":1767,"cached_prompt_tokens":0,"output_tokens":4406,"reasoning_tokens":3717,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":26.4,"cost_usd":0.004142,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.003951,"latency_s":24.82,"per_attempt":[{"passed":false,"prompt_tokens":845,"cached_prompt_tokens":832,"output_tokens":951,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.38,"cost_usd":0.001928,"error":null},{"passed":false,"prompt_tokens":2045,"cached_prompt_tokens":832,"output_tokens":963,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.44,"cost_usd":0.002024,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.105559,"latency_s":289.85,"per_attempt":[{"passed":false,"prompt_tokens":881,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":168.39,"cost_usd":0.065589,"error":null},{"passed":false,"prompt_tokens":926,"cached_prompt_tokens":832,"output_tokens":19970,"reasoning_tokens":19204,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":121.46,"cost_usd":0.039971,"error":null}]}}},{"task":"polyglot/grade-school","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000823,"latency_s":4.94,"per_attempt":[{"passed":false,"prompt_tokens":478,"cached_prompt_tokens":0,"output_tokens":344,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.15,"cost_usd":0.000357,"error":null},{"passed":true,"prompt_tokens":1022,"cached_prompt_tokens":0,"output_tokens":404,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.79,"cost_usd":0.000466,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.00619,"latency_s":40.77,"per_attempt":[{"passed":false,"prompt_tokens":476,"cached_prompt_tokens":0,"output_tokens":4091,"reasoning_tokens":3925,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":24.59,"cost_usd":0.00373,"error":null},{"passed":true,"prompt_tokens":893,"cached_prompt_tokens":0,"output_tokens":2635,"reasoning_tokens":2452,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":16.18,"cost_usd":0.002461,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.000889,"latency_s":5.38,"per_attempt":[{"passed":false,"prompt_tokens":478,"cached_prompt_tokens":0,"output_tokens":201,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.59,"cost_usd":0.000431,"error":null},{"passed":false,"prompt_tokens":879,"cached_prompt_tokens":0,"output_tokens":203,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.79,"cost_usd":0.000459,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.012676,"latency_s":39.81,"per_attempt":[{"passed":false,"prompt_tokens":514,"cached_prompt_tokens":0,"output_tokens":3966,"reasoning_tokens":3791,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":25.93,"cost_usd":0.007963,"error":null},{"passed":true,"prompt_tokens":887,"cached_prompt_tokens":0,"output_tokens":2330,"reasoning_tokens":2150,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.88,"cost_usd":0.004713,"error":null}]}}},{"task":"polyglot/grep","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001236,"latency_s":7.61,"per_attempt":[{"passed":false,"prompt_tokens":390,"cached_prompt_tokens":0,"output_tokens":608,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.0,"cost_usd":0.000586,"error":null},{"passed":false,"prompt_tokens":1301,"cached_prompt_tokens":0,"output_tokens":577,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.61,"cost_usd":0.000649,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.011019,"latency_s":72.64,"per_attempt":[{"passed":false,"prompt_tokens":388,"cached_prompt_tokens":0,"output_tokens":6585,"reasoning_tokens":6305,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":39.8,"cost_usd":0.005965,"error":null},{"passed":false,"prompt_tokens":969,"cached_prompt_tokens":0,"output_tokens":5508,"reasoning_tokens":5195,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":32.84,"cost_usd":0.005054,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002055,"latency_s":13.04,"per_attempt":[{"passed":false,"prompt_tokens":390,"cached_prompt_tokens":0,"output_tokens":490,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.48,"cost_usd":0.001003,"error":null},{"passed":false,"prompt_tokens":1187,"cached_prompt_tokens":0,"output_tokens":490,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.56,"cost_usd":0.001051,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.061802,"latency_s":200.86,"per_attempt":[{"passed":false,"prompt_tokens":426,"cached_prompt_tokens":0,"output_tokens":15024,"reasoning_tokens":14608,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":98.66,"cost_usd":0.030074,"error":null},{"passed":false,"prompt_tokens":1147,"cached_prompt_tokens":0,"output_tokens":15830,"reasoning_tokens":15254,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":102.2,"cost_usd":0.031729,"error":null}]}}},{"task":"polyglot/hangman","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00037,"latency_s":2.22,"per_attempt":[{"passed":true,"prompt_tokens":629,"cached_prompt_tokens":0,"output_tokens":341,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.22,"cost_usd":0.00037,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.011565,"latency_s":78.59,"per_attempt":[{"passed":false,"prompt_tokens":627,"cached_prompt_tokens":0,"output_tokens":4657,"reasoning_tokens":4424,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":28.39,"cost_usd":0.004254,"error":null},{"passed":false,"prompt_tokens":1095,"cached_prompt_tokens":0,"output_tokens":8002,"reasoning_tokens":7766,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":50.2,"cost_usd":0.007311,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001215,"latency_s":7.45,"per_attempt":[{"passed":false,"prompt_tokens":629,"cached_prompt_tokens":0,"output_tokens":277,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.78,"cost_usd":0.000592,"error":null},{"passed":false,"prompt_tokens":1147,"cached_prompt_tokens":0,"output_tokens":277,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.67,"cost_usd":0.000623,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.056278,"latency_s":181.78,"per_attempt":[{"passed":false,"prompt_tokens":665,"cached_prompt_tokens":0,"output_tokens":9499,"reasoning_tokens":9226,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":62.61,"cost_usd":0.019038,"error":null},{"passed":false,"prompt_tokens":1261,"cached_prompt_tokens":0,"output_tokens":18582,"reasoning_tokens":18237,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":119.17,"cost_usd":0.03724,"error":null}]}}},{"task":"polyglot/list-ops","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000473,"latency_s":3.04,"per_attempt":[{"passed":true,"prompt_tokens":471,"cached_prompt_tokens":0,"output_tokens":473,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.04,"cost_usd":0.000473,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.013746,"latency_s":93.9,"per_attempt":[{"passed":false,"prompt_tokens":469,"cached_prompt_tokens":0,"output_tokens":7147,"reasoning_tokens":6855,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":43.99,"cost_usd":0.006479,"error":null},{"passed":true,"prompt_tokens":987,"cached_prompt_tokens":0,"output_tokens":7964,"reasoning_tokens":7672,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":49.91,"cost_usd":0.007266,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001167,"latency_s":7.22,"per_attempt":[{"passed":false,"prompt_tokens":471,"cached_prompt_tokens":0,"output_tokens":270,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.58,"cost_usd":0.000568,"error":null},{"passed":false,"prompt_tokens":973,"cached_prompt_tokens":0,"output_tokens":270,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.64,"cost_usd":0.000598,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.03648,"latency_s":116.41,"per_attempt":[{"passed":false,"prompt_tokens":507,"cached_prompt_tokens":0,"output_tokens":10594,"reasoning_tokens":10278,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":69.85,"cost_usd":0.021218,"error":null},{"passed":true,"prompt_tokens":1053,"cached_prompt_tokens":0,"output_tokens":7599,"reasoning_tokens":7282,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":46.56,"cost_usd":0.015261,"error":null}]}}},{"task":"polyglot/paasio","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001322,"latency_s":8.07,"per_attempt":[{"passed":false,"prompt_tokens":478,"cached_prompt_tokens":0,"output_tokens":626,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.99,"cost_usd":0.000611,"error":null},{"passed":false,"prompt_tokens":1490,"cached_prompt_tokens":1088,"output_tokens":648,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.08,"cost_usd":0.00071,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.012963,"latency_s":88.23,"per_attempt":[{"passed":false,"prompt_tokens":476,"cached_prompt_tokens":0,"output_tokens":7535,"reasoning_tokens":6923,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":46.88,"cost_usd":0.006829,"error":null},{"passed":false,"prompt_tokens":1473,"cached_prompt_tokens":0,"output_tokens":6652,"reasoning_tokens":6009,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":41.35,"cost_usd":0.006134,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002496,"latency_s":15.54,"per_attempt":[{"passed":false,"prompt_tokens":478,"cached_prompt_tokens":0,"output_tokens":593,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.57,"cost_usd":0.001215,"error":null},{"passed":false,"prompt_tokens":1463,"cached_prompt_tokens":832,"output_tokens":609,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.97,"cost_usd":0.001281,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.131206,"latency_s":409.32,"per_attempt":[{"passed":false,"prompt_tokens":514,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":31950,"hit_token_cap":true,"no_code_in_reply":false,"latency_s":202.84,"cost_usd":0.065567,"error":null},{"passed":false,"prompt_tokens":1722,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":206.48,"cost_usd":0.065639,"error":null}]}}},{"task":"polyglot/phone-number","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.00111,"latency_s":6.06,"per_attempt":[{"passed":false,"prompt_tokens":951,"cached_prompt_tokens":0,"output_tokens":469,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.02,"cost_usd":0.000517,"error":null},{"passed":true,"prompt_tokens":1606,"cached_prompt_tokens":0,"output_tokens":480,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.04,"cost_usd":0.000593,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.011463,"latency_s":96.02,"per_attempt":[{"passed":false,"prompt_tokens":949,"cached_prompt_tokens":0,"output_tokens":6569,"reasoning_tokens":6277,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":60.04,"cost_usd":0.006007,"error":null},{"passed":false,"prompt_tokens":1425,"cached_prompt_tokens":0,"output_tokens":5904,"reasoning_tokens":5571,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":35.98,"cost_usd":0.005456,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002574,"latency_s":15.91,"per_attempt":[{"passed":false,"prompt_tokens":951,"cached_prompt_tokens":0,"output_tokens":569,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.45,"cost_usd":0.001195,"error":null},{"passed":false,"prompt_tokens":1960,"cached_prompt_tokens":832,"output_tokens":643,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.46,"cost_usd":0.001379,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.079216,"latency_s":240.34,"per_attempt":[{"passed":false,"prompt_tokens":987,"cached_prompt_tokens":0,"output_tokens":22108,"reasoning_tokens":21672,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":134.21,"cost_usd":0.044275,"error":null},{"passed":true,"prompt_tokens":1629,"cached_prompt_tokens":832,"output_tokens":17434,"reasoning_tokens":16997,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":106.13,"cost_usd":0.034941,"error":null}]}}},{"task":"polyglot/pig-latin","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000696,"latency_s":4.44,"per_attempt":[{"passed":true,"prompt_tokens":727,"cached_prompt_tokens":0,"output_tokens":693,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.44,"cost_usd":0.000696,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.00444,"latency_s":32.52,"per_attempt":[{"passed":true,"prompt_tokens":725,"cached_prompt_tokens":0,"output_tokens":4853,"reasoning_tokens":4590,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":32.52,"cost_usd":0.00444,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000906,"latency_s":5.51,"per_attempt":[{"passed":true,"prompt_tokens":727,"cached_prompt_tokens":0,"output_tokens":431,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.51,"cost_usd":0.000906,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.029544,"latency_s":95.15,"per_attempt":[{"passed":true,"prompt_tokens":763,"cached_prompt_tokens":0,"output_tokens":14749,"reasoning_tokens":14466,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":95.15,"cost_usd":0.029544,"error":null}]}}},{"task":"polyglot/poker","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001687,"latency_s":11.68,"per_attempt":[{"passed":true,"prompt_tokens":131,"cached_prompt_tokens":0,"output_tokens":1860,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.68,"cost_usd":0.001687,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.014761,"latency_s":214.5,"per_attempt":[{"passed":false,"prompt_tokens":129,"cached_prompt_tokens":0,"output_tokens":10000,"reasoning_tokens":9358,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":170.72,"cost_usd":0.009013,"error":null},{"passed":true,"prompt_tokens":1278,"cached_prompt_tokens":0,"output_tokens":6245,"reasoning_tokens":5611,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":43.78,"cost_usd":0.005748,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.003893,"latency_s":24.4,"per_attempt":[{"passed":false,"prompt_tokens":131,"cached_prompt_tokens":0,"output_tokens":952,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.19,"cost_usd":0.001912,"error":null},{"passed":false,"prompt_tokens":1411,"cached_prompt_tokens":0,"output_tokens":948,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.21,"cost_usd":0.001981,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.039678,"latency_s":122.61,"per_attempt":[{"passed":false,"prompt_tokens":167,"cached_prompt_tokens":0,"output_tokens":14202,"reasoning_tokens":13300,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":88.22,"cost_usd":0.028414,"error":null},{"passed":true,"prompt_tokens":1427,"cached_prompt_tokens":0,"output_tokens":5589,"reasoning_tokens":4603,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":34.39,"cost_usd":0.011264,"error":null}]}}},{"task":"polyglot/pov","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001563,"latency_s":10.2,"per_attempt":[{"passed":true,"prompt_tokens":888,"cached_prompt_tokens":0,"output_tokens":1638,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.2,"cost_usd":0.001563,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.021666,"latency_s":212.95,"per_attempt":[{"passed":false,"prompt_tokens":886,"cached_prompt_tokens":0,"output_tokens":8443,"reasoning_tokens":7953,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":51.64,"cost_usd":0.007687,"error":null},{"passed":false,"prompt_tokens":1698,"cached_prompt_tokens":1088,"output_tokens":15367,"reasoning_tokens":14856,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":161.31,"cost_usd":0.013978,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.005318,"latency_s":33.16,"per_attempt":[{"passed":false,"prompt_tokens":888,"cached_prompt_tokens":832,"output_tokens":1048,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.66,"cost_usd":0.002124,"error":null},{"passed":false,"prompt_tokens":2413,"cached_prompt_tokens":832,"output_tokens":1537,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":19.5,"cost_usd":0.003194,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.06906,"latency_s":212.86,"per_attempt":[{"passed":false,"prompt_tokens":924,"cached_prompt_tokens":0,"output_tokens":28102,"reasoning_tokens":27274,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":171.23,"cost_usd":0.056259,"error":null},{"passed":false,"prompt_tokens":2056,"cached_prompt_tokens":832,"output_tokens":6351,"reasoning_tokens":5525,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":41.63,"cost_usd":0.0128,"error":null}]}}},{"task":"polyglot/proverb","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000499,"latency_s":2.73,"per_attempt":[{"passed":false,"prompt_tokens":408,"cached_prompt_tokens":0,"output_tokens":209,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.4,"cost_usd":0.000229,"error":null},{"passed":true,"prompt_tokens":851,"cached_prompt_tokens":0,"output_tokens":206,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.33,"cost_usd":0.000271,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.003274,"latency_s":21.66,"per_attempt":[{"passed":true,"prompt_tokens":406,"cached_prompt_tokens":0,"output_tokens":3593,"reasoning_tokens":3494,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":21.66,"cost_usd":0.003274,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000466,"latency_s":3.06,"per_attempt":[{"passed":true,"prompt_tokens":408,"cached_prompt_tokens":0,"output_tokens":221,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.06,"cost_usd":0.000466,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.028141,"latency_s":87.45,"per_attempt":[{"passed":true,"prompt_tokens":444,"cached_prompt_tokens":0,"output_tokens":14057,"reasoning_tokens":13906,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":87.45,"cost_usd":0.028141,"error":null}]}}},{"task":"polyglot/react","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.000972,"latency_s":5.71,"per_attempt":[{"passed":false,"prompt_tokens":277,"cached_prompt_tokens":0,"output_tokens":427,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.77,"cost_usd":0.000412,"error":null},{"passed":false,"prompt_tokens":1322,"cached_prompt_tokens":0,"output_tokens":475,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.94,"cost_usd":0.00056,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.012098,"latency_s":161.42,"per_attempt":[{"passed":false,"prompt_tokens":275,"cached_prompt_tokens":0,"output_tokens":8872,"reasoning_tokens":8516,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":126.66,"cost_usd":0.008012,"error":null},{"passed":false,"prompt_tokens":1009,"cached_prompt_tokens":0,"output_tokens":4427,"reasoning_tokens":4071,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":34.76,"cost_usd":0.004085,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001428,"latency_s":9.05,"per_attempt":[{"passed":false,"prompt_tokens":277,"cached_prompt_tokens":0,"output_tokens":344,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.6,"cost_usd":0.000705,"error":null},{"passed":false,"prompt_tokens":1007,"cached_prompt_tokens":832,"output_tokens":344,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.45,"cost_usd":0.000723,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.10909,"latency_s":353.28,"per_attempt":[{"passed":false,"prompt_tokens":313,"cached_prompt_tokens":0,"output_tokens":32212,"reasoning_tokens":31525,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":209.15,"cost_usd":0.064443,"error":null},{"passed":true,"prompt_tokens":1790,"cached_prompt_tokens":0,"output_tokens":22270,"reasoning_tokens":21464,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":144.13,"cost_usd":0.044647,"error":null}]}}},{"task":"polyglot/rest-api","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002915,"latency_s":17.04,"per_attempt":[{"passed":false,"prompt_tokens":661,"cached_prompt_tokens":0,"output_tokens":1440,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.09,"cost_usd":0.001362,"error":null},{"passed":false,"prompt_tokens":2436,"cached_prompt_tokens":0,"output_tokens":1455,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.95,"cost_usd":0.001553,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.017829,"latency_s":198.47,"per_attempt":[{"passed":false,"prompt_tokens":659,"cached_prompt_tokens":0,"output_tokens":11207,"reasoning_tokens":10505,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":144.21,"cost_usd":0.010152,"error":null},{"passed":true,"prompt_tokens":1721,"cached_prompt_tokens":0,"output_tokens":8338,"reasoning_tokens":7650,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":54.26,"cost_usd":0.007676,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.004724,"latency_s":29.09,"per_attempt":[{"passed":false,"prompt_tokens":661,"cached_prompt_tokens":0,"output_tokens":913,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.83,"cost_usd":0.001866,"error":null},{"passed":true,"prompt_tokens":2268,"cached_prompt_tokens":0,"output_tokens":1361,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":17.26,"cost_usd":0.002858,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.090577,"latency_s":283.48,"per_attempt":[{"passed":false,"prompt_tokens":697,"cached_prompt_tokens":0,"output_tokens":23320,"reasoning_tokens":21828,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":148.93,"cost_usd":0.046682,"error":null},{"passed":false,"prompt_tokens":2787,"cached_prompt_tokens":0,"output_tokens":21864,"reasoning_tokens":20184,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":134.55,"cost_usd":0.043895,"error":null}]}}},{"task":"polyglot/robot-name","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.000511,"latency_s":3.11,"per_attempt":[{"passed":false,"prompt_tokens":226,"cached_prompt_tokens":0,"output_tokens":231,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.59,"cost_usd":0.00023,"error":null},{"passed":false,"prompt_tokens":719,"cached_prompt_tokens":0,"output_tokens":232,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":1.52,"cost_usd":0.000281,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.008917,"latency_s":93.08,"per_attempt":[{"passed":false,"prompt_tokens":224,"cached_prompt_tokens":0,"output_tokens":5349,"reasoning_tokens":5178,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":32.5,"cost_usd":0.004836,"error":null},{"passed":true,"prompt_tokens":655,"cached_prompt_tokens":0,"output_tokens":4461,"reasoning_tokens":4302,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":60.58,"cost_usd":0.00408,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.000798,"latency_s":4.89,"per_attempt":[{"passed":false,"prompt_tokens":226,"cached_prompt_tokens":0,"output_tokens":186,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.41,"cost_usd":0.000386,"error":null},{"passed":false,"prompt_tokens":678,"cached_prompt_tokens":0,"output_tokens":186,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.48,"cost_usd":0.000413,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.037956,"latency_s":127.61,"per_attempt":[{"passed":true,"prompt_tokens":262,"cached_prompt_tokens":0,"output_tokens":18970,"reasoning_tokens":18816,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":127.61,"cost_usd":0.037956,"error":null}]}}},{"task":"polyglot/scale-generator","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001993,"latency_s":11.43,"per_attempt":[{"passed":false,"prompt_tokens":1071,"cached_prompt_tokens":0,"output_tokens":768,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.7,"cost_usd":0.000798,"error":null},{"passed":false,"prompt_tokens":2335,"cached_prompt_tokens":0,"output_tokens":1068,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.73,"cost_usd":0.001195,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.013841,"latency_s":185.44,"per_attempt":[{"passed":false,"prompt_tokens":1069,"cached_prompt_tokens":0,"output_tokens":9216,"reasoning_tokens":8756,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":148.53,"cost_usd":0.008401,"error":null},{"passed":false,"prompt_tokens":1730,"cached_prompt_tokens":0,"output_tokens":5852,"reasoning_tokens":5451,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":36.91,"cost_usd":0.00544,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.002239,"latency_s":14.1,"per_attempt":[{"passed":true,"prompt_tokens":1071,"cached_prompt_tokens":832,"output_tokens":1100,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.1,"cost_usd":0.002239,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.131183,"latency_s":410.59,"per_attempt":[{"passed":false,"prompt_tokens":1107,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32766,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":201.9,"cost_usd":0.065602,"error":null},{"passed":false,"prompt_tokens":1152,"cached_prompt_tokens":832,"output_tokens":32768,"reasoning_tokens":32766,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":208.69,"cost_usd":0.06558,"error":null}]}}},{"task":"polyglot/sgf-parsing","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.003493,"latency_s":18.12,"per_attempt":[{"passed":false,"prompt_tokens":1510,"cached_prompt_tokens":1088,"output_tokens":1646,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.07,"cost_usd":0.001611,"error":null},{"passed":false,"prompt_tokens":3717,"cached_prompt_tokens":1088,"output_tokens":1703,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":8.05,"cost_usd":0.001883,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.024783,"latency_s":257.19,"per_attempt":[{"passed":false,"prompt_tokens":1508,"cached_prompt_tokens":1088,"output_tokens":16148,"reasoning_tokens":15188,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":160.42,"cost_usd":0.014662,"error":null},{"passed":false,"prompt_tokens":3116,"cached_prompt_tokens":2176,"output_tokens":10947,"reasoning_tokens":10028,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":96.77,"cost_usd":0.01012,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.004736,"latency_s":28.29,"per_attempt":[{"passed":false,"prompt_tokens":1510,"cached_prompt_tokens":832,"output_tokens":1131,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":14.69,"cost_usd":0.002328,"error":null},{"passed":false,"prompt_tokens":2883,"cached_prompt_tokens":832,"output_tokens":1130,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":13.6,"cost_usd":0.002408,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.131235,"latency_s":416.5,"per_attempt":[{"passed":false,"prompt_tokens":1546,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":216.33,"cost_usd":0.065629,"error":null},{"passed":false,"prompt_tokens":1591,"cached_prompt_tokens":832,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":200.17,"cost_usd":0.065606,"error":null}]}}},{"task":"polyglot/simple-linked-list","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000514,"latency_s":2.27,"per_attempt":[{"passed":true,"prompt_tokens":2190,"cached_prompt_tokens":2176,"output_tokens":376,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.27,"cost_usd":0.000514,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.009077,"latency_s":75.45,"per_attempt":[{"passed":false,"prompt_tokens":2188,"cached_prompt_tokens":2176,"output_tokens":5859,"reasoning_tokens":5553,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":49.23,"cost_usd":0.005448,"error":null},{"passed":true,"prompt_tokens":2709,"cached_prompt_tokens":2176,"output_tokens":3779,"reasoning_tokens":3435,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":26.22,"cost_usd":0.003628,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000781,"latency_s":4.6,"per_attempt":[{"passed":true,"prompt_tokens":2190,"cached_prompt_tokens":1664,"output_tokens":350,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.6,"cost_usd":0.000781,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.027344,"latency_s":83.9,"per_attempt":[{"passed":true,"prompt_tokens":2226,"cached_prompt_tokens":0,"output_tokens":13605,"reasoning_tokens":13242,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":83.9,"cost_usd":0.027344,"error":null}]}}},{"task":"polyglot/transpose","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000407,"latency_s":2.8,"per_attempt":[{"passed":true,"prompt_tokens":303,"cached_prompt_tokens":0,"output_tokens":418,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.8,"cost_usd":0.000407,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.008989,"latency_s":61.05,"per_attempt":[{"passed":true,"prompt_tokens":301,"cached_prompt_tokens":0,"output_tokens":9954,"reasoning_tokens":9754,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":61.05,"cost_usd":0.008989,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002329,"latency_s":14.38,"per_attempt":[{"passed":false,"prompt_tokens":303,"cached_prompt_tokens":0,"output_tokens":931,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":11.87,"cost_usd":0.00188,"error":null},{"passed":false,"prompt_tokens":1681,"cached_prompt_tokens":0,"output_tokens":174,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.51,"cost_usd":0.000449,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.033056,"latency_s":109.49,"per_attempt":[{"passed":true,"prompt_tokens":339,"cached_prompt_tokens":0,"output_tokens":16518,"reasoning_tokens":16326,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":109.49,"cost_usd":0.033056,"error":null}]}}},{"task":"polyglot/tree-building","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.000986,"latency_s":5.6,"per_attempt":[{"passed":false,"prompt_tokens":953,"cached_prompt_tokens":0,"output_tokens":407,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.85,"cost_usd":0.000462,"error":null},{"passed":false,"prompt_tokens":1687,"cached_prompt_tokens":1088,"output_tokens":419,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.75,"cost_usd":0.000524,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.037906,"latency_s":205.57,"per_attempt":[{"passed":false,"prompt_tokens":951,"cached_prompt_tokens":0,"output_tokens":9070,"reasoning_tokens":8785,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":55.23,"cost_usd":0.008258,"error":null},{"passed":false,"prompt_tokens":1566,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":150.34,"cost_usd":0.029648,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.002463,"latency_s":15.2,"per_attempt":[{"passed":false,"prompt_tokens":953,"cached_prompt_tokens":832,"output_tokens":574,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.27,"cost_usd":0.00118,"error":null},{"passed":false,"prompt_tokens":1863,"cached_prompt_tokens":832,"output_tokens":598,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":7.93,"cost_usd":0.001283,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.113276,"latency_s":287.57,"per_attempt":[{"passed":false,"prompt_tokens":989,"cached_prompt_tokens":0,"output_tokens":23799,"reasoning_tokens":23325,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":142.47,"cost_usd":0.047657,"error":null},{"passed":false,"prompt_tokens":1797,"cached_prompt_tokens":832,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":145.1,"cost_usd":0.065619,"error":null}]}}},{"task":"polyglot/two-bucket","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.001854,"latency_s":10.18,"per_attempt":[{"passed":false,"prompt_tokens":881,"cached_prompt_tokens":0,"output_tokens":1074,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.63,"cost_usd":0.001055,"error":null},{"passed":true,"prompt_tokens":2237,"cached_prompt_tokens":0,"output_tokens":639,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.55,"cost_usd":0.000799,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.007272,"latency_s":48.19,"per_attempt":[{"passed":true,"prompt_tokens":879,"cached_prompt_tokens":0,"output_tokens":7982,"reasoning_tokens":7639,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":48.19,"cost_usd":0.007272,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.001586,"latency_s":10.07,"per_attempt":[{"passed":true,"prompt_tokens":881,"cached_prompt_tokens":832,"output_tokens":779,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.07,"cost_usd":0.001586,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.13116,"latency_s":400.27,"per_attempt":[{"passed":false,"prompt_tokens":917,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":215.28,"cost_usd":0.065591,"error":null},{"passed":false,"prompt_tokens":962,"cached_prompt_tokens":832,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":184.99,"cost_usd":0.065569,"error":null}]}}},{"task":"polyglot/variable-length-quantity","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000432,"latency_s":2.73,"per_attempt":[{"passed":true,"prompt_tokens":775,"cached_prompt_tokens":0,"output_tokens":394,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":2.73,"cost_usd":0.000432,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006654,"latency_s":44.03,"per_attempt":[{"passed":true,"prompt_tokens":773,"cached_prompt_tokens":0,"output_tokens":7308,"reasoning_tokens":7095,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":44.03,"cost_usd":0.006654,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.001574,"latency_s":9.42,"per_attempt":[{"passed":false,"prompt_tokens":775,"cached_prompt_tokens":0,"output_tokens":353,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.87,"cost_usd":0.000753,"error":null},{"passed":true,"prompt_tokens":1333,"cached_prompt_tokens":0,"output_tokens":371,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.55,"cost_usd":0.000822,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.028107,"latency_s":83.56,"per_attempt":[{"passed":true,"prompt_tokens":811,"cached_prompt_tokens":0,"output_tokens":14029,"reasoning_tokens":13799,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":83.56,"cost_usd":0.028107,"error":null}]}}},{"task":"polyglot/wordy","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.001736,"latency_s":9.3,"per_attempt":[{"passed":false,"prompt_tokens":959,"cached_prompt_tokens":0,"output_tokens":857,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.25,"cost_usd":0.000867,"error":null},{"passed":false,"prompt_tokens":2059,"cached_prompt_tokens":0,"output_tokens":736,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":4.05,"cost_usd":0.000868,"error":null}]},"T-qwen36-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.016613,"latency_s":100.42,"per_attempt":[{"passed":false,"prompt_tokens":957,"cached_prompt_tokens":0,"output_tokens":8440,"reasoning_tokens":8105,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":49.96,"cost_usd":0.007692,"error":null},{"passed":false,"prompt_tokens":1515,"cached_prompt_tokens":0,"output_tokens":9744,"reasoning_tokens":9399,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":50.46,"cost_usd":0.008921,"error":null}]},"T-qwen38-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.00347,"latency_s":19.68,"per_attempt":[{"passed":false,"prompt_tokens":959,"cached_prompt_tokens":0,"output_tokens":821,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":10.36,"cost_usd":0.0017,"error":null},{"passed":false,"prompt_tokens":2483,"cached_prompt_tokens":832,"output_tokens":823,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.32,"cost_usd":0.00177,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.065433,"latency_s":192.92,"per_attempt":[{"passed":false,"prompt_tokens":995,"cached_prompt_tokens":0,"output_tokens":15576,"reasoning_tokens":15169,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":90.2,"cost_usd":0.031212,"error":null},{"passed":true,"prompt_tokens":1632,"cached_prompt_tokens":832,"output_tokens":17074,"reasoning_tokens":16545,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":102.72,"cost_usd":0.034221,"error":null}]}}},{"task":"polyglot/zebra-puzzle","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.004799,"latency_s":21.77,"per_attempt":[{"passed":false,"prompt_tokens":595,"cached_prompt_tokens":0,"output_tokens":2421,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":12.27,"cost_usd":0.002238,"error":null},{"passed":false,"prompt_tokens":3817,"cached_prompt_tokens":0,"output_tokens":2421,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":9.5,"cost_usd":0.002561,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.008048,"latency_s":51.82,"per_attempt":[{"passed":true,"prompt_tokens":593,"cached_prompt_tokens":0,"output_tokens":8876,"reasoning_tokens":8375,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":51.82,"cost_usd":0.008048,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.000185,"latency_s":1.11,"per_attempt":[{"passed":false,"prompt_tokens":595,"cached_prompt_tokens":0,"output_tokens":32,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.56,"cost_usd":0.0001,"error":null},{"passed":true,"prompt_tokens":838,"cached_prompt_tokens":832,"output_tokens":30,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":0.55,"cost_usd":8.5e-05,"error":null}]},"T-qwen38-think":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.006562,"latency_s":22.34,"per_attempt":[{"passed":true,"prompt_tokens":631,"cached_prompt_tokens":0,"output_tokens":3262,"reasoning_tokens":3230,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":22.34,"cost_usd":0.006562,"error":null}]}}},{"task":"polyglot/zipper","in_175_basis":true,"runs":{"T-qwen36-nothink":{"passed":true,"passed_first":true,"attempts":1,"cost_usd":0.000513,"latency_s":3.01,"per_attempt":[{"passed":true,"prompt_tokens":526,"cached_prompt_tokens":0,"output_tokens":512,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":3.01,"cost_usd":0.000513,"error":null}]},"T-qwen36-think":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.01206,"latency_s":72.96,"per_attempt":[{"passed":false,"prompt_tokens":524,"cached_prompt_tokens":0,"output_tokens":6984,"reasoning_tokens":6464,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":40.49,"cost_usd":0.006338,"error":null},{"passed":true,"prompt_tokens":1649,"cached_prompt_tokens":0,"output_tokens":6174,"reasoning_tokens":5654,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":32.47,"cost_usd":0.005722,"error":null}]},"T-qwen38-nothink":{"passed":true,"passed_first":false,"attempts":2,"cost_usd":0.002047,"latency_s":11.83,"per_attempt":[{"passed":false,"prompt_tokens":526,"cached_prompt_tokens":0,"output_tokens":456,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":5.64,"cost_usd":0.000944,"error":null},{"passed":true,"prompt_tokens":1343,"cached_prompt_tokens":832,"output_tokens":524,"reasoning_tokens":0,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":6.19,"cost_usd":0.001104,"error":null}]},"T-qwen38-think":{"passed":false,"passed_first":false,"attempts":2,"cost_usd":0.110866,"latency_s":322.08,"per_attempt":[{"passed":false,"prompt_tokens":562,"cached_prompt_tokens":0,"output_tokens":32768,"reasoning_tokens":32768,"hit_token_cap":true,"no_code_in_reply":true,"latency_s":206.54,"cost_usd":0.06557,"error":null},{"passed":false,"prompt_tokens":607,"cached_prompt_tokens":0,"output_tokens":22630,"reasoning_tokens":21813,"hit_token_cap":false,"no_code_in_reply":false,"latency_s":115.54,"cost_usd":0.045296,"error":null}]}}}]}
