{
 "id": "api-qwen38-qwen36-coding-175-thinking-on-off",
 "kind": "measurement",
 "status": "published",
 "title": "Qwen3.8-27B and Qwen3.6-35B-A3B on our API, 175 single-file Python tasks: thinking on vs off",
 "published": "2026-11-20",
 "summary": "Run on 2026-11-20 through our public API, api.tokenwatt.io (streamed chat completions, max_tokens 32,768, 8 in flight), with the harness of our published CodeRouter benchmark: 175 single-file Python tasks (Exercism plus Aider's hard set); the model returns the whole file, the task's tests run in a rented Lambda sandbox, and a failure gets one retry with the test output. Thinking off: reasoning_effort \"none\", temperature 0. Thinking on: the model's default effort, with Qwen's recommended sampling (temperature 0.6, top_p 0.95, top_k 20). Cost = reported tokens × our list prices, not our GPU cost. The Qwen3.8 thinking-on run came after we enabled MTP speculative decoding (18:55 UTC), the other three before it, so its times are not comparable. One run per configuration. Write-up: /blog/qwen38-qwen36-thinking-on-off-coding-benchmark.html",
 "conclusion": "With thinking on, Qwen3.8-27B solved 78.9% with one retry against 73.1% off (+5.7 points), for $7.02 instead of $0.27 (26×); 42 of its 247 attempts (17%) hit the 32,768-token cap. Qwen3.6-35B-A3B went from 78.3% to 82.3% for $0.16 → $1.27. On the same tasks Claude Haiku 5.5 solved 93.7% for $0.15 and Kimi K2.6 94.9% for $0.57.",
 "headline": {
  "value": "26×",
  "label": "Qwen3.8-27B cost, thinking on vs off, for +5.7 points"
 },
 "hardware": {
  "vendor": "NVIDIA",
  "accelerator": "NVIDIA H200",
  "count": 1,
  "rated_bw_tbs_per_gpu": 4.8,
  "driver": "",
  "host": "RunPod Secure Cloud 1x H200 SXM 141 GB (our API's server)"
 },
 "software": {
  "engine": "vLLM",
  "engine_version": "0.33.0",
  "kernels": "vllm/vllm-openai:v0.33.0, two engines under CUDA MPS; MTP on Qwen3.8 from 18:55 UTC"
 },
 "model": {
  "name": "Qwen3.8-27B-FP8 + Qwen3.6-35B-A3B-FP8",
  "shape": "",
  "quantization": "FP8 weights",
  "parallelism": "1 replica each × TP1, one shared GPU"
 },
 "pills": [
  "Coding eval · 175 tasks",
  "Our own API"
 ],
 "variables_measured": [
  "share of tasks solved on the first try and with one retry",
  "cost at list prices",
  "median time per task",
  "attempts that hit the output-token cap",
  "output and thinking tokens"
 ],
 "charts_sub": "Qwen3.6 = Qwen3.6-35B-A3B, Qwen3.8 = Qwen3.8-27B; off / on = thinking. One run per configuration, 175 tasks.",
 "charts": [
  {
   "title": "Solved with one retry",
   "unit": "% of 175 tasks",
   "digits": 1,
   "suffix": "%",
   "max": 100,
   "better": "higher is better",
   "note": "Axis runs to 100%.",
   "bars": [
    {
     "label": "Qwen3.6, off",
     "key": "base",
     "value": 78.3
    },
    {
     "label": "Qwen3.6, on",
     "key": "tuned",
     "value": 82.3
    },
    {
     "label": "Qwen3.8, off",
     "key": "base",
     "value": 73.1
    },
    {
     "label": "Qwen3.8, on",
     "key": "tuned",
     "value": 78.9
    }
   ]
  },
  {
   "title": "Solved on the first try",
   "unit": "% of 175 tasks",
   "digits": 1,
   "suffix": "%",
   "max": 100,
   "better": "higher is better",
   "note": "Axis runs to 100%.",
   "bars": [
    {
     "label": "Qwen3.6, off",
     "key": "base",
     "value": 68.0
    },
    {
     "label": "Qwen3.6, on",
     "key": "tuned",
     "value": 68.6
    },
    {
     "label": "Qwen3.8, off",
     "key": "base",
     "value": 60.6
    },
    {
     "label": "Qwen3.8, on",
     "key": "tuned",
     "value": 58.9
    }
   ]
  },
  {
   "title": "Cost of all 175 tasks",
   "unit": "US$ at our list prices, retries included",
   "digits": 2,
   "better": "lower is better",
   "bars": [
    {
     "label": "Qwen3.6, off",
     "key": "base",
     "value": 0.1565
    },
    {
     "label": "Qwen3.6, on",
     "key": "tuned",
     "value": 1.2732
    },
    {
     "label": "Qwen3.8, off",
     "key": "base",
     "value": 0.2706
    },
    {
     "label": "Qwen3.8, on",
     "key": "tuned",
     "value": 7.0151
    }
   ]
  },
  {
   "title": "Median time per task",
   "unit": "seconds, retry included",
   "digits": 1,
   "suffix": " s",
   "better": "lower is better",
   "note": "Qwen3.8 thinking on ran after MTP speculative decoding was enabled; the other three before.",
   "bars": [
    {
     "label": "Qwen3.6, off",
     "key": "base",
     "value": 3.2
    },
    {
     "label": "Qwen3.6, on",
     "key": "tuned",
     "value": 33.04
    },
    {
     "label": "Qwen3.8, off",
     "key": "base",
     "value": 5.54
    },
    {
     "label": "Qwen3.8, on",
     "key": "tuned",
     "value": 76.92
    }
   ]
  },
  {
   "title": "Attempts that hit the 32,768-token cap",
   "unit": "% of attempts",
   "digits": 1,
   "suffix": "%",
   "better": "lower is better",
   "bars": [
    {
     "label": "Qwen3.6, off",
     "key": "base",
     "value": 0.0
    },
    {
     "label": "Qwen3.6, on",
     "key": "tuned",
     "value": 0.9
    },
    {
     "label": "Qwen3.8, off",
     "key": "base",
     "value": 0.0
    },
    {
     "label": "Qwen3.8, on",
     "key": "tuned",
     "value": 17.0
    }
   ]
  }
 ],
 "tables": [
  {
   "title": "All four runs",
   "sub": "Qwen3.6 = Qwen3.6-35B-A3B, Qwen3.8 = Qwen3.8-27B. 175 tasks; cost is for all 175, retries included; per solved task = cost ÷ tasks solved with one retry; median time is per task, retry included.",
   "columns": [
    "Run",
    "First try",
    "One retry",
    "Cost",
    "Per solved task",
    "Median time"
   ],
   "highlight_rows": [
    3
   ],
   "rows": [
    [
     "Qwen3.6, thinking off",
     "68.0%",
     "78.3%",
     "$0.16",
     "$0.0011",
     "3.2 s"
    ],
    [
     "Qwen3.6, thinking on",
     "68.6%",
     "82.3%",
     "$1.27",
     "$0.0088",
     "33.0 s"
    ],
    [
     "Qwen3.8, thinking off",
     "60.6%",
     "73.1%",
     "$0.27",
     "$0.0021",
     "5.5 s"
    ],
    [
     "Qwen3.8, thinking on",
     "58.9%",
     "78.9%",
     "$7.02",
     "$0.0508",
     "76.9 s"
    ]
   ]
  },
  {
   "title": "Where the tokens went",
   "sub": "Attempts = first tries plus retries. \"No code\" = the reply had no Python code block, so no test ran: for Qwen3.8 thinking on, 41 of the 42 capped attempts plus 3 that ended below the cap (cause not investigated; the harness doesn't keep reply text). At cap = output reached max_tokens (32,768). Thinking = reasoning tokens ÷ output tokens, as the API reported them. Median / attempt = median output tokens per attempt. Capped cost = what the attempts at the cap cost.",
   "columns": [
    "Run",
    "Attempts",
    "At cap",
    "No code",
    "Output tokens",
    "Thinking",
    "Median / attempt",
    "Capped cost"
   ],
   "highlight_rows": [
    3
   ],
   "rows": [
    [
     "Qwen3.6, thinking off",
     "231",
     "0 (0.0%)",
     "0",
     "0.15M",
     "0%",
     "473",
     "$0.00"
    ],
    [
     "Qwen3.6, thinking on",
     "230",
     "2 (0.9%)",
     "2",
     "1.39M",
     "95%",
     "5,368",
     "$0.06"
    ],
    [
     "Qwen3.8, thinking off",
     "244",
     "0 (0.0%)",
     "0",
     "0.13M",
     "0%",
     "389",
     "$0.00"
    ],
    [
     "Qwen3.8, thinking on",
     "247",
     "42 (17.0%)",
     "44",
     "3.50M",
     "98%",
     "11,863",
     "$2.75"
    ]
   ]
  },
  {
   "title": "Thinking on vs off, task by task",
   "sub": "Tasks solved with one retry. Of the 14 tasks Qwen3.8 solved only with thinking off, 10 had a thinking-on attempt that hit the cap.",
   "columns": [
    "Model",
    "Only with thinking on",
    "Only with thinking off",
    "Both",
    "Neither"
   ],
   "rows": [
    [
     "Qwen3.6-35B-A3B",
     "14",
     "7",
     "130",
     "24"
    ],
    [
     "Qwen3.8-27B",
     "24",
     "14",
     "114",
     "23"
    ]
   ]
  },
  {
   "title": "By task set",
   "sub": "Tasks solved on the first try / with one retry.",
   "columns": [
    "Run",
    "Exercism (141)",
    "Aider's hard set (34)"
   ],
   "rows": [
    [
     "Qwen3.6, thinking off",
     "107 / 119",
     "12 / 18"
    ],
    [
     "Qwen3.6, thinking on",
     "111 / 126",
     "9 / 18"
    ],
    [
     "Qwen3.8, thinking off",
     "97 / 113",
     "9 / 15"
    ],
    [
     "Qwen3.8, thinking on",
     "95 / 121",
     "8 / 17"
    ]
   ]
  },
  {
   "title": "Context: other models on the same 175 tasks",
   "sub": "Reference rows from our published CodeRouter benchmark (coderouter.io/benchmark and the 2026-11-19 Gemini 4 Neon post): same tasks, prompt and grading; called through other providers on other days, priced at their list prices; times depend on each provider's load. Qwen3-Coder-30B ran on our API on 2026-11-12 with max_tokens 4,096 and was retired on 2026-11-19.",
   "columns": [
    "Model",
    "First try",
    "One retry",
    "Cost",
    "Median time"
   ],
   "highlight_rows": [
    6,
    7
   ],
   "rows": [
    [
     "Claude Opus 5.5",
     "98.3%",
     "100.0%",
     "$2.61",
     "4.8 s"
    ],
    [
     "Claude Sonnet 5.5",
     "93.7%",
     "100.0%",
     "$1.48",
     "3.9 s"
    ],
    [
     "GPT-6.2 Sol",
     "92.0%",
     "97.1%",
     "$0.88",
     "2.7 s"
    ],
    [
     "Qwen4-Max",
     "83.4%",
     "96.0%",
     "$1.47",
     "11.3 s"
    ],
    [
     "Kimi K2.6",
     "86.9%",
     "94.9%",
     "$0.57",
     "9.4 s"
    ],
    [
     "Claude Haiku 5.5",
     "72.0%",
     "93.7%",
     "$0.15",
     "5.5 s"
    ],
    [
     "Qwen3.6, thinking on (ours)",
     "68.6%",
     "82.3%",
     "$1.27",
     "33.0 s"
    ],
    [
     "Qwen3.8, thinking on (ours)",
     "58.9%",
     "78.9%",
     "$7.02",
     "76.9 s"
    ],
    [
     "Qwen3.6, thinking off (ours)",
     "68.0%",
     "78.3%",
     "$0.16",
     "3.2 s"
    ],
    [
     "Qwen3.8, thinking off (ours)",
     "60.6%",
     "73.1%",
     "$0.27",
     "5.5 s"
    ],
    [
     "Qwen3-Coder-30B (ours until Nov 19)",
     "64.0%",
     "64.6%",
     "$0.06",
     "3.7 s"
    ]
   ]
  }
 ],
 "setup": [
  [
   "Endpoint",
   "https://api.tokenwatt.io/v1/chat/completions, streamed (include_usage), our public API"
  ],
  [
   "Models",
   "qwen/qwen3.8-27b (Qwen/Qwen3.8-27B-FP8), qwen/qwen3.6-35b-a3b (Qwen/Qwen3.6-35B-A3B-FP8)"
  ],
  [
   "Serving",
   "1× H200, two vLLM 0.33.0 engines under CUDA MPS; MTP (3 draft tokens) on Qwen3.8 from 2026-11-20 18:55 UTC"
  ],
  [
   "Tasks",
   "175: Exercism Python (141) + Aider's polyglot hard set, Python (34); exercism/alphametics left out as in the published benchmark"
  ],
  [
   "Prompt",
   "system: return the complete file in one python code block; user: instructions + stub file (bench/coderouter-eval/run_eval.py)"
  ],
  [
   "Grading",
   "the task's own tests (pytest, 60 s), run in a rented Lambda gpu_1x_a10 instance; one retry with the failing output"
  ],
  [
   "Thinking off",
   "reasoning_effort \"none\", temperature 0"
  ],
  [
   "Thinking on",
   "no reasoning_effort (model default), temperature 0.6, top_p 0.95, top_k 20 (Qwen's recommended thinking-mode sampling)"
  ],
  [
   "Output budget",
   "max_tokens 32,768 per request, thinking included"
  ],
  [
   "Concurrency",
   "8 tasks in flight"
  ],
  [
   "Prices",
   "Qwen3.8: $0.06 in / $2.00 out / $0.03 cached input per million; Qwen3.6: $0.10 / $0.90 / $0.08 (our list prices)"
  ],
  [
   "Cost",
   "Σ reported tokens × list price, retries included; thinking billed as output"
  ],
  [
   "Median time",
   "per task, wall time of the request(s), retry included; test time not counted"
  ],
  [
   "When",
   "2026-11-20: Qwen3.8 off ~18:09–18:13, Qwen3.6 off ~18:13–18:16, Qwen3.6 on ~18:16–18:38, Qwen3.8 on ~19:11–20:01 UTC"
  ]
 ],
 "not_covered": [
  "Repeat runs: one run per configuration; a rerun could move a few tasks either way",
  "reasoning_effort low or medium, and output budgets other than 32,768 tokens",
  "Thinking on at temperature 0, or thinking off with Qwen's sampling: sampling differs between the on and off runs",
  "Qwen3.8 thinking-on latency without MTP (its run came after MTP was enabled)",
  "Agent sessions, multi-file repositories, long contexts or tool use: these are single-file, single-turn tasks",
  "Tasks unseen in training: the tasks are public",
  "Reply text: the harness keeps no model output, so why an attempt ran to the cap was not examined"
 ],
 "reproduce_title": "Reproduce",
 "reproduce_sub": "Harness and raw per-attempt records: bench/coderouter-eval/ in the TokenWatt repository (run_eval.py, lambda_eval_multi.py, results/T-tokenwatt-qwen38-36/). Run model-written code only in a sandbox.",
 "reproduce": "# one configuration (thinking on, Qwen3.8); tests run where this runs, so use a throwaway machine\npython3 run_eval.py --tasks tasks/exercism tasks/polyglot \\\n  --base-url https://api.tokenwatt.io/v1 --key-env TOKENWATT_API_KEY \\\n  --model qwen/qwen3.8-27b --label T-qwen38-think --prices prices-tokenwatt-qwen38-36.json \\\n  --concurrency 8 --max-tokens 32768 --stream --timeout 1500 \\\n  --extra-body '{\"temperature\":0.6,\"top_p\":0.95,\"top_k\":20}'\n\n# thinking off\n  ... --label T-qwen38-nothink --extra-body '{\"reasoning_effort\":\"none\"}'   # harness default temperature 0\n\n# 175-task summaries, this report and the per-task data (exercism/alphametics left out)\npython3 bench/coderouter-eval/results/T-tokenwatt-qwen38-36/make_site_report.py",
 "post": "/blog/qwen38-qwen36-thinking-on-off-coding-benchmark.html",
 "data": {
  "source": "bench/coderouter-eval/results/T-tokenwatt-qwen38-36/T-*.jsonl (TokenWatt repository)",
  "per_task": "/data/coding-suite/qwen38-qwen36-thinking-2026-11.json",
  "basis_tasks": 175,
  "max_tokens": 32768,
  "runs": {
   "Qwen3.6, thinking off": {
    "tasks": 175,
    "passed": 137,
    "passed_first": 119,
    "pass_rate_pct": 78.3,
    "first_try_pct": 68.0,
    "cost_usd": 0.1565,
    "cost_per_solved_task_usd": 0.00114,
    "median_latency_s": 3.2,
    "attempts": 231,
    "retried_tasks": 56,
    "solved_by_retry": 18,
    "attempts_at_cap": 0,
    "attempts_at_cap_pct": 0.0,
    "attempts_at_cap_no_code": 0,
    "attempts_no_code_below_cap": 0,
    "first_attempts_at_cap": 0,
    "tasks_with_cap": 0,
    "tasks_cap_both_attempts": 0,
    "tasks_with_cap_passed": 0,
    "cost_of_capped_attempts_usd": 0,
    "capped_attempt_latency_s": null,
    "prompt_tokens": 240103,
    "cached_prompt_tokens": 20672,
    "output_tokens": 147623,
    "reasoning_tokens": 0,
    "median_output_tokens_per_attempt": 473,
    "errors": 0,
    "Exercism (141 tasks)": {
     "tasks": 141,
     "passed_first": 107,
     "passed": 119
    },
    "Aider's hard set (34 tasks)": {
     "tasks": 34,
     "passed_first": 12,
     "passed": 18
    },
    "settings": {
     "label": "Qwen3.6-35B-A3B, thinking off",
     "model": "qwen/qwen3.6-35b-a3b",
     "thinking": "off",
     "request_body_extra": {
      "reasoning_effort": "none",
      "temperature": 0
     },
     "when": "2026-11-20 ~18:13–18:16 UTC"
    }
   },
   "Qwen3.6, thinking on": {
    "tasks": 175,
    "passed": 144,
    "passed_first": 120,
    "pass_rate_pct": 82.3,
    "first_try_pct": 68.6,
    "cost_usd": 1.2732,
    "cost_per_solved_task_usd": 0.00884,
    "median_latency_s": 33.04,
    "attempts": 230,
    "retried_tasks": 55,
    "solved_by_retry": 24,
    "attempts_at_cap": 2,
    "attempts_at_cap_pct": 0.9,
    "attempts_at_cap_no_code": 2,
    "attempts_no_code_below_cap": 0,
    "first_attempts_at_cap": 0,
    "tasks_with_cap": 2,
    "tasks_cap_both_attempts": 0,
    "tasks_with_cap_passed": 0,
    "cost_of_capped_attempts_usd": 0.0594,
    "capped_attempt_latency_s": {
     "median": 174.805,
     "min": 150.34,
     "max": 199.27
    },
    "prompt_tokens": 217615,
    "cached_prompt_tokens": 53312,
    "output_tokens": 1391628,
    "reasoning_tokens": 1318907,
    "median_output_tokens_per_attempt": 5367.5,
    "errors": 0,
    "Exercism (141 tasks)": {
     "tasks": 141,
     "passed_first": 111,
     "passed": 126
    },
    "Aider's hard set (34 tasks)": {
     "tasks": 34,
     "passed_first": 9,
     "passed": 18
    },
    "settings": {
     "label": "Qwen3.6-35B-A3B, thinking on",
     "model": "qwen/qwen3.6-35b-a3b",
     "thinking": "on (model default)",
     "request_body_extra": {
      "temperature": 0.6,
      "top_p": 0.95,
      "top_k": 20
     },
     "when": "2026-11-20 ~18:16–18:38 UTC"
    }
   },
   "Qwen3.8, thinking off": {
    "tasks": 175,
    "passed": 128,
    "passed_first": 106,
    "pass_rate_pct": 73.1,
    "first_try_pct": 60.6,
    "cost_usd": 0.2706,
    "cost_per_solved_task_usd": 0.00211,
    "median_latency_s": 5.54,
    "attempts": 244,
    "retried_tasks": 69,
    "solved_by_retry": 22,
    "attempts_at_cap": 0,
    "attempts_at_cap_pct": 0.0,
    "attempts_at_cap_no_code": 0,
    "attempts_no_code_below_cap": 0,
    "first_attempts_at_cap": 0,
    "tasks_with_cap": 0,
    "tasks_cap_both_attempts": 0,
    "tasks_with_cap_passed": 0,
    "cost_of_capped_attempts_usd": 0,
    "capped_attempt_latency_s": null,
    "prompt_tokens": 259378,
    "cached_prompt_tokens": 49920,
    "output_tokens": 128256,
    "reasoning_tokens": 0,
    "median_output_tokens_per_attempt": 389.0,
    "errors": 0,
    "Exercism (141 tasks)": {
     "tasks": 141,
     "passed_first": 97,
     "passed": 113
    },
    "Aider's hard set (34 tasks)": {
     "tasks": 34,
     "passed_first": 9,
     "passed": 15
    },
    "settings": {
     "label": "Qwen3.8-27B, thinking off",
     "model": "qwen/qwen3.8-27b",
     "thinking": "off",
     "request_body_extra": {
      "reasoning_effort": "none",
      "temperature": 0
     },
     "when": "2026-11-20 ~18:09–18:13 UTC"
    }
   },
   "Qwen3.8, thinking on": {
    "tasks": 175,
    "passed": 138,
    "passed_first": 103,
    "pass_rate_pct": 78.9,
    "first_try_pct": 58.9,
    "cost_usd": 7.0151,
    "cost_per_solved_task_usd": 0.05083,
    "median_latency_s": 76.92,
    "attempts": 247,
    "retried_tasks": 72,
    "solved_by_retry": 35,
    "attempts_at_cap": 42,
    "attempts_at_cap_pct": 17.0,
    "attempts_at_cap_no_code": 41,
    "attempts_no_code_below_cap": 3,
    "first_attempts_at_cap": 27,
    "tasks_with_cap": 30,
    "tasks_cap_both_attempts": 12,
    "tasks_with_cap_passed": 7,
    "cost_of_capped_attempts_usd": 2.7547,
    "capped_attempt_latency_s": {
     "median": 206.39499999999998,
     "min": 145.1,
     "max": 309.89
    },
    "prompt_tokens": 231817,
    "cached_prompt_tokens": 36608,
    "output_tokens": 3501139,
    "reasoning_tokens": 3429693,
    "median_output_tokens_per_attempt": 11863,
    "errors": 0,
    "Exercism (141 tasks)": {
     "tasks": 141,
     "passed_first": 95,
     "passed": 121
    },
    "Aider's hard set (34 tasks)": {
     "tasks": 34,
     "passed_first": 8,
     "passed": 17
    },
    "settings": {
     "label": "Qwen3.8-27B, thinking on",
     "model": "qwen/qwen3.8-27b",
     "thinking": "on (model default)",
     "request_body_extra": {
      "temperature": 0.6,
      "top_p": 0.95,
      "top_k": 20
     },
     "when": "2026-11-20 ~19:11–20:01 UTC (after MTP)"
    }
   }
  },
  "thinking_on_vs_off_tasks": {
   "Qwen3.6-35B-A3B": {
    "only_thinking_on": 14,
    "only_thinking_off": 7,
    "both": 130,
    "neither": 24,
    "lost_tasks_with_capped_attempt": 0
   },
   "Qwen3.8-27B": {
    "only_thinking_on": 24,
    "only_thinking_off": 14,
    "both": 114,
    "neither": 23,
    "lost_tasks_with_capped_attempt": 10
   }
  },
  "derived": {
   "qwen38_cost_ratio_on_off": 25.9,
   "qwen38_points_on_minus_off": 5.7,
   "qwen38_first_try_points_on_minus_off": -1.7,
   "qwen36_cost_ratio_on_off": 8.1,
   "qwen36_points_on_minus_off": 4.0,
   "qwen38_on_cost_vs_opus": 2.69,
   "qwen38_on_points_below_opus": 21.1,
   "qwen38_on_capped_cost_share_pct": 39.3,
   "qwen38_on_thinking_share_pct": 98.0,
   "qwen36_on_thinking_share_pct": 94.8,
   "qwen36_on_vs_qwen38_on_cost_pct": 18.1
  },
  "qwen3_coder_30b_2026_11_12": {
   "tasks": 175,
   "passed": 113,
   "passed_first": 112,
   "pass_rate_pct": 64.6,
   "first_try_pct": 64.0,
   "cost_usd": 0.0571,
   "cost_per_solved_task_usd": 0.00051,
   "median_latency_s": 3.73,
   "attempts": 238,
   "retried_tasks": 63,
   "solved_by_retry": 1,
   "attempts_at_cap": 0,
   "attempts_at_cap_pct": 0.0,
   "attempts_at_cap_no_code": 0,
   "attempts_no_code_below_cap": 0,
   "first_attempts_at_cap": 0,
   "tasks_with_cap": 0,
   "tasks_cap_both_attempts": 0,
   "tasks_with_cap_passed": 0,
   "cost_of_capped_attempts_usd": 0,
   "capped_attempt_latency_s": null,
   "prompt_tokens": 244124,
   "cached_prompt_tokens": 127392,
   "output_tokens": 142899,
   "reasoning_tokens": 0,
   "median_output_tokens_per_attempt": 426.0,
   "errors": 0
  },
  "reference_published_benchmark": [
   {
    "model": "Claude Opus 5.5",
    "first_try_pct": 98.3,
    "with_one_retry_pct": 100.0,
    "cost_usd": 2.61,
    "median_s": 4.8
   },
   {
    "model": "Claude Sonnet 5.5",
    "first_try_pct": 93.7,
    "with_one_retry_pct": 100.0,
    "cost_usd": 1.48,
    "median_s": 3.9
   },
   {
    "model": "GPT-6.2 Sol",
    "first_try_pct": 92.0,
    "with_one_retry_pct": 97.1,
    "cost_usd": 0.88,
    "median_s": 2.7
   },
   {
    "model": "Qwen4-Max",
    "first_try_pct": 83.4,
    "with_one_retry_pct": 96.0,
    "cost_usd": 1.47,
    "median_s": 11.3
   },
   {
    "model": "Kimi K2.6",
    "first_try_pct": 86.9,
    "with_one_retry_pct": 94.9,
    "cost_usd": 0.57,
    "median_s": 9.4
   },
   {
    "model": "Claude Haiku 5.5",
    "first_try_pct": 72.0,
    "with_one_retry_pct": 93.7,
    "cost_usd": 0.15,
    "median_s": 5.5
   }
  ],
  "reference_source": "https://www.coderouter.io/blog/gemini-4-neon-coding-benchmark-2026"
 }
}
