{
 "id": "h100-nvl-qwen38-27b-mtp-speculative-decoding-vllm",
 "kind": "measurement",
 "status": "published",
 "title": "MTP speculative decoding on Qwen3.8-27B FP8, 1× H100 NVL, vLLM 0.33.0: 0 to 4 draft tokens",
 "published": "2026-11-20",
 "summary": "Measured 2026-11-20 on one H100 NVL (94 GB, 3.9 TB/s, PCIe, 310 W power limit on this host), RunPod Secure Cloud, $3.19/h, 31.4 minutes, $1.67. vllm/vllm-openai:v0.33.0, Qwen/Qwen3.8-27B-FP8 (revision c3a462cc), which ships a one-layer MTP head; vLLM's speculative config with method mtp and 1 to 4 draft tokens (num_speculative_tokens). Chat completions, streamed, 512 output tokens, thinking off, default sampling (temperature 1.0, top-p 0.95, top-k 20) unless noted; three short prompts (prose, code, JSON); one engine start per configuration. Not measured on our production H200. Write-up: /blog/qwen38-27b-mtp-speculative-decoding.html",
 "conclusion": "With 3 draft tokens, single-request decode went from 74.8 to 198.4 tok/s (2.65×), and total output rose 101%, 49% and 19% at 4, 16 and 48 requests in flight. 4 drafts were faster for one request (2.90×) but gave less total output than 3 at 16 and 48 in flight. GSM8K-200 scored 99.0% with MTP against 98.0% without. KV cache capacity fell 24% (709,527 to 541,801 tokens).",
 "headline": {
  "value": "2.65×",
  "label": "single-request decode, 3 draft tokens vs MTP off"
 },
 "hardware": {
  "vendor": "NVIDIA",
  "accelerator": "NVIDIA H100 NVL",
  "count": 1,
  "rated_bw_tbs_per_gpu": 3.9,
  "driver": "595.112.07 (CUDA 13.2)",
  "host": "RunPod Secure Cloud 1x H100 NVL 94 GB (US-GA-2), 310 W power limit, $3.19/h"
 },
 "software": {
  "engine": "vLLM",
  "engine_version": "0.33.0",
  "kernels": "vllm/vllm-openai:v0.33.0, FLASH_ATTN, FlashInfer DeepGEMM FP8"
 },
 "model": {
  "name": "Qwen3.8-27B-FP8",
  "shape": "Dense",
  "quantization": "FP8 weights, BF16 KV cache",
  "parallelism": "1 replica × TP1"
 },
 "pills": [
  "Speculative decoding · MTP"
 ],
 "variables_measured": [
  "single-request decode speed (output tokens/s)",
  "aggregate output tokens/s at 4, 16, 48 in flight",
  "time to first token",
  "mean acceptance length",
  "KV cache capacity (tokens)",
  "GSM8K accuracy"
 ],
 "charts_sub": "Medians. Draft tokens = num_speculative_tokens.",
 "charts": [
  {
   "title": "One request at a time",
   "unit": "decode tok/s, median of 9 runs",
   "digits": 1,
   "better": "higher is better",
   "bars": [
    {
     "label": "MTP off",
     "key": "base",
     "value": 74.77
    },
    {
     "label": "1 draft token",
     "key": "base",
     "value": 130.97
    },
    {
     "label": "2 draft tokens",
     "key": "base",
     "value": 166.93
    },
    {
     "label": "3 draft tokens",
     "key": "tuned",
     "value": 198.39
    },
    {
     "label": "4 draft tokens",
     "key": "base",
     "value": 216.94
    }
   ]
  },
  {
   "title": "16 requests in flight",
   "unit": "total output tok/s",
   "digits": 0,
   "better": "higher is better",
   "bars": [
    {
     "label": "MTP off",
     "key": "base",
     "value": 903.18
    },
    {
     "label": "1 draft token",
     "key": "base",
     "value": 1159.14
    },
    {
     "label": "2 draft tokens",
     "key": "base",
     "value": 1294.43
    },
    {
     "label": "3 draft tokens",
     "key": "tuned",
     "value": 1345.16
    },
    {
     "label": "4 draft tokens",
     "key": "base",
     "value": 1198.92
    }
   ]
  },
  {
   "title": "48 requests in flight",
   "unit": "total output tok/s",
   "digits": 0,
   "better": "higher is better",
   "note": "Measured for MTP off, 3 and 4 draft tokens only.",
   "bars": [
    {
     "label": "MTP off",
     "key": "base",
     "value": 1742.11
    },
    {
     "label": "1 draft token",
     "key": "base",
     "value": null
    },
    {
     "label": "2 draft tokens",
     "key": "base",
     "value": null
    },
    {
     "label": "3 draft tokens",
     "key": "tuned",
     "value": 2065.54
    },
    {
     "label": "4 draft tokens",
     "key": "base",
     "value": 1940.6
    }
   ]
  },
  {
   "title": "KV cache capacity",
   "unit": "tokens, from the engine's startup log",
   "digits": 0,
   "better": "higher is better",
   "note": "0.92 GPU memory share, 32,768-token max context.",
   "bars": [
    {
     "label": "MTP off",
     "key": "base",
     "value": 709527
    },
    {
     "label": "1 draft token",
     "key": "base",
     "value": 624482
    },
    {
     "label": "2 draft tokens",
     "key": "base",
     "value": 570163
    },
    {
     "label": "3 draft tokens",
     "key": "tuned",
     "value": 541801
    },
    {
     "label": "4 draft tokens",
     "key": "base",
     "value": 514618
    }
   ]
  }
 ],
 "tables": [
  {
   "title": "One request at a time",
   "sub": "Thinking off, default sampling, 3 prompts × 3 runs. Tokens per step = vLLM's mean acceptance length (1 + accepted ÷ drafts).",
   "columns": [
    "Configuration",
    "Decode tok/s, median (range)",
    "vs MTP off",
    "Tokens per step",
    "TTFT, median",
    "KV cache, tokens"
   ],
   "highlight_rows": [
    3
   ],
   "rows": [
    [
     "MTP off",
     "74.8 (74.3–75.3)",
     "—",
     "1",
     "75 ms",
     "709,527"
    ],
    [
     "1 draft token",
     "131.0 (114.0–133.8)",
     "1.75×",
     "1.87",
     "77 ms",
     "624,482 (−12%)"
    ],
    [
     "2 draft tokens",
     "166.9 (130.3–183.1)",
     "2.23×",
     "2.55",
     "81 ms",
     "570,163 (−20%)"
    ],
    [
     "3 draft tokens",
     "198.4 (138.3–222.3)",
     "2.65×",
     "3.14",
     "80 ms",
     "541,801 (−24%)"
    ],
    [
     "4 draft tokens",
     "216.9 (136.5–252.0)",
     "2.90×",
     "3.64",
     "88 ms",
     "514,618 (−27%)"
    ]
   ]
  },
  {
   "title": "Under load",
   "sub": "Closed loop: 4, 16, 48 requests in flight (12, 32, 96 requests). Total output tok/s (median per-request decode tok/s), change vs MTP off.",
   "columns": [
    "Configuration",
    "4 in flight",
    "16 in flight",
    "48 in flight"
   ],
   "highlight_rows": [
    3
   ],
   "rows": [
    [
     "MTP off",
     "277 (71.0)",
     "903 (60.1)",
     "1,742 (40.2)"
    ],
    [
     "1 draft token",
     "440 (119.3), +59%",
     "1,159 (86.1), +28%",
     "—"
    ],
    [
     "2 draft tokens",
     "513 (151.8), +85%",
     "1,294 (106.6), +43%",
     "—"
    ],
    [
     "3 draft tokens",
     "557 (175.4), +101%",
     "1,345 (115.8), +49%",
     "2,066 (58.0), +19%"
    ],
    [
     "4 draft tokens",
     "565 (176.0), +104%",
     "1,199 (107.2), +33%",
     "1,941 (57.6), +11%"
    ]
   ]
  },
  {
   "title": "By content",
   "sub": "Single-request decode tok/s, median per prompt.",
   "columns": [
    "Output",
    "MTP off",
    "1 draft token",
    "2 draft tokens",
    "3 draft tokens",
    "4 draft tokens"
   ],
   "rows": [
    [
     "Prose (~450-word explanation)",
     "74.8",
     "115.1",
     "132.9",
     "140.7",
     "140.6"
    ],
    [
     "Code (Python function + pytest)",
     "74.8",
     "131.0",
     "166.9",
     "198.4",
     "216.9"
    ],
    [
     "JSON (transform 8 orders)",
     "74.5",
     "133.7",
     "182.8",
     "220.3",
     "247.8"
    ],
    [
     "Reasoning text (thinking on, 1 request)",
     "74.2",
     "119.7",
     "140.4",
     "147.9",
     "161.4"
    ],
    [
     "Greedy (temperature 0), all prompts",
     "75.1",
     "131.6",
     "177.3",
     "203.0",
     "219.0"
    ]
   ]
  },
  {
   "title": "Acceptance",
   "sub": "Mean acceptance length (acceptance rate), from vLLM's /metrics spec-decode counters per phase.",
   "columns": [
    "Configuration",
    "1 request",
    "Greedy",
    "Thinking on",
    "16 in flight",
    "48 in flight"
   ],
   "rows": [
    [
     "1 draft token",
     "1.87 (0.87)",
     "1.89 (0.89)",
     "1.78 (0.79)",
     "1.87 (0.87)",
     "—"
    ],
    [
     "2 draft tokens",
     "2.55 (0.78)",
     "2.67 (0.83)",
     "2.28 (0.64)",
     "2.55 (0.77)",
     "—"
    ],
    [
     "3 draft tokens",
     "3.14 (0.71)",
     "3.28 (0.76)",
     "2.60 (0.53)",
     "3.09 (0.70)",
     "3.09 (0.70)"
    ],
    [
     "4 draft tokens",
     "3.64 (0.66)",
     "3.75 (0.69)",
     "3.12 (0.53)",
     "3.47 (0.62)",
     "3.52 (0.63)"
    ]
   ]
  },
  {
   "title": "Memory",
   "sub": "From the engine startup log: weights loaded, memory left for the KV cache, and the cache's capacity in tokens.",
   "columns": [
    "Configuration",
    "Weights",
    "KV cache memory",
    "Per cached token",
    "KV cache, tokens"
   ],
   "rows": [
    [
     "MTP off",
     "27.55 GiB",
     "53.91 GiB",
     "79.7 KiB",
     "709,527"
    ],
    [
     "1 draft token",
     "27.99 GiB",
     "53.47 GiB",
     "89.8 KiB",
     "624,482"
    ],
    [
     "2 draft tokens",
     "27.99 GiB",
     "51.68 GiB",
     "95.0 KiB",
     "570,163"
    ],
    [
     "3 draft tokens",
     "27.99 GiB",
     "51.74 GiB",
     "100.1 KiB",
     "541,801"
    ],
    [
     "4 draft tokens",
     "27.99 GiB",
     "51.72 GiB",
     "105.4 KiB",
     "514,618"
    ]
   ]
  },
  {
   "title": "Quality checks",
   "sub": "GSM8K: Bench's 5-shot suite, 200 questions, thinking off, temperature 0, 32 in flight, numeric scorer. Greedy outputs (temperature 0) with MTP were byte-identical across 1–4 draft tokens and from run to run, but not identical to MTP off; on the JSON prompt they differed at the first token, a near-tie decided before any draft exists.",
   "columns": [
    "Check",
    "MTP off",
    "3 draft tokens"
   ],
   "rows": [
    [
     "GSM8K-200",
     "98.0% (196/200)",
     "99.0% (198/200)"
    ],
    [
     "Greedy outputs",
     "reference",
     "not byte-identical"
    ],
    [
     "JSON prompt, first-token top 2 (logprob)",
     "``` −0.48, { −0.98",
     "{ −0.53, ``` −0.90"
    ]
   ]
  }
 ],
 "setup": [
  [
   "GPU",
   "1× NVIDIA H100 NVL 94 GB HBM3, 3.9 TB/s, PCIe; power limit 310 W on this host"
  ],
  [
   "Host",
   "RunPod Secure Cloud, US-GA-2, $3.19/h; 31.4 min, $1.67"
  ],
  [
   "Driver",
   "595.112.07 (CUDA 13.2)"
  ],
  [
   "Engine",
   "vllm/vllm-openai:v0.33.0, one engine start per configuration"
  ],
  [
   "Model",
   "Qwen/Qwen3.8-27B-FP8 @ c3a462cc3d38cf351f3b26cbd763469eeb74ef81; MTP head: 1 layer (mtp.safetensors, 0.48 GB)"
  ],
  [
   "Flags",
   "--max-model-len 32768 --tensor-parallel-size 1 --language-model-only --reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_coder --enable-prompt-tokens-details --max-num-seqs 256"
  ],
  [
   "MTP flag",
   "--speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":N}', N = 1, 2, 3, 4"
  ],
  [
   "Cache block",
   "832 tokens in every configuration (hybrid attention)"
  ],
  [
   "Requests",
   "chat completions, streamed (include_usage), max_tokens 512"
  ],
  [
   "Thinking",
   "off (chat_template_kwargs.enable_thinking=false) unless noted; thinking-on run: 1 request, 1,024 reasoning tokens"
  ],
  [
   "Sampling",
   "model defaults (temperature 1.0, top-p 0.95, top-k 20); greedy runs temperature 0 (3 prompts × 2)"
  ],
  [
   "Decode tok/s",
   "(completion tokens − 1) ÷ (last token chunk − first token chunk)"
  ],
  [
   "TTFT",
   "first token chunk − send"
  ],
  [
   "Load",
   "closed loop, 4 / 16 / 48 in flight (12 / 32 / 96 requests)"
  ]
 ],
 "not_covered": [
  "Our production H200 (4.8 TB/s, shared with a second model under CUDA MPS)",
  "Long prompts: inputs were about 100 to 1,000 tokens; decode with MTP at long context was not measured",
  "Full-context KV layout: 32,768-token max context and a 0.92 memory share, not 262,144 and a shared GPU",
  "48 in flight for 1 and 2 draft tokens; GSM8K for 1, 2 and 4 draft tokens",
  "FP8 KV cache with MTP",
  "Accuracy beyond GSM8K-200",
  "Repeat engine starts: one start per configuration"
 ],
 "reproduce_title": "Reproduce",
 "reproduce_sub": "Raw request timings, engine logs and the exact argv per configuration: bench/results/qwen38-mtp-2026-11/ in the TokenWatt repository (README.md, raw/nvl/<config>/).",
 "reproduce": "# baseline\nvllm serve Qwen/Qwen3.8-27B-FP8 --revision c3a462cc3d38cf351f3b26cbd763469eeb74ef81 \\\n  --served-model-name qwen3.8-27b --max-model-len 32768 --tensor-parallel-size 1 \\\n  --language-model-only --reasoning-parser qwen3 --enable-auto-tool-choice \\\n  --tool-call-parser qwen3_coder --enable-prompt-tokens-details --max-num-seqs 256\n\n# MTP, N = 1..4 (3 is vLLM's recipe value)\nvllm serve ... --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}'\n\n# keep the image's CUDA_HOME and PATH in the launching shell: without nvcc, FlashInfer's FP8\n# DeepGEMM JIT fails during CUDA-graph capture (\"Assertion failed: !cubin.empty() || isPathValid(path_)\")",
 "post": "/blog/qwen38-27b-mtp-speculative-decoding.html",
 "data": {
  "source": "bench/results/qwen38-mtp-2026-11/summary.json (TokenWatt repository)",
  "configs": {
   "MTP off": {
    "num_speculative_tokens": 0,
    "kv_tokens": 709527,
    "kv_cache_memory_gib": 53.91,
    "weights_gib": 27.55,
    "single": {
     "n": 9,
     "median_decode_tps": 74.77,
     "min_decode_tps": 74.34,
     "max_decode_tps": 75.3,
     "median_ttft_s": 0.075,
     "by_prompt_median_decode_tps": {
      "prose": 74.84,
      "code": 74.82,
      "json": 74.53
     }
    },
    "greedy": {
     "n": 6,
     "median_decode_tps": 75.095,
     "min_decode_tps": 74.64,
     "max_decode_tps": 75.35,
     "median_ttft_s": 0.079,
     "by_prompt_median_decode_tps": {
      "prose": 75.23,
      "code": 75.12,
      "json": 74.67
     }
    },
    "think": {
     "n": 1,
     "median_decode_tps": 74.22,
     "min_decode_tps": 74.22,
     "max_decode_tps": 74.22,
     "median_ttft_s": 0.083,
     "by_prompt_median_decode_tps": {
      "code": 74.22
     }
    },
    "conc4": {
     "n": 12,
     "median_decode_tps": 70.97,
     "min_decode_tps": 70.66,
     "max_decode_tps": 71.09,
     "median_ttft_s": 0.171,
     "aggregate_output_tps": 276.61,
     "by_prompt_median_decode_tps": {
      "prose": 70.98,
      "code": 70.96,
      "json": 70.895
     }
    },
    "conc16": {
     "n": 32,
     "median_decode_tps": 60.125,
     "min_decode_tps": 57.43,
     "max_decode_tps": 60.76,
     "median_ttft_s": 0.477,
     "aggregate_output_tps": 903.18,
     "by_prompt_median_decode_tps": {
      "prose": 60.11,
      "code": 60.23,
      "json": 60.095
     }
    },
    "conc48": {
     "n": 96,
     "median_decode_tps": 40.195,
     "min_decode_tps": 36.63,
     "max_decode_tps": 40.82,
     "median_ttft_s": 1.277,
     "aggregate_output_tps": 1742.11,
     "by_prompt_median_decode_tps": {
      "prose": 40.19,
      "code": 40.205,
      "json": 40.19
     }
    },
    "gsm8k_200": {
     "n": 200,
     "correct": 196,
     "accuracy": 0.98
    },
    "first_token_top5_temperature0": {
     "prose": [
      [
       "Imagine",
       -0.0542
      ],
      [
       "At",
       -4.0542
      ],
      [
       "Think",
       -4.4292
      ],
      [
       "A",
       -4.8042
      ],
      [
       "To",
       -5.5542
      ]
     ],
     "code": [
      [
       "```",
       -0.0095
      ],
      [
       "from",
       -5.2595
      ],
      [
       "\"\"\"",
       -6.2595
      ],
      [
       "#",
       -7.0095
      ],
      [
       "import",
       -7.5095
      ]
     ],
     "json": [
      [
       "```",
       -0.4814
      ],
      [
       "{",
       -0.9814
      ],
      [
       "Let",
       -6.2314
      ],
      [
       "To",
       -6.2314
      ],
      [
       "I",
       -6.2314
      ]
     ]
    }
   },
   "1 draft token": {
    "num_speculative_tokens": 1,
    "kv_tokens": 624482,
    "kv_cache_memory_gib": 53.47,
    "weights_gib": 27.99,
    "single": {
     "n": 9,
     "median_decode_tps": 130.97,
     "min_decode_tps": 114.04,
     "max_decode_tps": 133.78,
     "median_ttft_s": 0.077,
     "by_prompt_median_decode_tps": {
      "prose": 115.07,
      "code": 130.97,
      "json": 133.65
     },
     "acceptance_rate": 0.8692,
     "mean_acceptance_length": 1.869,
     "accepted_per_position": {
      "0": 0.8692
     }
    },
    "greedy": {
     "n": 6,
     "median_decode_tps": 131.56,
     "min_decode_tps": 120.13,
     "max_decode_tps": 133.62,
     "median_ttft_s": 0.08,
     "by_prompt_median_decode_tps": {
      "prose": 120.345,
      "code": 131.56,
      "json": 133.62
     },
     "acceptance_rate": 0.8927,
     "mean_acceptance_length": 1.893,
     "accepted_per_position": {
      "0": 0.8927
     }
    },
    "think": {
     "n": 1,
     "median_decode_tps": 119.68,
     "min_decode_tps": 119.68,
     "max_decode_tps": 119.68,
     "median_ttft_s": 0.075,
     "by_prompt_median_decode_tps": {
      "code": 119.68
     },
     "acceptance_rate": 0.7853,
     "mean_acceptance_length": 1.785,
     "accepted_per_position": {
      "0": 0.7853
     }
    },
    "conc4": {
     "n": 12,
     "median_decode_tps": 119.26,
     "min_decode_tps": 103.92,
     "max_decode_tps": 125.27,
     "median_ttft_s": 0.131,
     "aggregate_output_tps": 439.65,
     "by_prompt_median_decode_tps": {
      "prose": 106.21,
      "code": 119.26,
      "json": 123.555
     },
     "acceptance_rate": 0.8794,
     "mean_acceptance_length": 1.879,
     "accepted_per_position": {
      "0": 0.8794
     }
    },
    "conc16": {
     "n": 32,
     "median_decode_tps": 86.095,
     "min_decode_tps": 71.93,
     "max_decode_tps": 93.53,
     "median_ttft_s": 0.397,
     "aggregate_output_tps": 1159.14,
     "by_prompt_median_decode_tps": {
      "prose": 76.75,
      "code": 86.12,
      "json": 90.29
     },
     "acceptance_rate": 0.8712,
     "mean_acceptance_length": 1.871,
     "accepted_per_position": {
      "0": 0.8712
     }
    }
   },
   "2 draft tokens": {
    "num_speculative_tokens": 2,
    "kv_tokens": 570163,
    "kv_cache_memory_gib": 51.68,
    "weights_gib": 27.99,
    "single": {
     "n": 9,
     "median_decode_tps": 166.93,
     "min_decode_tps": 130.32,
     "max_decode_tps": 183.09,
     "median_ttft_s": 0.081,
     "by_prompt_median_decode_tps": {
      "prose": 132.93,
      "code": 166.93,
      "json": 182.75
     },
     "acceptance_rate": 0.7762,
     "mean_acceptance_length": 2.552,
     "accepted_per_position": {
      "0": 0.8432,
      "1": 0.7091
     }
    },
    "greedy": {
     "n": 6,
     "median_decode_tps": 177.255,
     "min_decode_tps": 143.6,
     "max_decode_tps": 181.62,
     "median_ttft_s": 0.078,
     "by_prompt_median_decode_tps": {
      "prose": 143.955,
      "code": 177.255,
      "json": 181.395
     },
     "acceptance_rate": 0.8325,
     "mean_acceptance_length": 2.665,
     "accepted_per_position": {
      "0": 0.8854,
      "1": 0.7795
     }
    },
    "think": {
     "n": 1,
     "median_decode_tps": 140.36,
     "min_decode_tps": 140.36,
     "max_decode_tps": 140.36,
     "median_ttft_s": 0.082,
     "by_prompt_median_decode_tps": {
      "code": 140.36
     },
     "acceptance_rate": 0.6417,
     "mean_acceptance_length": 2.283,
     "accepted_per_position": {
      "0": 0.7411,
      "1": 0.5424
     }
    },
    "conc4": {
     "n": 12,
     "median_decode_tps": 151.78,
     "min_decode_tps": 116.64,
     "max_decode_tps": 169.53,
     "median_ttft_s": 0.121,
     "aggregate_output_tps": 512.55,
     "by_prompt_median_decode_tps": {
      "prose": 119.13,
      "code": 151.78,
      "json": 165.17
     },
     "acceptance_rate": 0.7853,
     "mean_acceptance_length": 2.571,
     "accepted_per_position": {
      "0": 0.8539,
      "1": 0.7166
     }
    },
    "conc16": {
     "n": 32,
     "median_decode_tps": 106.56,
     "min_decode_tps": 76.01,
     "max_decode_tps": 123.52,
     "median_ttft_s": 0.381,
     "aggregate_output_tps": 1294.43,
     "by_prompt_median_decode_tps": {
      "prose": 83.11,
      "code": 106.59,
      "json": 117.74
     },
     "acceptance_rate": 0.7743,
     "mean_acceptance_length": 2.549,
     "accepted_per_position": {
      "0": 0.8405,
      "1": 0.7081
     }
    }
   },
   "3 draft tokens": {
    "num_speculative_tokens": 3,
    "kv_tokens": 541801,
    "kv_cache_memory_gib": 51.74,
    "weights_gib": 27.99,
    "single": {
     "n": 9,
     "median_decode_tps": 198.39,
     "min_decode_tps": 138.28,
     "max_decode_tps": 222.34,
     "median_ttft_s": 0.08,
     "by_prompt_median_decode_tps": {
      "prose": 140.69,
      "code": 198.39,
      "json": 220.26
     },
     "acceptance_rate": 0.7127,
     "mean_acceptance_length": 3.138,
     "accepted_per_position": {
      "0": 0.8442,
      "1": 0.7061,
      "2": 0.5878
     }
    },
    "greedy": {
     "n": 6,
     "median_decode_tps": 203.03,
     "min_decode_tps": 152.48,
     "max_decode_tps": 220.41,
     "median_ttft_s": 0.077,
     "by_prompt_median_decode_tps": {
      "prose": 152.75,
      "code": 203.03,
      "json": 219.795
     },
     "acceptance_rate": 0.7591,
     "mean_acceptance_length": 3.277,
     "accepted_per_position": {
      "0": 0.8678,
      "1": 0.7591,
      "2": 0.6503
     }
    },
    "think": {
     "n": 1,
     "median_decode_tps": 147.92,
     "min_decode_tps": 147.92,
     "max_decode_tps": 147.92,
     "median_ttft_s": 0.082,
     "by_prompt_median_decode_tps": {
      "code": 147.92
     },
     "acceptance_rate": 0.5344,
     "mean_acceptance_length": 2.603,
     "accepted_per_position": {
      "0": 0.7048,
      "1": 0.5242,
      "2": 0.374
     }
    },
    "conc4": {
     "n": 12,
     "median_decode_tps": 175.405,
     "min_decode_tps": 120.93,
     "max_decode_tps": 208.04,
     "median_ttft_s": 0.126,
     "aggregate_output_tps": 556.83,
     "by_prompt_median_decode_tps": {
      "prose": 123.26,
      "code": 175.405,
      "json": 199.45
     },
     "acceptance_rate": 0.7138,
     "mean_acceptance_length": 3.141,
     "accepted_per_position": {
      "0": 0.8514,
      "1": 0.7058,
      "2": 0.5843
     }
    },
    "conc16": {
     "n": 32,
     "median_decode_tps": 115.805,
     "min_decode_tps": 77.23,
     "max_decode_tps": 146.76,
     "median_ttft_s": 0.371,
     "aggregate_output_tps": 1345.16,
     "by_prompt_median_decode_tps": {
      "prose": 85.01,
      "code": 115.82,
      "json": 135.55
     },
     "acceptance_rate": 0.6959,
     "mean_acceptance_length": 3.088,
     "accepted_per_position": {
      "0": 0.8357,
      "1": 0.6879,
      "2": 0.5641
     }
    },
    "conc48": {
     "n": 96,
     "median_decode_tps": 57.965,
     "min_decode_tps": 35.34,
     "max_decode_tps": 84.33,
     "median_ttft_s": 0.526,
     "aggregate_output_tps": 2065.54,
     "by_prompt_median_decode_tps": {
      "prose": 39.905,
      "code": 56.72,
      "json": 64.12
     },
     "acceptance_rate": 0.6971,
     "mean_acceptance_length": 3.091,
     "accepted_per_position": {
      "0": 0.8303,
      "1": 0.6916,
      "2": 0.5694
     }
    },
    "gsm8k_200": {
     "n": 200,
     "correct": 198,
     "accuracy": 0.99,
     "acceptance_rate": 0.8631,
     "mean_acceptance_length": 3.589
    },
    "first_token_top5_temperature0": {
     "prose": [
      [
       "Imagine",
       -0.0505
      ],
      [
       "At",
       -3.9255
      ],
      [
       "Think",
       -4.8005
      ],
      [
       "A",
       -5.0505
      ],
      [
       "Inside",
       -5.5505
      ]
     ],
     "code": [
      [
       "```",
       -0.0144
      ],
      [
       "from",
       -4.8894
      ],
      [
       "\"\"\"",
       -5.5144
      ],
      [
       "def",
       -6.8894
      ],
      [
       "#",
       -6.8894
      ]
     ],
     "json": [
      [
       "{",
       -0.5259
      ],
      [
       "```",
       -0.9009
      ],
      [
       "I",
       -7.0259
      ],
      [
       "Let",
       -7.7759
      ],
      [
       "[",
       -7.9009
      ]
     ]
    }
   },
   "4 draft tokens": {
    "num_speculative_tokens": 4,
    "kv_tokens": 514618,
    "kv_cache_memory_gib": 51.72,
    "weights_gib": 27.99,
    "single": {
     "n": 9,
     "median_decode_tps": 216.94,
     "min_decode_tps": 136.49,
     "max_decode_tps": 251.98,
     "median_ttft_s": 0.088,
     "by_prompt_median_decode_tps": {
      "prose": 140.58,
      "code": 216.94,
      "json": 247.76
     },
     "acceptance_rate": 0.6594,
     "mean_acceptance_length": 3.637,
     "accepted_per_position": {
      "0": 0.8491,
      "1": 0.7178,
      "2": 0.5807,
      "3": 0.4899
     }
    },
    "greedy": {
     "n": 6,
     "median_decode_tps": 219.03,
     "min_decode_tps": 148.33,
     "max_decode_tps": 247.79,
     "median_ttft_s": 0.087,
     "by_prompt_median_decode_tps": {
      "prose": 148.7,
      "code": 219.03,
      "json": 247.42
     },
     "acceptance_rate": 0.6883,
     "mean_acceptance_length": 3.753,
     "accepted_per_position": {
      "0": 0.8753,
      "1": 0.7457,
      "2": 0.621,
      "3": 0.511
     }
    },
    "think": {
     "n": 1,
     "median_decode_tps": 161.39,
     "min_decode_tps": 161.39,
     "max_decode_tps": 161.39,
     "median_ttft_s": 0.096,
     "by_prompt_median_decode_tps": {
      "code": 161.39
     },
     "acceptance_rate": 0.5297,
     "mean_acceptance_length": 3.119,
     "accepted_per_position": {
      "0": 0.747,
      "1": 0.561,
      "2": 0.4421,
      "3": 0.3689
     }
    },
    "conc4": {
     "n": 12,
     "median_decode_tps": 175.965,
     "min_decode_tps": 106.81,
     "max_decode_tps": 215.61,
     "median_ttft_s": 0.13,
     "aggregate_output_tps": 565.41,
     "by_prompt_median_decode_tps": {
      "prose": 115.39,
      "code": 175.965,
      "json": 214.2
     },
     "acceptance_rate": 0.6342,
     "mean_acceptance_length": 3.537,
     "accepted_per_position": {
      "0": 0.8259,
      "1": 0.6828,
      "2": 0.5603,
      "3": 0.4678
     }
    },
    "conc16": {
     "n": 32,
     "median_decode_tps": 107.155,
     "min_decode_tps": 62.43,
     "max_decode_tps": 169.54,
     "median_ttft_s": 0.29,
     "aggregate_output_tps": 1198.92,
     "by_prompt_median_decode_tps": {
      "prose": 73.16,
      "code": 107.53,
      "json": 130.18
     },
     "acceptance_rate": 0.617,
     "mean_acceptance_length": 3.468,
     "accepted_per_position": {
      "0": 0.8155,
      "1": 0.6719,
      "2": 0.5376,
      "3": 0.443
     }
    },
    "conc48": {
     "n": 96,
     "median_decode_tps": 57.56,
     "min_decode_tps": 32.16,
     "max_decode_tps": 90.17,
     "median_ttft_s": 0.623,
     "aggregate_output_tps": 1940.6,
     "by_prompt_median_decode_tps": {
      "prose": 36.835,
      "code": 55.705,
      "json": 67.155
     },
     "acceptance_rate": 0.6312,
     "mean_acceptance_length": 3.525,
     "accepted_per_position": {
      "0": 0.825,
      "1": 0.6822,
      "2": 0.5545,
      "3": 0.4632
     }
    },
    "first_token_top5_temperature0": {
     "prose": [
      [
       "Imagine",
       -0.0505
      ],
      [
       "At",
       -3.9255
      ],
      [
       "Think",
       -4.8005
      ],
      [
       "A",
       -5.0505
      ],
      [
       "Inside",
       -5.5505
      ]
     ],
     "code": [
      [
       "```",
       -0.0144
      ],
      [
       "from",
       -4.8894
      ],
      [
       "\"\"\"",
       -5.5144
      ],
      [
       "def",
       -6.8894
      ],
      [
       "#",
       -6.8894
      ]
     ],
     "json": [
      [
       "{",
       -0.5259
      ],
      [
       "```",
       -0.9009
      ],
      [
       "I",
       -7.0259
      ],
      [
       "Let",
       -7.7759
      ],
      [
       "[",
       -7.9009
      ]
     ]
    }
   }
  }
 }
}
