{
 "id": "h100-qwen38-27b-prefix-cache-4k-vllm",
 "status": "published",
 "title": "Prefix caching on a hybrid model, one H100: Qwen3.8-27B FP8, vLLM 0.31.0, 4,000-token prompts",
 "published": "2026-11-13",
 "summary": "Quick-mode measurement on one H100 SXM on RunPod Secure Cloud (November 5, 2026): Qwen3.8-27B FP8, a hybrid model (48 Gated DeltaNet and 16 full-attention layers), on vLLM 0.31.0 with default automatic prefix caching (mamba cache mode 'align', 784-token cache blocks with the default BF16 KV cache), --max-num-seqs 512, text-only, 4,000-token prompts (outputs median 256 tokens). The same SLO search ran four times with 0%, 50%, 75% and 90% of every prompt drawn from eight shared prefixes; cache hit rates are vLLM's own prefix-cache counters (0.0%, 37.0%, 58.6%, 74.2%). Capacity within p99 TTFT \u2264 2 s and p99 TPOT \u2264 50 ms rose from 1.41 to 4.77 requests per second. No accuracy gate (random-token prompts); the model answered a smoke-test question correctly at each start.",
 "hardware": {
  "vendor": "NVIDIA",
  "accelerator": "NVIDIA H100 80GB HBM3",
  "count": 1,
  "rated_bw_tbs_per_gpu": 3.35,
  "driver": "595.91.07",
  "host": "RunPod Secure Cloud 1x H100 SXM 80GB (US-MO-1, pod h4dvtjc8j7mu7e)"
 },
 "software": {
  "engine": "vllm",
  "engine_version": "0.31.0",
  "kernels": "vllm/vllm-openai:v0.31.0 --max-model-len 32768 --max-num-seqs 512 --language-model-only --enable-prompt-tokens-details"
 },
 "model": {
  "name": "Qwen3.8-27B-FP8",
  "shape": "",
  "quantization": "FP8 weights, BF16 KV cache",
  "parallelism": "1 replica \u00d7 TP1"
 },
 "slo": {
  "ttft_p99_ms": 2000,
  "tpot_p99_ms": 50
 },
 "workload": {
  "trace": "in4k",
  "input_tokens": "fixed 4,000",
  "output_tokens": "median 256 \u00b7 p90 1,024",
  "arrival": "Poisson, rate swept to SLO boundary"
 },
 "power": {
  "source": "GPU board power sum (NVIDIA)",
  "kind": "board",
  "sampling_hz": 1,
  "window": "120 s steady state"
 },
 "accuracy": {
  "suite": "none",
  "threshold_pts": 0.5
 },
 "runs": 2,
 "variance_band_pct": 4.3,
 "fleet": {
  "cards": 64
 },
 "configs": [
  {
   "key": "other",
   "label": "No prompt reuse (0% cache hits)",
   "engine": "vllm 0.31.0 \u00b7 KV BF16",
   "mbu_pct": null,
   "node_power_kw": 0.58,
   "goodput_tok_s": 616.2,
   "ttft_p99_ms": 1089.0,
   "tpot_p99_ms": 42.0,
   "slo_pass_count": 2,
   "repeats": 2,
   "throttle_counts": {
    "sw_power_cap": 141
   },
   "accuracy": null
  },
  {
   "key": "other",
   "label": "50% shared prefix (37.0% cache hits)",
   "engine": "vllm 0.31.0 \u00b7 KV BF16",
   "mbu_pct": null,
   "node_power_kw": 0.619,
   "goodput_tok_s": 1024.8,
   "ttft_p99_ms": 797.0,
   "tpot_p99_ms": 42.9,
   "slo_pass_count": 2,
   "repeats": 2,
   "throttle_counts": {
    "sw_power_cap": 167
   },
   "accuracy": null
  },
  {
   "key": "other",
   "label": "75% shared prefix (58.6% cache hits)",
   "engine": "vllm 0.31.0 \u00b7 KV BF16",
   "mbu_pct": null,
   "node_power_kw": 0.634,
   "goodput_tok_s": 1411.8,
   "ttft_p99_ms": 602.0,
   "tpot_p99_ms": 45.4,
   "slo_pass_count": 2,
   "repeats": 2,
   "throttle_counts": {
    "sw_power_cap": 178
   },
   "accuracy": null
  },
  {
   "key": "other",
   "label": "90% shared prefix (74.2% cache hits)",
   "engine": "vllm 0.31.0 \u00b7 KV BF16",
   "mbu_pct": null,
   "node_power_kw": 0.66,
   "goodput_tok_s": 1969.8,
   "ttft_p99_ms": 410.0,
   "tpot_p99_ms": 44.7,
   "slo_pass_count": 2,
   "repeats": 2,
   "throttle_counts": {
    "sw_power_cap": 205
   },
   "accuracy": null
  }
 ],
 "not_covered": [
  "Whole-node power (no BMC access on this host)",
  "Quick mode: short windows, few repeats",
  "Synthetic shared prefixes (8-prefix pool, random tokens)",
  "Accuracy: no accuracy gate was run"
 ],
 "reproduce": "tokenwatt run --spec h100-hyb-bf16-in4k-prefix-cache.spec.yaml --label 'hyb-bf16-in4k-0' --role other --out runs/hyb-bf16-in4k-0.json\ntokenwatt run --spec h100-hyb-bf16-in4k-prefix-cache.spec.yaml --label 'hyb-bf16-in4k-0.5' --role other --out runs/hyb-bf16-in4k-0-5.json\ntokenwatt run --spec h100-hyb-bf16-in4k-prefix-cache.spec.yaml --label 'hyb-bf16-in4k-0.75' --role other --out runs/hyb-bf16-in4k-0-75.json\ntokenwatt run --spec h100-hyb-bf16-in4k-prefix-cache.spec.yaml --label 'hyb-bf16-in4k-0.9' --role other --out runs/hyb-bf16-in4k-0-9.json\n\ntokenwatt report runs/hyb-bf16-in4k-0.json runs/hyb-bf16-in4k-0-5.json runs/hyb-bf16-in4k-0-75.json runs/hyb-bf16-in4k-0-9.json --title 'Prefix caching on a hybrid model, one H100: Qwen3.8-27B FP8, vLLM 0.31.0, 4,000-token prompts'",
 "spec": {
  "name": "h100-hyb-bf16-in4k-prefix-cache",
  "hardware": {
   "adapter": "nvidia",
   "rated_bw_tbs_per_gpu": null,
   "gpus": null,
   "host": "RunPod Secure Cloud 1x H100 SXM 80GB (US-MO-1, pod h4dvtjc8j7mu7e)"
  },
  "engine": {
   "endpoint": "http://127.0.0.1:8000/v1",
   "name": "vllm",
   "version": "0.31.0",
   "kernels": "vllm/vllm-openai:v0.31.0 --max-model-len 32768 --max-num-seqs 512 --language-model-only --enable-prompt-tokens-details",
   "api": "completions",
   "model": "qwen3.8-27b",
   "ignore_eos": true,
   "api_key_env": null,
   "extra_body": {}
  },
  "model": {
   "name": "Qwen3.8-27B-FP8",
   "config": null,
   "weight_dtype": "fp8",
   "kv_dtype": "bf16",
   "quantization": "",
   "replicas": 1,
   "tensor_parallel": 1,
   "vocab_size": null,
   "geometry": {}
  },
  "slo": {
   "ttft_p99_ms": 2000,
   "tpot_p99_ms": 50
  },
  "workload": {
   "trace": "in4k",
   "input_tokens": {
    "dist": "fixed",
    "median": 4000,
    "p90": null,
    "value": 4000,
    "low": null,
    "high": null,
    "min": 4000,
    "max": 4000
   },
   "output_tokens": {
    "dist": "lognormal",
    "median": 256,
    "p90": 1024,
    "value": null,
    "low": null,
    "high": null,
    "min": 8,
    "max": 2048
   },
   "seed": 0,
   "shared_prefix": 0.0,
   "prefix_pool": 8
  },
  "measurement": {
   "warmup_s": 45,
   "window_s": 120,
   "search_window_s": 60,
   "repeats": 2,
   "request_timeout_s": 300,
   "max_concurrency": 8192
  },
  "sweep": {
   "rates": null,
   "start_rps": 0.4,
   "growth": 1.6,
   "max_rps": 2000,
   "bisect_steps": 3,
   "backoff": 0.9,
   "max_backoffs": 4,
   "confirm": "backoff",
   "floor_rps": null,
   "confirm_steps": 5
  },
  "accuracy": {
   "suite": null,
   "name": "",
   "scorer": "numeric",
   "max_drop_pts": 0.5,
   "seeds": 3,
   "max_tokens": 512,
   "stop": null,
   "concurrency": 32,
   "limit": null
  },
  "power": {
   "source": "board",
   "sampling_hz": 1,
   "url": null,
   "username": null,
   "password_env": null,
   "verify_tls": false,
   "chassis": null,
   "host": null,
   "command": null,
   "field": null
  },
  "not_covered": [
   "Whole-node power (no BMC access on this host)",
   "Quick mode: short windows, few repeats",
   "Synthetic shared prefixes (8-prefix pool, random tokens)"
  ]
 }
}