{
 "id": "h200-qwen38-qwen36-prefix-cache-real-agent-traffic",
 "kind": "measurement",
 "status": "published",
 "title": "Prefix-cache hits on real coding-agent traffic: Qwen3.8-27B and Qwen3.6-35B-A3B on one H200, vLLM 0.33.0",
 "published": "2026-11-20",
 "summary": "Read from our API gateway's request log (metadata only: model, prompt tokens, cached prompt tokens) on 2026-11-20 at about 17:50 UTC, covering 2026-11-19 and 2026-11-20. Requests on our own self-serve API keys, in practice one person on our team running a coding agent; prompts under 1,000 tokens left out; 3,149 requests. Both models are hybrid attention (Gated DeltaNet + full attention) with FP8 weights, served by two vLLM 0.33.0 engines sharing one H200 under CUDA MPS, prefix caching on, BF16 KV cache, 262,144-token context; Qwen3.8-27B's cache block on this node is 832 tokens. Hit rate = cached prompt tokens ÷ prompt tokens, from vLLM's per-request prompt_tokens_details.cached_tokens. Write-up: /blog/prefix-cache-real-agent-traffic-hybrid-qwen.html",
 "conclusion": "vLLM served 94.3% of Qwen3.8-27B's prompt tokens and 94.8% of Qwen3.6-35B-A3B's from its prefix cache, on median prompts of 114,258 and 85,817 tokens. The median request recomputed 2,690 and 1,562 tokens. On a synthetic test with ~1,000-token prompts, 90% shared, Qwen3.8-27B got 55.6%: the whole-block rule costs about one block per request, a large share of a short prompt and under 1% of a long one.",
 "headline": {
  "value": "94.3%",
  "label": "of Qwen3.8-27B prompt tokens from cache, real agent traffic"
 },
 "hardware": {
  "vendor": "NVIDIA",
  "accelerator": "NVIDIA H200",
  "count": 1,
  "rated_bw_tbs_per_gpu": 4.8,
  "driver": "",
  "host": "RunPod Secure Cloud 1x H200 SXM 141 GB (our API's server)"
 },
 "software": {
  "engine": "vLLM",
  "engine_version": "0.33.0",
  "kernels": "vllm/vllm-openai:v0.33.0, two engines under CUDA MPS, --enable-prompt-tokens-details"
 },
 "model": {
  "name": "Qwen3.8-27B-FP8 + Qwen3.6-35B-A3B-FP8",
  "shape": "",
  "quantization": "FP8 weights, BF16 KV cache",
  "parallelism": "1 replica each × TP1, one shared GPU"
 },
 "pills": [
  "Real traffic · Nov 19–20, 2026"
 ],
 "variables_measured": [
  "prefix-cache hit rate (token-weighted, %)",
  "prompt tokens per request",
  "uncached prompt tokens per request",
  "requests with zero cache hits"
 ],
 "charts_sub": "Token-weighted: cached prompt tokens ÷ all prompt tokens in each group.",
 "charts": [
  {
   "title": "Qwen3.8-27B: prompt tokens from cache",
   "unit": "% of prompt tokens, by prompt length",
   "digits": 1,
   "suffix": "%",
   "max": 100,
   "better": "higher is better",
   "note": "Axis runs to 100%. n = requests in each group.",
   "bars": [
    {
     "label": "1K–8K (n 12)",
     "key": "base",
     "value": 9.6
    },
    {
     "label": "8K–32K (n 44)",
     "key": "base",
     "value": 42.7
    },
    {
     "label": "32K–64K (n 147)",
     "key": "base",
     "value": 92.7
    },
    {
     "label": "64K+ (n 810)",
     "key": "base",
     "value": 94.9
    },
    {
     "label": "All (n 1,013)",
     "key": "tuned",
     "value": 94.3
    }
   ]
  },
  {
   "title": "Qwen3.6-35B-A3B: prompt tokens from cache",
   "unit": "% of prompt tokens, by prompt length",
   "digits": 1,
   "suffix": "%",
   "max": 100,
   "better": "higher is better",
   "note": "Axis runs to 100%. n = requests in each group.",
   "bars": [
    {
     "label": "1K–8K (n 12)",
     "key": "base",
     "value": 7.6
    },
    {
     "label": "8K–32K (n 203)",
     "key": "base",
     "value": 86.0
    },
    {
     "label": "32K–64K (n 348)",
     "key": "base",
     "value": 87.8
    },
    {
     "label": "64K+ (n 1,573)",
     "key": "base",
     "value": 95.8
    },
    {
     "label": "All (n 2,136)",
     "key": "tuned",
     "value": 94.8
    }
   ]
  }
 ],
 "tables": [
  {
   "title": "All numbers",
   "sub": "Prompts of 1,000 tokens or more. p10 and p90 are the 10th and 90th percentiles.",
   "columns": [
    "Metric",
    "Qwen3.8-27B",
    "Qwen3.6-35B-A3B"
   ],
   "highlight_rows": [
    2
   ],
   "rows": [
    [
     "Requests",
     "1,013",
     "2,136"
    ],
    [
     "Prompt tokens, median (p10–p90)",
     "114,258 (40,706–153,171)",
     "85,817 (31,888–121,662)"
    ],
    [
     "Prompt tokens from cache (token-weighted)",
     "94.3%",
     "94.8%"
    ],
    [
     "Hit rate per request, median (p10)",
     "97.6% (89.9%)",
     "98.0% (87.3%)"
    ],
    [
     "Uncached tokens per request, median (p90)",
     "2,690 (6,356)",
     "1,562 (5,830)"
    ],
    [
     "Requests with zero cache hits",
     "44",
     "32"
    ],
    [
     "Cache block size",
     "832 tokens",
     "not recorded"
    ]
   ]
  },
  {
   "title": "By prompt length",
   "sub": "Requests (n) and token-weighted hit rate per group. Qwen3.8 requests of 8K–32K tokens recomputed a median of 11,175 tokens each.",
   "columns": [
    "Prompt length",
    "Qwen3.8 n",
    "Qwen3.8 from cache",
    "Qwen3.6 n",
    "Qwen3.6 from cache"
   ],
   "rows": [
    [
     "1K–8K tokens",
     "12",
     "9.6%",
     "12",
     "7.6%"
    ],
    [
     "8K–32K",
     "44",
     "42.7%",
     "203",
     "86.0%"
    ],
    [
     "32K–64K",
     "147",
     "92.7%",
     "348",
     "87.8%"
    ],
    [
     "64K and more",
     "810",
     "94.9%",
     "1,573",
     "95.8%"
    ]
   ]
  },
  {
   "title": "Context: the synthetic test",
   "text": true,
   "sub": "From the 2026-11-16 post (/blog/prefix-cache-hybrid-qwen38-2026.html); different GPU, engine version and workload.",
   "columns": [
    "",
    "Synthetic test (2026-11-16 post)",
    "Real coding-agent traffic (this report)"
   ],
   "rows": [
    [
     "GPU and engine",
     "1× H100 SXM, vLLM 0.31.0",
     "1× H200, vLLM 0.33.0"
    ],
    [
     "Qwen3.8-27B cache block",
     "784 tokens",
     "832 tokens"
    ],
    [
     "Prompts",
     "median 1,024 tokens, 90% from 8 shared prefixes",
     "median 114,258 tokens, growing conversations"
    ],
    [
     "Qwen3.8-27B prompt tokens from cache",
     "55.6%",
     "94.3%"
    ],
    [
     "Dense-attention model, same workload",
     "89.2% (Qwen3-30B-A3B)",
     "not measured"
    ]
   ]
  }
 ],
 "setup": [
  [
   "GPU",
   "1× NVIDIA H200 SXM 141 GB, RunPod Secure Cloud (our API's server)"
  ],
  [
   "Engines",
   "vLLM 0.33.0 (vllm/vllm-openai:v0.33.0), two engines on one GPU under CUDA MPS"
  ],
  [
   "Models",
   "Qwen/Qwen3.8-27B-FP8 (about half of GPU memory), Qwen/Qwen3.6-35B-A3B-FP8 (most of the rest)"
  ],
  [
   "Architecture",
   "Both hybrid attention: Gated DeltaNet + full attention (Qwen3.8: 16 of 64 layers full attention)"
  ],
  [
   "Prefix caching",
   "on (vLLM default), BF16 KV cache, 262,144-token context"
  ],
  [
   "Cache block",
   "Qwen3.8-27B: 832 tokens (vLLM startup log); Qwen3.6: not recorded"
  ],
  [
   "Traffic",
   "Our own self-serve API keys; in practice one person on our team running a coding agent"
  ],
  [
   "Window",
   "2026-11-19 to 2026-11-20, read 2026-11-20 ~17:50 UTC"
  ],
  [
   "Filter",
   "Prompts of 1,000 tokens or more; other traffic sources left out"
  ],
  [
   "Source",
   "API gateway request log: model, prompt tokens, cached prompt tokens (no prompt or completion text)"
  ],
  [
   "Hit rate",
   "Σ cached prompt tokens ÷ Σ prompt tokens (token-weighted), per model and per length group"
  ],
  [
   "Per-request rate",
   "cached ÷ prompt tokens of each request; median and p10 across requests"
  ]
 ],
 "not_covered": [
  "Other users, other coding agents or other kinds of traffic: one person's sessions over about two days",
  "Session-level behaviour: the request log has no session ids",
  "High load: hit rates were measured on a lightly loaded node; at high load vLLM evicts cached blocks sooner",
  "A dense-attention model on the same traffic",
  "FP8 KV cache (larger cache blocks) on real traffic",
  "Qwen3.6-35B-A3B's cache block size"
 ],
 "reproduce_title": "How to measure it on your own traffic",
 "reproduce_sub": "The request log itself isn't published: it is our team's usage. This is how the same numbers come out of any vLLM deployment.",
 "reproduce": "# start vLLM so that usage reports cached prompt tokens\nvllm serve Qwen/Qwen3.8-27B-FP8 --enable-prompt-tokens-details ...   # prefix caching is on by default\n\n# every response's usage then carries:\n#   usage.prompt_tokens\n#   usage.prompt_tokens_details.cached_tokens\n# log both per request, then:\n#   token-weighted hit rate = sum(cached_tokens) / sum(prompt_tokens)\n\n# the cache block size is in the engine's startup log:\n#   \"Setting attention block size to 832 tokens to ensure that attention page size is >= mamba page size.\"",
 "post": "/blog/prefix-cache-real-agent-traffic-hybrid-qwen.html",
 "data": {
  "source": "API gateway request log, metadata only; channel: our own self-serve keys (one internal user)",
  "window": {
   "from": "2026-11-19",
   "to": "2026-11-20",
   "read_at": "2026-11-20T17:50Z"
  },
  "min_prompt_tokens": 1000,
  "models": {
   "qwen3.8-27b": {
    "requests": 1013,
    "prompt_tokens_median": 114258,
    "prompt_tokens_p10": 40706,
    "prompt_tokens_p90": 153171,
    "token_weighted_hit_pct": 94.3,
    "per_request_hit_median_pct": 97.6,
    "per_request_hit_p10_pct": 89.9,
    "uncached_tokens_median": 2690,
    "uncached_tokens_p90": 6356,
    "zero_hit_requests": 44,
    "cache_block_tokens": 832,
    "by_prompt_length": {
     "1k-8k": {
      "n": 12,
      "hit_pct": 9.6
     },
     "8k-32k": {
      "n": 44,
      "hit_pct": 42.7,
      "uncached_tokens_median": 11175
     },
     "32k-64k": {
      "n": 147,
      "hit_pct": 92.7
     },
     "64k+": {
      "n": 810,
      "hit_pct": 94.9
     }
    }
   },
   "qwen3.6-35b-a3b": {
    "requests": 2136,
    "prompt_tokens_median": 85817,
    "prompt_tokens_p10": 31888,
    "prompt_tokens_p90": 121662,
    "token_weighted_hit_pct": 94.8,
    "per_request_hit_median_pct": 98.0,
    "per_request_hit_p10_pct": 87.3,
    "uncached_tokens_median": 1562,
    "uncached_tokens_p90": 5830,
    "zero_hit_requests": 32,
    "cache_block_tokens": null,
    "by_prompt_length": {
     "1k-8k": {
      "n": 12,
      "hit_pct": 7.6
     },
     "8k-32k": {
      "n": 203,
      "hit_pct": 86.0
     },
     "32k-64k": {
      "n": 348,
      "hit_pct": 87.8
     },
     "64k+": {
      "n": 1573,
      "hit_pct": 95.8
     }
    }
   }
  }
 }
}
