{
  "id": "h200x4-deepseek-v4.1-flash-fp8-vllm",
  "status": "published",
  "title": "DeepSeek-V4.1-Flash on 4\u00d7 H200 with vLLM 0.32.0 (TP4, FP8 KV cache)",
  "published": "2026-10-27",
  "summary": "Quick-mode measurement on a RunPod Secure Cloud node with 4\u00d7 H200 141 GB (October 27, 2026), following vLLM's published DeepSeek-V4.1-Flash recipe for H200: tensor parallel 4, FP8 KV cache, Engram tables offloaded to host memory, otherwise default settings. Chat-like workload with random-token prompts (median 1,024 input, 256 output tokens). Within p99 TTFT \u2264 2 s and p99 TPOT \u2264 50 ms the node served 5.09 requests/s: 2,047 output and about 8,180 input tokens per second, 90% on the GSM8K check. One node, quick mode (2-minute windows, 2 repeats), no prompt-cache hits.",
  "hardware": {
    "vendor": "NVIDIA",
    "accelerator": "NVIDIA H200",
    "count": 4,
    "rated_bw_tbs_per_gpu": 4.8,
    "driver": "595.91.07",
    "host": "RunPod secure cloud pod, "
  },
  "software": {
    "engine": "vLLM",
    "engine_version": "0.32.0",
    "kernels": "vllm/vllm-openai:v0.32.0 --kv-cache-dtype fp8 --tokenizer-mode deepseek_v41 --engram-config {\"cpu_offload\":true} --gpu-memory-utilization 0.95"
  },
  "model": {
    "name": "DeepSeek-V4.1-Flash",
    "shape": "MoE",
    "quantization": "FP8 weights, FP8 KV cache",
    "parallelism": "1 replica \u00d7 TP4"
  },
  "slo": {
    "ttft_p99_ms": 2000.0,
    "tpot_p99_ms": 50.0
  },
  "workload": {
    "trace": "chat-mix-v1",
    "input_tokens": "median 1,024 \u00b7 p90 4,096",
    "output_tokens": "median 256 \u00b7 p90 1,024",
    "arrival": "Poisson, rate swept to SLO boundary"
  },
  "power": {
    "source": "GPU board power sum (NVIDIA)",
    "kind": "board",
    "sampling_hz": 1,
    "window": "120 s steady state"
  },
  "accuracy": {
    "suite": "gsm8k-5shot",
    "threshold_pts": 1.0
  },
  "runs": 2,
  "variance_band_pct": 1.7,
  "fleet": {
    "cards": 64
  },
  "configs": [
    {
      "key": "baseline",
      "label": "DeepSeek-V4.1-Flash \u00b7 4\u00d7 H200 \u00b7 TP4 \u00b7 vLLM 0.32.0",
      "engine": "vLLM 0.32.0 \u00b7 KV FP8",
      "mbu_pct": 60.9,
      "node_power_kw": 2.377,
      "goodput_tok_s": 2047.4,
      "ttft_p99_ms": 379.0,
      "tpot_p99_ms": 45.3,
      "slo_pass_count": 2,
      "repeats": 2,
      "throttle_counts": {
        "sw_power_cap": 204
      },
      "accuracy": {
        "score": 90.0,
        "delta_pts": 0.0,
        "spread_pts": 0.0
      }
    }
  ],
  "not_covered": [
    "Whole-node power (no BMC access on this host)",
    "Context lengths beyond the chat-mix-v1 trace",
    "Quick mode: short windows, few repeats \u2014 validation only, not for publishing"
  ],
  "reproduce": "tokenwatt run --spec lambda-deepseek-v4.1-flash-tp4-dp1.spec.yaml --label 'dsv41flash-h200x4-tp4dp1-tpot50' --role baseline --out runs/dsv41flash-h200x4-tp4dp1-tpot50.json\n\ntokenwatt report runs/dsv41flash-h200x4-tp4dp1-tpot50.json --title 'DeepSeek-V4.1-Flash on 4\u00d7 H200 with vLLM 0.32.0 (TP4, FP8 KV cache)'",
  "spec": {
    "name": "lambda-deepseek-v4.1-flash-tp4-dp1",
    "hardware": {
      "adapter": "nvidia",
      "rated_bw_tbs_per_gpu": null,
      "gpus": null,
      "host": "RunPod secure cloud pod, "
    },
    "engine": {
      "endpoint": "http://127.0.0.1:8000/v1",
      "name": "vLLM",
      "version": "0.32.0",
      "kernels": "vllm/vllm-openai:v0.32.0 --kv-cache-dtype fp8 --tokenizer-mode deepseek_v41 --engram-config {\"cpu_offload\":true} --gpu-memory-utilization 0.95",
      "api": "completions",
      "model": "deepseek-ai/DeepSeek-V4.1-Flash",
      "ignore_eos": true,
      "api_key_env": null,
      "extra_body": {}
    },
    "model": {
      "name": "DeepSeek-V4.1-Flash",
      "config": "/root/tokenwatt-bench/models/deepseek-ai__DeepSeek-V4.1-Flash/config.json",
      "weight_dtype": "fp8",
      "kv_dtype": "fp8",
      "quantization": "",
      "replicas": 1,
      "tensor_parallel": 4,
      "vocab_size": 129280,
      "geometry": {}
    },
    "slo": {
      "ttft_p99_ms": 2000.0,
      "tpot_p99_ms": 50.0
    },
    "workload": {
      "trace": "chat-mix-v1",
      "input_tokens": null,
      "output_tokens": null,
      "seed": 0
    },
    "measurement": {
      "warmup_s": 45,
      "window_s": 120,
      "search_window_s": 60,
      "repeats": 2,
      "request_timeout_s": 300,
      "max_concurrency": 8192
    },
    "sweep": {
      "rates": null,
      "start_rps": 2.0,
      "growth": 2.0,
      "max_rps": 2000,
      "bisect_steps": 3,
      "backoff": 0.9,
      "max_backoffs": 4,
      "confirm": "backoff",
      "floor_rps": null,
      "confirm_steps": 5
    },
    "accuracy": {
      "suite": "/root/tokenwatt-bench/suites/gsm8k-200.jsonl",
      "name": "gsm8k-5shot",
      "scorer": "numeric",
      "max_drop_pts": 1.0,
      "seeds": 1,
      "max_tokens": 384,
      "stop": [
        "\n\nQuestion:"
      ],
      "concurrency": 32,
      "limit": 50
    },
    "power": {
      "source": "board",
      "sampling_hz": 1,
      "url": null,
      "username": null,
      "password_env": null,
      "verify_tls": false,
      "chassis": null,
      "host": null,
      "command": null,
      "field": null
    },
    "not_covered": [
      "Whole-node power (no BMC access on this host)",
      "Context lengths beyond the chat-mix-v1 trace",
      "Quick mode: short windows, few repeats \u2014 validation only, not for publishing"
    ]
  }
}