{
 "id": "mi350x-qwen3-8b-amd-vs-default-vllm",
 "status": "published",
 "title": "Qwen3-8B BF16 on 1\u00d7 AMD Instinct MI350X: vLLM's ROCm image vs AMD's image and settings",
 "published": "2026-11-18",
 "summary": "Quick-mode measurement on a RunPod Secure Cloud MI350X, 2026-11-13, $5.49 per hour on the day. Three software setups on the same host: vLLM 0.31.0's own ROCm image (vllm/vllm-openai-rocm:v0.31.0) with default settings; the same image with VLLM_ROCM_USE_AITER=1; and AMD's image (rocm/vllm:rocm10.1.0_ubuntu24.04_py3.14-pytorch_2.13.0_vllm-0.29.0, AMD's vLLM 0.29.1.dev build, which Docker Hub describes as a ROCm RC image) with the settings from AMD's vLLM optimization guide: VLLM_ROCM_USE_AITER=1, --attention-backend ROCM_AITER_FA, TORCH_BLAS_PREFER_HIPBLASLT=1. AMD's image removed the first-token tail (p99 382 ms vs 1,545 ms) and served 2.4\u00d7 the tokens. Per million output tokens at $5.49/h: $0.66, $0.46, $0.28. The AMD-image run averaged 973 W against the 1,000 W limit. Write-up: /blog/mi350x-vs-h100-software-and-price.html",
 "hardware": {
  "vendor": "AMD",
  "accelerator": "AMD Instinct MI350X",
  "count": 1,
  "rated_bw_tbs_per_gpu": 8.0,
  "driver": "7.1.3.31500000 \u00b7 ROCm 7.2.3",
  "host": "RunPod Secure Cloud pod, one host for all runs (two pods in a row: vLLM's image, then AMD's image); $5.49 per hour on 2026-11-13"
 },
 "software": {
  "engine": "vLLM",
  "engine_version": "varies by configuration",
  "kernels": "vllm/vllm-openai-rocm:v0.31.0"
 },
 "model": {
  "name": "Qwen3-8B",
  "shape": "Dense",
  "quantization": "BF16 weights, BF16 KV cache",
  "parallelism": "1 replica \u00d7 TP1"
 },
 "slo": {
  "ttft_p99_ms": 2000.0,
  "tpot_p99_ms": 50.0
 },
 "workload": {
  "trace": "chat-mix-v1",
  "input_tokens": "median 1,024 \u00b7 p90 4,096",
  "output_tokens": "median 256 \u00b7 p90 1,024",
  "arrival": "Poisson, rate swept to SLO boundary"
 },
 "power": {
  "source": "GPU board power sum (AMD)",
  "kind": "board",
  "sampling_hz": 1,
  "window": "120 s steady state"
 },
 "accuracy": {
  "suite": "gsm8k-5shot",
  "threshold_pts": 1.0
 },
 "runs": 2,
 "variance_band_pct": 1.8,
 "fleet": {
  "cards": 64
 },
 "configs": [
  {
   "key": "baseline",
   "label": "vLLM ROCm image, defaults",
   "engine": "vLLM 0.31.0 \u00b7 KV BF16",
   "mbu_pct": 27.0,
   "node_power_kw": 0.817,
   "goodput_tok_s": 2304.3,
   "ttft_p99_ms": 1545.0,
   "tpot_p99_ms": 25.6,
   "slo_pass_count": 2,
   "repeats": 2,
   "throttle_counts": {},
   "accuracy": {
    "score": 92.0,
    "delta_pts": 0.0,
    "spread_pts": 0.0
   }
  },
  {
   "key": "other",
   "label": "vLLM ROCm image + AITER",
   "engine": "vLLM 0.31.0 \u00b7 KV BF16",
   "mbu_pct": 23.7,
   "node_power_kw": 0.847,
   "goodput_tok_s": 3352.4,
   "ttft_p99_ms": 1598.0,
   "tpot_p99_ms": 44.3,
   "slo_pass_count": 2,
   "repeats": 2,
   "throttle_counts": {},
   "accuracy": {
    "score": 92.0,
    "delta_pts": 0.0,
    "spread_pts": 0.0
   }
  },
  {
   "key": "tuned",
   "label": "AMD image + AMD settings",
   "engine": "vLLM 0.29.1.dev0+g98dff2a81.d20261002 \u00b7 KV BF16",
   "mbu_pct": 27.8,
   "node_power_kw": 0.973,
   "goodput_tok_s": 5427.2,
   "ttft_p99_ms": 382.0,
   "tpot_p99_ms": 41.9,
   "slo_pass_count": 2,
   "repeats": 2,
   "throttle_counts": {},
   "accuracy": {
    "score": 94.0,
    "delta_pts": 2.0,
    "spread_pts": 0.0
   }
  }
 ],
 "not_covered": [
  "Whole-node power (no BMC access on this host)",
  "Context lengths beyond the chat-mix-v1 trace",
  "Quick mode: short windows, few repeats \u2014 validation only, not for publishing",
  "Power: AMD reports the whole OAM package (amd-smi socket_power, 1,000 W limit), NVIDIA the board; the AMD adapter doesn't read throttle reasons",
  "Accuracy: quick mode scores GSM8K on 50 questions with one seed, so one question is 2 points and the 1-point gate can't tell a real drop from sampling noise. A \u22124 pt flag is two questions",
  "Engine versions differ by configuration: vLLM 0.31.0 (vLLM's image) vs AMD's 0.29.1.dev build; the AMD-image gain mixes the image, the attention backend and hipBLASLt, not measured separately"
 ],
 "reproduce": "Raw run files, logs and the exact launch commands: bench/results/amd-mi350x-2026-11/ in the TokenWatt repository (README.md).\n\ntokenwatt run --spec lambda-qwen3-8b-tp1-dp1.spec.yaml --label 'vLLM ROCm image, defaults' --role baseline --out runs/vllm-rocm-image-defaults.json\ntokenwatt run --spec lambda-qwen3-8b-tp1-dp1.spec.yaml --label 'vLLM ROCm image + AITER' --role other --out runs/vllm-rocm-image-aiter.json\ntokenwatt run --spec lambda-qwen3-8b-tp1-dp1.spec.yaml --label 'AMD image + AMD settings' --role tuned --out runs/amd-image-amd-settings.json\n\ntokenwatt report runs/vllm-rocm-image-defaults.json runs/vllm-rocm-image-aiter.json runs/amd-image-amd-settings.json --title 'Qwen3-8B BF16 on 1\u00d7 AMD Instinct MI350X: vLLM's ROCm image vs AMD's image and settings'",
 "spec": {
  "name": "lambda-qwen3-8b-tp1-dp1",
  "hardware": {
   "adapter": "amd",
   "rated_bw_tbs_per_gpu": null,
   "gpus": null,
   "host": "RunPod secure cloud pod, "
  },
  "engine": {
   "endpoint": "http://127.0.0.1:8000/v1",
   "name": "vLLM",
   "version": "0.31.0",
   "kernels": "vllm/vllm-openai-rocm:v0.31.0",
   "api": "completions",
   "model": "Qwen/Qwen3-8B",
   "ignore_eos": true,
   "api_key_env": null,
   "extra_body": {}
  },
  "model": {
   "name": "Qwen3-8B",
   "config": "/root/tokenwatt-bench/models/Qwen__Qwen3-8B/config.json",
   "weight_dtype": "bf16",
   "kv_dtype": "bf16",
   "quantization": "",
   "replicas": 1,
   "tensor_parallel": 1,
   "vocab_size": 151936,
   "geometry": {}
  },
  "slo": {
   "ttft_p99_ms": 2000.0,
   "tpot_p99_ms": 50.0
  },
  "workload": {
   "trace": "chat-mix-v1",
   "input_tokens": null,
   "output_tokens": null,
   "seed": 0,
   "shared_prefix": 0.0,
   "prefix_pool": 1
  },
  "measurement": {
   "warmup_s": 45,
   "window_s": 120,
   "search_window_s": 60,
   "repeats": 2,
   "request_timeout_s": 300,
   "max_concurrency": 8192
  },
  "sweep": {
   "rates": null,
   "start_rps": 1.0,
   "growth": 1.6,
   "max_rps": 2000,
   "bisect_steps": 3,
   "backoff": 0.9,
   "max_backoffs": 4,
   "confirm": "backoff",
   "floor_rps": null,
   "confirm_steps": 5
  },
  "accuracy": {
   "suite": "/root/tokenwatt-bench/suites/gsm8k-200.jsonl",
   "name": "gsm8k-5shot",
   "scorer": "numeric",
   "max_drop_pts": 1.0,
   "seeds": 1,
   "max_tokens": 384,
   "stop": [
    "\n\nQuestion:"
   ],
   "concurrency": 32,
   "limit": 50
  },
  "power": {
   "source": "board",
   "sampling_hz": 1,
   "url": null,
   "username": null,
   "password_env": null,
   "verify_tls": false,
   "chassis": null,
   "host": null,
   "command": null,
   "field": null
  },
  "not_covered": [
   "Whole-node power (no BMC access on this host)",
   "Context lengths beyond the chat-mix-v1 trace",
   "Quick mode: short windows, few repeats \u2014 validation only, not for publishing"
  ]
 }
}