{
  "id": "h100-h200-qwen38-27b-pi-adapter-fp8-lora-vllm",
  "kind": "measurement",
  "status": "published",
  "title": "harness-hub's pi adapter (LoRA) on Qwen3.8-27B FP8, vLLM 0.33.0: Terminal-Bench 2.1 subset under pi, cost and stability on a shared H200",
  "published": "2026-11-21",
  "summary": "Three tests, 2026-11-20/21, on rented GPUs (RunPod Secure Cloud): (1) LoRA on the FP8 checkpoint, one H100 NVL, speed and memory; (2) 25 Terminal-Bench 2.1 tasks under pi 1.5.2 (Harbor 0.26.0), base model and adapter served by one FP8 engine on an H100 NVL, the tasks run on a Lambda host, thinking off, temperature 0.8, top-p 1.0, 100 turns and 60 min per task, one attempt; (3) our production H200 layout (Qwen3.8 at 52% of the GPU with LoRA + Qwen3.6 at 43%, CUDA MPS, 262K context) under the long-context agent load that crashed MTP on 11-20. Model Qwen/Qwen3.8-27B-FP8 (revision c3a462cc); adapter harness-hub/Qwen3.8-27B-pi-adapted (3cad999e, Apache-2.0, LoRA r32). Write-up: /blog/qwen38-27b-pi-adapter-fp8-terminal-bench.html",
  "conclusion": "On 25 Terminal-Bench 2.1 tasks under pi, the adapter solved 17 and the base model 14 on the FP8 checkpoint (5 tasks solved only by the adapter, 2 only by the base: sign test p = 0.45). On the 12 tasks both solved, the adapter needed 9.5 turns against 28.5 and 1,432 output tokens against 5,254 (medians). One adapter request decodes about 20% slower than the base model (72.5 vs 91.0 tok/s on the H200). The production layout with LoRA ran the 11-20 crash load for 10 minutes without a fault.",
  "headline": {
    "value": "3.0×",
    "label": "fewer turns per solved task, pi adapter vs base model, FP8"
  },
  "hardware": {
    "vendor": "NVIDIA",
    "accelerator": "NVIDIA H100 NVL / H200",
    "count": 1,
    "rated_bw_tbs_per_gpu": 3.9,
    "driver": "595.112.07 (CUDA 13.2)",
    "host": "RunPod Secure Cloud: 1x H100 NVL 94 GB (US-GA-2, 310 W) for the LoRA and Terminal-Bench runs; 1x H200 SXM 141 GB (US-NC-1) for the production-layout run"
  },
  "software": {
    "engine": "vLLM",
    "engine_version": "0.33.0",
    "kernels": "vllm/vllm-openai:v0.33.0, LoRA (punica Triton kernels), FP8 block-scaled weights"
  },
  "model": {
    "name": "Qwen3.8-27B-FP8 + harness-hub/Qwen3.8-27B-pi-adapted",
    "shape": "Dense (hybrid attention)",
    "quantization": "FP8 weights, BF16 KV cache; LoRA r32 on attention + MLP",
    "parallelism": "1 replica × TP1"
  },
  "pills": [
    "LoRA adapter · pi coding agent",
    "Terminal-Bench 2.1 · 25 tasks"
  ],
  "variables_measured": [
    "Terminal-Bench 2.1 tasks solved, base model vs adapter",
    "turns and output tokens per solved task",
    "single-request decode speed",
    "KV cache size",
    "faults under long-context load on a shared GPU",
    "tool-call success"
  ],
  "charts_sub": "FP8 checkpoint, our runs; HF's BF16 grades for the same tasks in the table below.",
  "charts": [
    {
      "title": "Median turns per solved task",
      "unit": "turns, 12 tasks both solved",
      "digits": 1,
      "better": "lower is better",
      "bars": [
        {
          "label": "Base model",
          "key": "base",
          "value": 28.5
        },
        {
          "label": "pi adapter",
          "key": "tuned",
          "value": 9.5
        }
      ]
    },
    {
      "title": "Median output tokens per solved task",
      "unit": "tokens, 12 tasks both solved",
      "digits": 0,
      "better": "lower is better",
      "bars": [
        {
          "label": "Base model",
          "key": "base",
          "value": 5253.5
        },
        {
          "label": "pi adapter",
          "key": "tuned",
          "value": 1432.0
        }
      ]
    },
    {
      "title": "Tasks solved of 25",
      "unit": "tasks",
      "digits": 0,
      "better": "higher is better",
      "max": 25,
      "bars": [
        {
          "label": "Base model",
          "key": "base",
          "value": 14.0
        },
        {
          "label": "pi adapter",
          "key": "tuned",
          "value": 17.0
        }
      ]
    }
  ],
  "tables": [
    {
      "title": "Terminal-Bench 2.1, 25 tasks under pi, one attempt each",
      "sub": "The tasks were selected: the 17 where HF's single BF16 attempts of the two models differed (path-tracing left out: both timed out) and 8 of the 27 light tasks both solved, drawn at random. A rerun at temperature 0.8 regresses toward the mean on any weights, so the BF16 columns are not a baseline.",
      "columns": [
        "Tasks",
        "HF BF16, base",
        "HF BF16, adapter",
        "FP8, base",
        "FP8, adapter"
      ],
      "highlight_rows": [
        3
      ],
      "rows": [
        [
          "15 the adapter gained in BF16",
          "0",
          "15",
          "7",
          "10"
        ],
        [
          "2 the adapter lost in BF16",
          "2",
          "0",
          "1",
          "1"
        ],
        [
          "8 both solved in BF16 (random)",
          "8",
          "8",
          "6",
          "6"
        ],
        [
          "All 25",
          "10",
          "23",
          "14",
          "17"
        ]
      ]
    },
    {
      "title": "Work per solved task (12 tasks both models solved on FP8)",
      "sub": "Medians. Agent time includes the commands the agent ran.",
      "columns": [
        "",
        "Base model",
        "pi adapter"
      ],
      "rows": [
        [
          "Turns",
          "28.5",
          "9.5"
        ],
        [
          "Output tokens",
          "5,254",
          "1,432"
        ],
        [
          "Agent time",
          "5.8 min",
          "6.6 min"
        ],
        [
          "Attempts at the 100-turn cap (of 25)",
          "9",
          "1"
        ]
      ]
    },
    {
      "title": "Cost of serving the adapter",
      "sub": "LoRA restricted to the attention + MLP layers (--lora-target-modules qkv_proj o_proj gate_up_proj down_proj).",
      "columns": [
        "Measure",
        "Without LoRA",
        "With LoRA"
      ],
      "rows": [
        [
          "One request, H100 NVL, decode tok/s",
          "75.8",
          "base 76.4 · adapter 65.3 (-13.8%)"
        ],
        [
          "One request, our H200, decode tok/s",
          "91.4 (public API)",
          "base 91.0 · adapter 72.5 (-20%)"
        ],
        [
          "Mixed base + adapter traffic, 16 in flight, H100 NVL",
          "931 tok/s",
          "834 tok/s (-10.5%)"
        ],
        [
          "Qwen3.8 KV cache on our H200 (0.52 of the GPU)",
          "668,668 tokens",
          "632,371 tokens (-5.4%)"
        ]
      ]
    },
    {
      "title": "Our production H200 layout under the 11-20 crash load",
      "sub": "6 + 6 concurrent coding-agent sessions, prompts growing to ~100-125K tokens, CUDA MPS, 10 minutes.",
      "columns": [
        "Configuration",
        "Result"
      ],
      "rows": [
        [
          "Qwen3.8 0.52 + LoRA (half its sessions on the adapter), Qwen3.6 0.43",
          "no fault in 10 min; Qwen3.8 108 requests, Qwen3.6 863"
        ],
        [
          "Qwen3.8 0.52, Qwen3.6 0.43, no LoRA (11-20)",
          "no fault in 10 min"
        ],
        [
          "Qwen3.8 0.57 + MTP, Qwen3.6 0.38 (11-20)",
          "illegal memory access at 1.0 and 2.1 min"
        ]
      ]
    }
  ],
  "setup": [
    [
      "Adapter",
      "harness-hub/Qwen3.8-27B-pi-adapted at 3cad999e: LoRA r32, alpha 64, q/k/v/o (16 full-attention layers) + gate/up/down; trained with RL for pi, thinking off, on BF16 weights"
    ],
    [
      "Engine",
      "vllm serve Qwen/Qwen3.8-27B-FP8 --enable-lora --max-lora-rank 32 --lora-target-modules qkv_proj o_proj gate_up_proj down_proj --lora-modules qwen3.8-27b-pi=<dir>"
    ],
    [
      "Terminal-Bench",
      "Harbor 0.26.0, agent pi 1.5.2 (the version in HF's trajectories), openai-completions API, context 131,072, max 8,192 tokens per call, 100 turns, 3,600 s per task, one attempt; tool parser qwen3_xml, thinking off, temperature 0.8, top-p 1.0"
    ],
    [
      "Two passes",
      "first pass: 26 trials at once on one H100 NVL, cut at its time limit with 17 trials running; rerun of those 17 and 2 cancelled ones, 19 at once, same settings. Each (task, model) has one graded attempt"
    ],
    [
      "Production-layout run",
      "separate H200 SXM, production's exact engine flags plus LoRA, CUDA MPS, the 11-20 load generator; then tool calls (20 cases), single-request speed and one pi run on the same engines"
    ],
    [
      "Our API",
      "since 2026-11-21 19:29 UTC: harness-hub/qwen3.8-27b-pi-adapted, Qwen3.8-27B's price, thinking off by default (the caller's choice wins)"
    ]
  ],
  "not_covered": [
    "A BF16 control on our own stack: FP8 drift and run-to-run noise are not separated",
    "More than one attempt per task, or all 89 Terminal-Bench tasks",
    "SWE-bench Pro, and the opencode and Hermes adapters",
    "Capacity under a latency target with adapter traffic on the H200",
    "Stability beyond the 10-minute run"
  ],
  "reproduce_title": "Reproduce",
  "reproduce_sub": "Serve the adapter next to the base model and point pi at it; the Terminal-Bench tasks come from harborframework/terminal-bench-2.1 on Hugging Face.",
  "reproduce": "vllm serve Qwen/Qwen3.8-27B-FP8 --revision c3a462cc3d38cf351f3b26cbd763469eeb74ef81 \\\n  --served-model-name qwen3.8-27b --max-model-len 131072 --language-model-only \\\n  --reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_xml \\\n  --default-chat-template-kwargs '{\"enable_thinking\": false}' \\\n  --enable-lora --max-lora-rank 32 --lora-target-modules qkv_proj o_proj gate_up_proj down_proj \\\n  --lora-modules qwen3.8-27b-pi=harness-hub/Qwen3.8-27B-pi-adapted\n\n# Harbor, agent pi 1.5.2, both models as two agents in one job (each task's two trials start together)\nharbor run --config job.json --yes",
  "post": "/blog/qwen38-27b-pi-adapter-fp8-terminal-bench.html",
  "data": {
    "source": "bench/results/pi-adapter-fp8-2026-11, lora-hosting-2026-11, lora-prodshape-2026-11 (TokenWatt repository)",
    "tasks": 25,
    "strata": {
      "gained": {
        "graded": [
          "adaptive-rejection-sampler",
          "bn-fit-modify",
          "cancel-async-tasks",
          "custom-memory-heap-crash",
          "db-wal-recovery",
          "financial-document-processor",
          "fix-ocaml-gc",
          "large-scale-text-editing",
          "mailman",
          "mteb-leaderboard",
          "regex-log",
          "reshard-c4-data",
          "tune-mjcf",
          "vulnerable-secret",
          "winning-avg-corewars"
        ],
        "fp8_base": 7.0,
        "fp8_pi": 10.0,
        "bf16_base": 0.0,
        "bf16_pi": 15.0,
        "all_base": {
          "solved": 7,
          "graded": 15,
          "cut_or_ungraded": 0
        },
        "all_pi": {
          "solved": 10,
          "graded": 15,
          "cut_or_ungraded": 0
        },
        "tasks": [
          "adaptive-rejection-sampler",
          "bn-fit-modify",
          "cancel-async-tasks",
          "custom-memory-heap-crash",
          "db-wal-recovery",
          "financial-document-processor",
          "fix-ocaml-gc",
          "large-scale-text-editing",
          "mailman",
          "mteb-leaderboard",
          "regex-log",
          "reshard-c4-data",
          "tune-mjcf",
          "vulnerable-secret",
          "winning-avg-corewars"
        ]
      },
      "lost": {
        "graded": [
          "crack-7z-hash",
          "sanitize-git-repo"
        ],
        "fp8_base": 1.0,
        "fp8_pi": 1.0,
        "bf16_base": 2.0,
        "bf16_pi": 0.0,
        "all_base": {
          "solved": 1,
          "graded": 2,
          "cut_or_ungraded": 0
        },
        "all_pi": {
          "solved": 1,
          "graded": 2,
          "cut_or_ungraded": 0
        },
        "tasks": [
          "crack-7z-hash",
          "sanitize-git-repo"
        ]
      },
      "both-solved": {
        "graded": [
          "build-cython-ext",
          "cobol-modernization",
          "extract-elf",
          "hf-model-inference",
          "kv-store-grpc",
          "merge-diff-arc-agi-task",
          "modernize-scientific-stack",
          "prove-plus-comm"
        ],
        "fp8_base": 6.0,
        "fp8_pi": 6.0,
        "bf16_base": 8.0,
        "bf16_pi": 8.0,
        "all_base": {
          "solved": 6,
          "graded": 8,
          "cut_or_ungraded": 0
        },
        "all_pi": {
          "solved": 6,
          "graded": 8,
          "cut_or_ungraded": 0
        },
        "tasks": [
          "build-cython-ext",
          "cobol-modernization",
          "extract-elf",
          "hf-model-inference",
          "kv-store-grpc",
          "merge-diff-arc-agi-task",
          "modernize-scientific-stack",
          "prove-plus-comm"
        ]
      }
    },
    "both_solved_fp8": {
      "tasks": 12,
      "median_turns": {
        "base": 28.5,
        "pi": 9.5
      },
      "median_output_tokens": {
        "base": 5253.5,
        "pi": 1432.0
      },
      "median_agent_s": {
        "base": 345.079999,
        "pi": 396.014107
      }
    },
    "sign_test": {
      "adapter_only": 5,
      "base_only": 2,
      "p_two_sided": 0.45
    },
    "gpu_cost_usd": 15.24
  }
}
