{
 "title": "Same open model, different providers (OpenRouter, 2026-11-05)",
 "method": "bench/provider-audit-2026-11/README.md",
 "snapshot_taken": "2026-11-05T07:09:19.736240+00:00",
 "omitted_endpoints": 9,
 "models": {
  "kimi-k2.6": {
   "openrouter_id": "moonshotai/kimi-k2.6",
   "reasoning": {
    "enabled": false
   },
   "endpoints": {
    "moonshotai": {
     "provider": "Moonshot AI",
     "declared_quantization": "unknown",
     "price_per_m": {
      "input": 0.95,
      "output": 4.0,
      "cache_read": 0.16
     },
     "context_length": 262144,
     "gsm8k": {
      "answered": 200,
      "correct": 197,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 98.5,
      "wilson95": [
       95.68335194437212,
       99.48858548155077
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 106,
      "failed": 0,
      "truncated": 32,
      "acc_pct": 70.66666666666667,
      "wilson95": [
       62.93750243784035,
       77.36369047746993
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 10
     },
     "cache": {
      "hits": 3,
      "warm_ok": 3,
      "cached_share": [
       0.909,
       0.909,
       0.909
      ],
      "prompt_cost_discount": [
       0.756,
       0.756,
       0.756
      ],
      "expected_discount": 0.756
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32418,
       "ref_prompt_tokens": 32420
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1106.1373129487038,
       "ttft_p50_ci": [
        1086.0355622135103,
        1176.3127706944942
       ],
       "ttft_p90_ms": 6513.158554863186,
       "ttft_p99_ms": 9960.203588213772,
       "ttft_max_ms": 10581.935917027295,
       "decode_tok_s_p50": 39.344265689262116,
       "decode_p50_ci": [
        36.58983733309496,
        41.58484910953157
       ],
       "decode_tok_s_p10": 35.38466867066942,
       "tpot_p99_ms": 29.45759087342338,
       "e2e_p50_s": 6.153436208609492,
       "e2e_p99_s": 15.088375834450126,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.99
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1154.218541458249,
       "ttft_p50_ci": [
        1102.816999424249,
        1218.008229508996
       ],
       "ttft_p90_ms": 2219.1163330338964,
       "ttft_p99_ms": 11582.560239508746,
       "ttft_max_ms": 12899.522542022169,
       "decode_tok_s_p50": 37.07082597529943,
       "decode_p50_ci": [
        35.23754070255828,
        39.80300480407273
       ],
       "decode_tok_s_p10": 34.400060535154715,
       "tpot_p99_ms": 29.956378429032977,
       "e2e_p50_s": 6.7695917915552855,
       "e2e_p99_s": 16.393393610958007,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.99
      }
     },
     "billing": {
      "requests": 445,
      "prompt_delta_median": -2,
      "completion_ratio_median": 1.0265486725663717,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999998,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 445,
      "failed_attempts_by_status": {
       "429": 1
      },
      "final_failures_by_status": {}
     }
    },
    "chutes/int4": {
     "provider": "Chutes",
     "declared_quantization": "int4",
     "price_per_m": {
      "input": 0.39,
      "output": 1.7999999999999998,
      "cache_read": 0.039
     },
     "context_length": 262144,
     "gsm8k": {
      "answered": 200,
      "correct": 198,
      "failed": 0,
      "truncated": 1,
      "acc_pct": 99.0,
      "wilson95": [
       96.42775148038206,
       99.72533993962251
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 109,
      "failed": 0,
      "truncated": 28,
      "acc_pct": 72.66666666666667,
      "wilson95": [
       65.03534444650609,
       79.16596391222133
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 9,
      "unsupported_by_endpoint": 1
     },
     "cache": {
      "hits": 3,
      "warm_ok": 3,
      "cached_share": [
       0.994,
       0.994,
       0.994
      ],
      "prompt_cost_discount": [
       0.895,
       0.895,
       0.895
      ],
      "expected_discount": 0.895
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32421,
       "ref_prompt_tokens": 32421
      },
      "ctx225000": {
       "ok": false,
       "found": false,
       "prompt_tokens": null,
       "ref_prompt_tokens": 250846
      },
      "ctx145000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 161693,
       "ref_prompt_tokens": 161690
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1206.707312259823,
       "ttft_p50_ci": [
        972.5803127512336,
        1242.0212915167212
       ],
       "ttft_p90_ms": 1395.8081501536071,
       "ttft_p99_ms": 2387.644030377269,
       "ttft_max_ms": 2506.1177918687463,
       "decode_tok_s_p50": 48.151625561189775,
       "decode_p50_ci": [
        44.668574528909446,
        51.436125601154615
       ],
       "decode_tok_s_p10": 34.04917986079646,
       "tpot_p99_ms": 59.97242885192472,
       "e2e_p50_s": 5.26187500031665,
       "e2e_p99_s": 13.611277288813142,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 1.0
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1163.1861459463835,
       "ttft_p50_ci": [
        1016.2832923233509,
        1252.3266873322427
       ],
       "ttft_p90_ms": 1768.2643704116347,
       "ttft_p99_ms": 6256.426754277199,
       "ttft_max_ms": 6917.951417155564,
       "decode_tok_s_p50": 50.02200146152538,
       "decode_p50_ci": [
        48.64997094803259,
        51.36216527658015
       ],
       "decode_tok_s_p10": 23.965783942180316,
       "tpot_p99_ms": 47.988118952533334,
       "e2e_p50_s": 5.267167708370835,
       "e2e_p99_s": 10.649021667176857,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 1.0
      }
     },
     "billing": {
      "requests": 445,
      "prompt_delta_median": 0,
      "completion_ratio_median": 1.0092167855929324,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999998,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 447,
      "failed_attempts_by_status": {
       "404": 1,
       "400": 1
      },
      "final_failures_by_status": {
       "404": 1,
       "400": 1
      }
     }
    },
    "deepinfra/fp4": {
     "provider": "DeepInfra",
     "declared_quantization": "fp4",
     "price_per_m": {
      "input": 0.75,
      "output": 3.5,
      "cache_read": 0.15
     },
     "context_length": 262144,
     "gsm8k": {
      "answered": 200,
      "correct": 195,
      "failed": 0,
      "truncated": 2,
      "acc_pct": 97.5,
      "wilson95": [
       94.2820855848383,
       98.92754385292123
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 107,
      "failed": 0,
      "truncated": 30,
      "acc_pct": 71.33333333333333,
      "wilson95": [
       63.63493407951355,
       77.96629731693578
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 10
     },
     "cache": {
      "hits": 3,
      "warm_ok": 3,
      "cached_share": [
       0.987,
       0.987,
       0.987
      ],
      "prompt_cost_discount": [
       0.79,
       0.79,
       0.79
      ],
      "expected_discount": 0.79
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32422,
       "ref_prompt_tokens": 32422
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 856.1516464687884,
       "ttft_p50_ci": [
        814.6188333630562,
        976.7683958634734
       ],
       "ttft_p90_ms": 1420.2853327617051,
       "ttft_p99_ms": 3532.57546530105,
       "ttft_max_ms": 4549.709375016391,
       "decode_tok_s_p50": 12.067684318529729,
       "decode_p50_ci": [
        11.259643736838417,
        12.718823184149675
       ],
       "decode_tok_s_p10": 9.454642596438024,
       "tpot_p99_ms": 156.2541182568512,
       "e2e_p50_s": 17.689927270635962,
       "e2e_p99_s": 32.02403395269066,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.54
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 859.3005416914821,
       "ttft_p50_ci": [
        761.5355001762509,
        981.6747079603374
       ],
       "ttft_p90_ms": 1156.2004034407437,
       "ttft_p99_ms": 3023.138030804693,
       "ttft_max_ms": 3863.832333125174,
       "decode_tok_s_p50": 9.98913447080428,
       "decode_p50_ci": [
        9.624293751650473,
        10.79752699236747
       ],
       "decode_tok_s_p10": 9.006029153755387,
       "tpot_p99_ms": 115.1095379426158,
       "e2e_p50_s": 21.12874475028366,
       "e2e_p99_s": 23.697705727312712,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.54
      }
     },
     "billing": {
      "requests": 445,
      "prompt_delta_median": 0,
      "completion_ratio_median": 1.009009009009009,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999998,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 445,
      "failed_attempts_by_status": {},
      "final_failures_by_status": {}
     }
    },
    "coreweave/fp4": {
     "provider": "CoreWeave",
     "declared_quantization": "fp4",
     "price_per_m": {
      "input": 0.65,
      "output": 3.41,
      "cache_read": 0.15
     },
     "context_length": 262144,
     "gsm8k": {
      "answered": 200,
      "correct": 198,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 99.0,
      "wilson95": [
       96.42775148038206,
       99.72533993962251
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 109,
      "failed": 0,
      "truncated": 28,
      "acc_pct": 72.66666666666667,
      "wilson95": [
       65.03534444650609,
       79.16596391222133
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 10
     },
     "cache": {
      "hits": 3,
      "warm_ok": 3,
      "cached_share": [
       0.987,
       0.987,
       0.987
      ],
      "prompt_cost_discount": [
       0.759,
       0.759,
       0.759
      ],
      "expected_discount": 0.759
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32420,
       "ref_prompt_tokens": 32420
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 350.87654180824757,
       "ttft_p50_ci": [
        340.97231226041913,
        369.4963129237294
       ],
       "ttft_p90_ms": 507.0655453950167,
       "ttft_p99_ms": 615.2566735260189,
       "ttft_max_ms": 630.6212926283479,
       "decode_tok_s_p50": 195.5614382264406,
       "decode_p50_ci": [
        188.56220609493164,
        199.879012267772
       ],
       "decode_tok_s_p10": 113.17915462853232,
       "tpot_p99_ms": 16.590230858898792,
       "e2e_p50_s": 1.387173145543784,
       "e2e_p99_s": 3.7031179748754948,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.58
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 366.7537709698081,
       "ttft_p50_ci": [
        348.78295892849565,
        390.5125423334539
       ],
       "ttft_p90_ms": 533.259671367705,
       "ttft_p99_ms": 740.9513955283909,
       "ttft_max_ms": 801.9738346338272,
       "decode_tok_s_p50": 187.60736882935228,
       "decode_p50_ci": [
        171.03174949812234,
        198.91891064866638
       ],
       "decode_tok_s_p10": 102.20639801892169,
       "tpot_p99_ms": 14.935284516675836,
       "e2e_p50_s": 1.4399596662260592,
       "e2e_p99_s": 3.4606169670727103,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.57
      }
     },
     "billing": {
      "requests": 445,
      "prompt_delta_median": 0,
      "completion_ratio_median": 1.0084033613445378,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999997,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 445,
      "failed_attempts_by_status": {},
      "final_failures_by_status": {}
     }
    },
    "novita": {
     "provider": "Novita",
     "declared_quantization": "unknown",
     "price_per_m": {
      "input": 0.7999999999999999,
      "output": 3.4,
      "cache_read": 0.16
     },
     "context_length": 262144,
     "gsm8k": {
      "answered": 200,
      "correct": 196,
      "failed": 0,
      "truncated": 1,
      "acc_pct": 98.0,
      "wilson95": [
       94.97121344948062,
       99.21956998236058
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 109,
      "failed": 0,
      "truncated": 31,
      "acc_pct": 72.66666666666667,
      "wilson95": [
       65.03534444650609,
       79.16596391222133
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 10
     },
     "cache": {
      "hits": 2,
      "warm_ok": 3,
      "cached_share": [
       0.0,
       0.909,
       0.909
      ],
      "prompt_cost_discount": [
       0.728,
       0.728
      ],
      "expected_discount": 0.728
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32418,
       "ref_prompt_tokens": 32420
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 2501.8543545156717,
       "ttft_p50_ci": [
        1868.1486672721803,
        2756.52856240049
       ],
       "ttft_p90_ms": 4544.066854659474,
       "ttft_p99_ms": 15325.81664497964,
       "ttft_max_ms": 15540.966417640448,
       "decode_tok_s_p50": 35.820040355669605,
       "decode_p50_ci": [
        35.20213264215591,
        36.921347945989325
       ],
       "decode_tok_s_p10": 34.109752730938,
       "tpot_p99_ms": 29.814657267959948,
       "e2e_p50_s": 7.428392416331917,
       "e2e_p99_s": 21.100419844109563,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.99
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1759.3522500246763,
       "ttft_p50_ci": [
        1280.9092290699482,
        2405.566603410989
       ],
       "ttft_p90_ms": 8718.796703778213,
       "ttft_p99_ms": 15172.409997470677,
       "ttft_max_ms": 15721.945250406861,
       "decode_tok_s_p50": 35.52238550538143,
       "decode_p50_ci": [
        34.00546747995219,
        36.752333591616946
       ],
       "decode_tok_s_p10": 33.16660501024312,
       "tpot_p99_ms": 30.698202159162143,
       "e2e_p50_s": 7.275775499641895,
       "e2e_p99_s": 21.220667377142234,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.99
      }
     },
     "billing": {
      "requests": 445,
      "prompt_delta_median": -2,
      "completion_ratio_median": 1.027027027027027,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999998,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 445,
      "failed_attempts_by_status": {},
      "final_failures_by_status": {}
     }
    },
    "baidu/fp4": {
     "provider": "Baidu",
     "declared_quantization": "fp4",
     "price_per_m": {
      "input": 0.95,
      "output": 4.0,
      "cache_read": 0.16
     },
     "context_length": 262144,
     "gsm8k": {
      "answered": 200,
      "correct": 197,
      "failed": 0,
      "truncated": 2,
      "acc_pct": 98.5,
      "wilson95": [
       95.68335194437212,
       99.48858548155077
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 108,
      "failed": 0,
      "truncated": 30,
      "acc_pct": 72.0,
      "wilson95": [
       64.3342019497018,
       78.5670679278866
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 9,
      "unsupported_by_endpoint": 1
     },
     "cache": {
      "hits": 3,
      "warm_ok": 3,
      "cached_share": [
       0.64,
       0.62,
       0.7
      ],
      "prompt_cost_discount": [
       0.051,
       0.017,
       0.152
      ],
      "expected_discount": 0.532
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32422,
       "ref_prompt_tokens": 32422
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 646.3581873103976,
       "ttft_p50_ci": [
        604.8134579323232,
        735.41493806988
       ],
       "ttft_p90_ms": 1166.2051205523312,
       "ttft_p99_ms": 1337.186947800219,
       "ttft_max_ms": 1347.2014162689447,
       "decode_tok_s_p50": 69.80407807438826,
       "decode_p50_ci": [
        67.15579525024839,
        71.02047142919403
       ],
       "decode_tok_s_p10": 63.453879880521605,
       "tpot_p99_ms": 16.501784389276974,
       "e2e_p50_s": 3.5920808953233063,
       "e2e_p99_s": 4.557451028423384,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.555
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 697.7056255564094,
       "ttft_p50_ci": [
        634.3499994836748,
        769.9533752165735
       ],
       "ttft_p90_ms": 1046.0779713466766,
       "ttft_p99_ms": 1530.386720728129,
       "ttft_max_ms": 1686.8155421689153,
       "decode_tok_s_p50": 65.13170599043261,
       "decode_p50_ci": [
        63.08874156112931,
        67.26997666471536
       ],
       "decode_tok_s_p10": 58.0493245833303,
       "tpot_p99_ms": 18.58725049940111,
       "e2e_p50_s": 3.8832428962923586,
       "e2e_p99_s": 4.814957289109007,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.57
      }
     },
     "billing": {
      "requests": 444,
      "prompt_delta_median": 0.0,
      "completion_ratio_median": 1.0085476329631793,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999998,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 445,
      "failed_attempts_by_status": {
       "429": 41,
       "404": 1
      },
      "final_failures_by_status": {
       "404": 1
      }
     }
    }
   }
  },
  "deepseek-v4-pro-0813": {
   "openrouter_id": "deepseek/deepseek-v4-pro-0813",
   "reasoning": {
    "enabled": false
   },
   "endpoints": {
    "wafer/us": {
     "provider": "Wafer",
     "declared_quantization": "mxfp4",
     "price_per_m": {
      "input": 0.10700000000000001,
      "output": 4.2780000000000005,
      "cache_read": 0.1023
     },
     "context_length": 1048576,
     "gsm8k": {
      "answered": 200,
      "correct": 195,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 97.5,
      "wilson95": [
       94.2820855848383,
       98.92754385292123
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 112,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 74.66666666666667,
      "wilson95": [
       67.15044398939715,
       80.9509798127474
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 10
     },
     "cache": {
      "hits": 2,
      "warm_ok": 3,
      "cached_share": [
       0.0,
       0.947,
       0.947
      ],
      "prompt_cost_discount": [
       0.042,
       0.042
      ],
      "expected_discount": 0.042
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 33227,
       "ref_prompt_tokens": 33227
      },
      "ctx820000": {
       "ok": false,
       "found": false,
       "prompt_tokens": null,
       "ref_prompt_tokens": 936724
      },
      "ctx560000": {
       "ok": false,
       "found": false,
       "prompt_tokens": null,
       "ref_prompt_tokens": 639764
      },
      "ctx200000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 228506,
       "ref_prompt_tokens": 228506
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 214.4100833684206,
       "ttft_p50_ci": [
        208.82943831384182,
        221.33689559996128
       ],
       "ttft_p90_ms": 310.7590911909938,
       "ttft_p99_ms": 891.1978899221863,
       "ttft_max_ms": 1208.8508745655417,
       "decode_tok_s_p50": 89.87581613216472,
       "decode_p50_ci": [
        85.08025912872257,
        94.29369885378375
       ],
       "decode_tok_s_p10": 70.81869169881736,
       "tpot_p99_ms": 15.892575214518676,
       "e2e_p50_s": 2.474557041656226,
       "e2e_p99_s": 3.402682061195373,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.4575
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 230.66222900524735,
       "ttft_p50_ci": [
        215.32472968101501,
        239.58487529307604
       ],
       "ttft_p90_ms": 352.49489182606345,
       "ttft_p99_ms": 1604.8487818893038,
       "ttft_max_ms": 2071.894207969308,
       "decode_tok_s_p50": 85.98097498478668,
       "decode_p50_ci": [
        81.14922228056913,
        90.80796463294246
       ],
       "decode_tok_s_p10": 65.15233393295776,
       "tpot_p99_ms": 17.208706319762683,
       "e2e_p50_s": 2.600504687987268,
       "e2e_p99_s": 4.44181892116554,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.4525
      }
     },
     "billing": {
      "requests": 446,
      "prompt_delta_median": 0.0,
      "completion_ratio_median": 1.011111111111111,
      "cost_ratio_median": 0.9999999999999999,
      "cost_ratio_minmax": [
       0.9999999999999997,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 449,
      "failed_attempts_by_status": {
       "400": 1,
       "429": 10
      },
      "final_failures_by_status": {
       "400": 1,
       "429": 2
      }
     }
    },
    "novita/fp8": {
     "provider": "Novita",
     "declared_quantization": "fp8",
     "price_per_m": {
      "input": 0.9900000000000001,
      "output": 2.9699999999999998,
      "cache_read": 0.032999999999999995
     },
     "context_length": 1048576,
     "gsm8k": {
      "answered": 200,
      "correct": 196,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 98.0,
      "wilson95": [
       94.97121344948062,
       99.21956998236058
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 108,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 72.0,
      "wilson95": [
       64.3342019497018,
       78.5670679278866
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 10
     },
     "cache": {
      "hits": 3,
      "warm_ok": 3,
      "cached_share": [
       0.891,
       0.891,
       0.891
      ],
      "prompt_cost_discount": [
       0.861,
       0.861,
       0.861
      ],
      "expected_discount": 0.861
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 33226,
       "ref_prompt_tokens": 33226
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1533.1385205499828,
       "ttft_p50_ci": [
        1206.5403335727751,
        1643.04208336398
       ],
       "ttft_p90_ms": 1956.154000107199,
       "ttft_p99_ms": 3111.8962051905687,
       "ttft_max_ms": 3448.31270724535,
       "decode_tok_s_p50": 42.28035987771287,
       "decode_p50_ci": [
        41.78354401578727,
        43.5917662134289
       ],
       "decode_tok_s_p10": 39.56579400800948,
       "tpot_p99_ms": 27.815372351482136,
       "e2e_p50_s": 6.139101000037044,
       "e2e_p99_s": 8.503051417227834,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 1.0
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1516.2594583816826,
       "ttft_p50_ci": [
        1346.0141457617283,
        1667.0026876963675
       ],
       "ttft_p90_ms": 1823.626412358135,
       "ttft_p99_ms": 1878.0153765156865,
       "ttft_max_ms": 1893.1542504578829,
       "decode_tok_s_p50": 41.588659782748,
       "decode_p50_ci": [
        40.4205555137944,
        42.75559126792271
       ],
       "decode_tok_s_p10": 38.72899457172975,
       "tpot_p99_ms": 27.126696303056264,
       "e2e_p50_s": 6.325050145853311,
       "e2e_p99_s": 7.034565188903361,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 1.0
      }
     },
     "billing": {
      "requests": 445,
      "prompt_delta_median": 0,
      "completion_ratio_median": 1.04,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999997,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 445,
      "failed_attempts_by_status": {},
      "final_failures_by_status": {}
     }
    },
    "deepinfra/fp8": {
     "provider": "DeepInfra",
     "declared_quantization": "fp8",
     "price_per_m": {
      "input": 1.3,
      "output": 2.6,
      "cache_read": 0.09999999999999999
     },
     "context_length": 1048576,
     "gsm8k": {
      "answered": 200,
      "correct": 196,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 98.0,
      "wilson95": [
       94.97121344948062,
       99.21956998236058
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 107,
      "failed": 0,
      "truncated": 2,
      "acc_pct": 71.33333333333333,
      "wilson95": [
       63.63493407951355,
       77.96629731693578
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 9,
      "unsupported_by_endpoint": 1
     },
     "cache": {
      "hits": 3,
      "warm_ok": 3,
      "cached_share": [
       0.947,
       0.947,
       0.947
      ],
      "prompt_cost_discount": [
       0.874,
       0.874,
       0.874
      ],
      "expected_discount": 0.874
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 33227,
       "ref_prompt_tokens": 33227
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1092.4987490288913,
       "ttft_p50_ci": [
        1003.986313007772,
        1253.7334794178605
       ],
       "ttft_p90_ms": 1640.380287729204,
       "ttft_p99_ms": 3137.869229447096,
       "ttft_max_ms": 3363.3972918614745,
       "decode_tok_s_p50": 37.80190469454956,
       "decode_p50_ci": [
        31.694256498164766,
        41.28479366226424
       ],
       "decode_tok_s_p10": 25.700710747451247,
       "tpot_p99_ms": 43.5861081802684,
       "e2e_p50_s": 6.374834041111171,
       "e2e_p99_s": 10.251418909225613,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.47
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1231.6706459969282,
       "ttft_p50_ci": [
        1145.0736662372947,
        1433.7883959524333
       ],
       "ttft_p90_ms": 1688.9264257624745,
       "ttft_p99_ms": 2385.752486549318,
       "ttft_max_ms": 2404.165832325816,
       "decode_tok_s_p50": 42.18286951256111,
       "decode_p50_ci": [
        35.77658688956185,
        43.93304257303256
       ],
       "decode_tok_s_p10": 24.849268657383192,
       "tpot_p99_ms": 43.508028716887885,
       "e2e_p50_s": 6.119721458759159,
       "e2e_p99_s": 9.938479476151988,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.465
      }
     },
     "billing": {
      "requests": 444,
      "prompt_delta_median": 0.0,
      "completion_ratio_median": 1.0112359550561798,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999998,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 445,
      "failed_attempts_by_status": {
       "404": 1
      },
      "final_failures_by_status": {
       "404": 1
      }
     }
    },
    "coreweave/fp8": {
     "provider": "CoreWeave",
     "declared_quantization": "fp8",
     "price_per_m": {
      "input": 1.31,
      "output": 3.9600000000000004,
      "cache_read": 0.044
     },
     "context_length": 1048576,
     "gsm8k": {
      "answered": 200,
      "correct": 196,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 98.0,
      "wilson95": [
       94.97121344948062,
       99.21956998236058
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 109,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 72.66666666666667,
      "wilson95": [
       65.03534444650609,
       79.16596391222133
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 9,
      "schema_valid_other_tool": 1
     },
     "cache": {
      "hits": 2,
      "warm_ok": 3,
      "cached_share": [
       0.0,
       0.892,
       0.892
      ],
      "prompt_cost_discount": [
       0.862,
       0.862
      ],
      "expected_discount": 0.862
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 33226,
       "ref_prompt_tokens": 33226
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 411.7522705346346,
       "ttft_p50_ci": [
        273.9261668175459,
        507.7008339576423
       ],
       "ttft_p90_ms": 686.0336704179645,
       "ttft_p99_ms": 1435.9071985911573,
       "ttft_max_ms": 1818.434457294643,
       "decode_tok_s_p50": 76.31640454578019,
       "decode_p50_ci": [
        72.31201981157434,
        84.98311205414612
       ],
       "decode_tok_s_p10": 56.26971605805155,
       "tpot_p99_ms": 29.35764389318512,
       "e2e_p50_s": 3.029624895658344,
       "e2e_p99_s": 6.294030140973628,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.4825
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 463.16631278023124,
       "ttft_p50_ci": [
        310.3786874562502,
        565.0013540871441
       ],
       "ttft_p90_ms": 879.0554506704211,
       "ttft_p99_ms": 1228.5932520776985,
       "ttft_max_ms": 1363.3555425330997,
       "decode_tok_s_p50": 72.19054374465117,
       "decode_p50_ci": [
        65.9544100772795,
        77.21121161623579
       ],
       "decode_tok_s_p10": 56.58496031934091,
       "tpot_p99_ms": 20.28303729576157,
       "e2e_p50_s": 3.2794686453416944,
       "e2e_p99_s": 4.355362408487126,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.4725
      }
     },
     "billing": {
      "requests": 445,
      "prompt_delta_median": 0,
      "completion_ratio_median": 1.0112997957099081,
      "cost_ratio_median": 0.9999999999999999,
      "cost_ratio_minmax": [
       0.9999999999999997,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 445,
      "failed_attempts_by_status": {},
      "final_failures_by_status": {}
     }
    }
   }
  },
  "glm-5.3": {
   "openrouter_id": "z-ai/glm-5.3",
   "reasoning": {
    "effort": "low"
   },
   "endpoints": {
    "wafer/us": {
     "provider": "Wafer",
     "declared_quantization": "fp4",
     "price_per_m": {
      "input": 0.0699,
      "output": 1.4,
      "cache_read": 0.0689
     },
     "context_length": 1048576,
     "gsm8k": {
      "answered": 200,
      "correct": 197,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 98.5,
      "wilson95": [
       95.68335194437212,
       99.48858548155077
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 126,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 84.0,
      "wilson95": [
       77.29589426564779,
       89.00606827244333
      ]
     },
     "cruxeval_o_800": {
      "answered": 800,
      "correct": 704,
      "failed": 0,
      "acc_pct": 88.0,
      "wilson95": [
       85.56458643323526,
       90.07220569098425
      ]
     },
     "tool_calls": {
      "valid": 10
     },
     "cache": {
      "hits": 3,
      "warm_ok": 3,
      "cached_share": [
       0.989,
       0.989,
       0.989
      ],
      "prompt_cost_discount": [
       0.014,
       0.014,
       0.014
      ],
      "expected_discount": 0.014
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32424,
       "ref_prompt_tokens": 32424
      },
      "ctx820000": {
       "ok": false,
       "found": false,
       "prompt_tokens": null,
       "ref_prompt_tokens": 914133
      },
      "ctx560000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 624303,
       "ref_prompt_tokens": 624303
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 239.24143752083182,
       "ttft_p50_ci": [
        234.6468954347074,
        268.1528339162469
       ],
       "ttft_p90_ms": 422.13969565927994,
       "ttft_p99_ms": 533.7320936005563,
       "ttft_max_ms": 543.208833783865,
       "decode_tok_s_p50": 127.20901157803722,
       "decode_p50_ci": [
        112.60739521460229,
        137.187446543269
       ],
       "decode_tok_s_p10": 85.20771790734767,
       "tpot_p99_ms": 16.345079132641228,
       "e2e_p50_s": 1.8823789991438389,
       "e2e_p99_s": 3.521599562652409,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.5
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 247.2311044111848,
       "ttft_p50_ci": [
        236.5862913429737,
        266.9716249220073
       ],
       "ttft_p90_ms": 437.7070788294077,
       "ttft_p99_ms": 2286.2383866403243,
       "ttft_max_ms": 3142.3037918284535,
       "decode_tok_s_p50": 120.85277959420529,
       "decode_p50_ci": [
        112.27194807577587,
        132.8959505853309
       ],
       "decode_tok_s_p10": 98.28632714243767,
       "tpot_p99_ms": 31.306708746463354,
       "e2e_p50_s": 1.9652008963748813,
       "e2e_p99_s": 6.512384990733116,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.52
      }
     },
     "billing": {
      "requests": 446,
      "prompt_delta_median": 0.0,
      "completion_ratio_median": 1.0285714285714285,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999998,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 1097,
      "failed_attempts_by_status": {
       "400": 1
      },
      "final_failures_by_status": {
       "400": 1
      }
     }
    },
    "baidu/fp8": {
     "provider": "Baidu",
     "declared_quantization": "fp8",
     "price_per_m": {
      "input": 0.28,
      "output": 0.88,
      "cache_read": 0.052000000000000005
     },
     "context_length": 1048576,
     "gsm8k": {
      "answered": 196,
      "correct": 193,
      "failed": 4,
      "truncated": 0,
      "acc_pct": 98.46938775510205,
      "wilson95": [
       95.59717816475863,
       99.47812147255412
      ]
     },
     "cruxeval_o": {
      "answered": 148,
      "correct": 136,
      "failed": 2,
      "truncated": 0,
      "acc_pct": 91.89189189189189,
      "wilson95": [
       86.36284040808162,
       95.30120949655586
      ]
     },
     "cruxeval_o_800": {
      "answered": 681,
      "correct": 616,
      "failed": 119,
      "acc_pct": 90.45521292217327,
      "wilson95": [
       88.0159106031136,
       92.44065044691608
      ]
     },
     "tool_calls": {
      "valid": 9,
      "unsupported_by_endpoint": 1
     },
     "cache": {
      "hits": 2,
      "warm_ok": 2,
      "cached_share": [
       0.98,
       0.98
      ],
      "prompt_cost_discount": [
       0.798,
       0.798
      ],
      "expected_discount": 0.798
     },
     "context_probes": {
      "ctx29000": {
       "ok": false,
       "found": false,
       "prompt_tokens": null,
       "ref_prompt_tokens": 32423
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 30,
       "failed": 10,
       "retried": 21,
       "ttft_p50_ms": 1052.9745002277195,
       "ttft_p50_ci": [
        898.6189574934542,
        1196.9644580967724
       ],
       "ttft_p90_ms": 1613.733616378159,
       "ttft_p99_ms": 2750.710987783969,
       "ttft_max_ms": 2878.8652503862977,
       "decode_tok_s_p50": 79.34535677954628,
       "decode_p50_ci": [
        76.62724478690396,
        81.13264533083178
       ],
       "decode_tok_s_p10": 73.67292615353485,
       "tpot_p99_ms": 14.071747220434887,
       "e2e_p50_s": 3.5440414999611676,
       "e2e_p99_s": 5.120903036491946,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.4575
      },
      "concurrent": {
       "requests": 40,
       "ok": 28,
       "failed": 12,
       "retried": 24,
       "ttft_p50_ms": 1047.7449791505933,
       "ttft_p50_ci": [
        967.5943329930305,
        1199.57633363083
       ],
       "ttft_p90_ms": 1565.733917336911,
       "ttft_p99_ms": 3426.5684317611167,
       "ttft_max_ms": 3890.2874579653144,
       "decode_tok_s_p50": 79.67452803927411,
       "decode_p50_ci": [
        77.35359736961263,
        83.8817936858804
       ],
       "decode_tok_s_p10": 71.97355183040509,
       "tpot_p99_ms": 14.772218372579196,
       "e2e_p50_s": 3.5928865619935095,
       "e2e_p99_s": 5.712684965375812,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.45
      }
     },
     "billing": {
      "requests": 417,
      "prompt_delta_median": 0,
      "completion_ratio_median": 1.0325224748810153,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999998,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 1168,
      "failed_attempts_by_status": {
       "429": 1566,
       "404": 2
      },
      "final_failures_by_status": {
       "429": 216,
       "404": 2
      }
     }
    },
    "deepinfra/fp4": {
     "provider": "DeepInfra",
     "declared_quantization": "fp4",
     "price_per_m": {
      "input": 0.5625,
      "output": 2.5,
      "cache_read": 0.125
     },
     "context_length": 1048576,
     "gsm8k": {
      "answered": 200,
      "correct": 195,
      "failed": 0,
      "truncated": 1,
      "acc_pct": 97.5,
      "wilson95": [
       94.2820855848383,
       98.92754385292123
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 134,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 89.33333333333333,
      "wilson95": [
       83.37524992356727,
       93.32702046363619
      ]
     },
     "cruxeval_o_800": {
      "answered": 800,
      "correct": 714,
      "failed": 0,
      "acc_pct": 89.25,
      "wilson95": [
       86.91291389668137,
       91.21193060004533
      ]
     },
     "tool_calls": {
      "valid": 10
     },
     "cache": {
      "hits": 3,
      "warm_ok": 3,
      "cached_share": [
       0.98,
       0.98,
       0.98
      ],
      "prompt_cost_discount": [
       0.762,
       0.762,
       0.762
      ],
      "expected_discount": 0.762
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32425,
       "ref_prompt_tokens": 32425
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 354.4818121008575,
       "ttft_p50_ci": [
        312.4288124963641,
        403.6179375834763
       ],
       "ttft_p90_ms": 1129.9425575882199,
       "ttft_p99_ms": 2392.8862754628058,
       "ttft_max_ms": 2802.3050418123603,
       "decode_tok_s_p50": 74.5264162047261,
       "decode_p50_ci": [
        69.73017453305502,
        77.51520319737315
       ],
       "decode_tok_s_p10": 54.77697844033624,
       "tpot_p99_ms": 39.254316408374336,
       "e2e_p50_s": 3.2671199166215956,
       "e2e_p99_s": 8.223352817706761,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.4325
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 464.2697083763778,
       "ttft_p50_ci": [
        381.88460376113653,
        690.5570831149817
       ],
       "ttft_p90_ms": 1344.827582873404,
       "ttft_p99_ms": 3375.66002245061,
       "ttft_max_ms": 4144.100666046143,
       "decode_tok_s_p50": 65.24209934976203,
       "decode_p50_ci": [
        60.614083176283366,
        75.79572227261747
       ],
       "decode_tok_s_p10": 46.52925606541938,
       "tpot_p99_ms": 32.685163025720854,
       "e2e_p50_s": 3.5420641037635505,
       "e2e_p99_s": 8.298905469710007,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.445
      }
     },
     "billing": {
      "requests": 445,
      "prompt_delta_median": 0,
      "completion_ratio_median": 1.0317460317460319,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       1.0,
       1.0
      ]
     },
     "errors": {
      "requests": 1095,
      "failed_attempts_by_status": {},
      "final_failures_by_status": {}
     }
    },
    "novita/fp8": {
     "provider": "Novita",
     "declared_quantization": "fp8",
     "price_per_m": {
      "input": 0.63,
      "output": 1.9800000000000002,
      "cache_read": 0.117
     },
     "context_length": 1048576,
     "gsm8k": {
      "answered": 200,
      "correct": 198,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 99.0,
      "wilson95": [
       96.42775148038206,
       99.72533993962251
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 127,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 84.66666666666667,
      "wilson95": [
       78.04188029480453,
       89.56012072442564
      ]
     },
     "cruxeval_o_800": {
      "answered": 800,
      "correct": 705,
      "failed": 0,
      "acc_pct": 88.125,
      "wilson95": [
       85.69905192217531,
       90.1865454392949
      ]
     },
     "tool_calls": {
      "valid": 9,
      "unsupported_by_endpoint": 1
     },
     "cache": {
      "hits": 2,
      "warm_ok": 3,
      "cached_share": [
       0.994,
       0.0,
       0.907
      ],
      "prompt_cost_discount": [
       0.809,
       0.739
      ],
      "expected_discount": 0.809
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32423,
       "ref_prompt_tokens": 32423
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 2448.433500248939,
       "ttft_p50_ci": [
        2204.035207629204,
        2840.2036665938795
       ],
       "ttft_p90_ms": 7443.943724595022,
       "ttft_p99_ms": 31548.429698152464,
       "ttft_max_ms": 34331.96004200727,
       "decode_tok_s_p50": 55.12575416788411,
       "decode_p50_ci": [
        51.843269013292925,
        72.68978857074825
       ],
       "decode_tok_s_p10": 47.434412207615146,
       "tpot_p99_ms": 22.363379067792813,
       "e2e_p50_s": 6.375302062835544,
       "e2e_p99_s": 33.697743968954306,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.225
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 2460.5259792879224,
       "ttft_p50_ci": [
        2116.6835823096335,
        2647.856270428747
       ],
       "ttft_p90_ms": 2904.9192374572162,
       "ttft_p99_ms": 5685.232115369288,
       "ttft_max_ms": 6609.962707385421,
       "decode_tok_s_p50": 54.87817628733472,
       "decode_p50_ci": [
        51.378487350371955,
        75.42602746615793
       ],
       "decode_tok_s_p10": 47.39735917521553,
       "tpot_p99_ms": 23.032413461999077,
       "e2e_p50_s": 6.3439305000938475,
       "e2e_p99_s": 8.763990171840414,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.225
      }
     },
     "billing": {
      "requests": 444,
      "prompt_delta_median": 0.0,
      "completion_ratio_median": 1.0227272727272727,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999997,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 1095,
      "failed_attempts_by_status": {
       "404": 1
      },
      "final_failures_by_status": {
       "404": 1
      }
     }
    },
    "coreweave/nvfp4": {
     "provider": "CoreWeave",
     "declared_quantization": "nvfp4",
     "price_per_m": {
      "input": 0.75,
      "output": 3.0,
      "cache_read": 0.19
     },
     "context_length": 1048576,
     "gsm8k": {
      "answered": 200,
      "correct": 198,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 99.0,
      "wilson95": [
       96.42775148038206,
       99.72533993962251
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 136,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 90.66666666666667,
      "wilson95": [
       84.9434822154413,
       94.35886513404022
      ]
     },
     "cruxeval_o_800": {
      "answered": 800,
      "correct": 713,
      "failed": 0,
      "acc_pct": 89.125,
      "wilson95": [
       86.77769970779538,
       91.09833955168062
      ]
     },
     "tool_calls": {
      "valid": 10
     },
     "cache": {
      "hits": 2,
      "warm_ok": 3,
      "cached_share": [
       0.98,
       0.98,
       0.0
      ],
      "prompt_cost_discount": [
       0.731,
       0.731
      ],
      "expected_discount": 0.731
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32423,
       "ref_prompt_tokens": 32423
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 467.723208013922,
       "ttft_p50_ci": [
        452.4967912584543,
        530.2761453203857
       ],
       "ttft_p90_ms": 686.9279120117427,
       "ttft_p99_ms": 1122.2923208866266,
       "ttft_max_ms": 1318.350374698639,
       "decode_tok_s_p50": 160.4481700291082,
       "decode_p50_ci": [
        138.75671287417853,
        169.0387008592498
       ],
       "decode_tok_s_p10": 119.62915713450731,
       "tpot_p99_ms": 12.469231825387432,
       "e2e_p50_s": 1.7610152708366513,
       "e2e_p99_s": 3.5182431478332723,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.4575
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 471.17864480242133,
       "ttft_p50_ci": [
        447.96572905033827,
        575.8068128488958
       ],
       "ttft_p90_ms": 791.4815253578128,
       "ttft_p99_ms": 3110.482392767443,
       "ttft_max_ms": 4041.58404096961,
       "decode_tok_s_p50": 153.28124108084995,
       "decode_p50_ci": [
        140.90799146217998,
        166.6108855291519
       ],
       "decode_tok_s_p10": 120.26009634844291,
       "tpot_p99_ms": 9.39167801520071,
       "e2e_p50_s": 1.90204693749547,
       "e2e_p99_s": 4.487436482524498,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.4525
      }
     },
     "billing": {
      "requests": 445,
      "prompt_delta_median": 0,
      "completion_ratio_median": 1.0307692307692307,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       0.9999999999999998,
       1.0000000000000002
      ]
     },
     "errors": {
      "requests": 1095,
      "failed_attempts_by_status": {},
      "final_failures_by_status": {}
     }
    }
   }
  },
  "qwen3.8-2.4t-a95b": {
   "openrouter_id": "qwen/qwen3.8-2.4t-a95b",
   "reasoning": {
    "effort": "low"
   },
   "endpoints": {
    "alibaba": {
     "provider": "Alibaba",
     "declared_quantization": "unknown",
     "price_per_m": {
      "input": 2.0,
      "output": 6.0,
      "cache_read": 0.25
     },
     "context_length": 1000000,
     "gsm8k": {
      "answered": 200,
      "correct": 198,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 99.0,
      "wilson95": [
       96.42775148038206,
       99.72533993962251
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 147,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 98.0,
      "wilson95": [
       94.28523341357584,
       99.31753722843516
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 8,
      "unsupported_by_endpoint": 1,
      "schema_valid_other_tool": 1
     },
     "cache": {
      "hits": 3,
      "warm_ok": 3,
      "cached_share": [
       0.93,
       0.93,
       0.93
      ],
      "prompt_cost_discount": [
       0.814,
       0.814,
       0.814
      ],
      "expected_discount": 0.814
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32465,
       "ref_prompt_tokens": 32456
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1377.2526243701577,
       "ttft_p50_ci": [
        1269.0960834734142,
        1484.2291874811053
       ],
       "ttft_p90_ms": 2224.5744838379323,
       "ttft_p99_ms": 2991.863486068323,
       "ttft_max_ms": 3178.2475411891937,
       "decode_tok_s_p50": 38.15199274390601,
       "decode_p50_ci": [
        36.95202185637537,
        39.35877398036364
       ],
       "decode_tok_s_p10": 34.8408539290516,
       "tpot_p99_ms": 30.557960356505955,
       "e2e_p50_s": 9.045434354338795,
       "e2e_p99_s": 11.992619669679552,
       "completion_tokens_median": 283.0,
       "chunks_per_token_median": 0.3044185699358113
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1454.3109787628055,
       "ttft_p50_ci": [
        1303.7722082808614,
        1584.9870834499598
       ],
       "ttft_p90_ms": 2198.539770487696,
       "ttft_p99_ms": 2390.94387575984,
       "ttft_max_ms": 2441.095291636884,
       "decode_tok_s_p50": 39.14741146141537,
       "decode_p50_ci": [
        38.146808711288955,
        39.72993040836463
       ],
       "decode_tok_s_p10": 36.27943469572399,
       "tpot_p99_ms": 29.183536313023456,
       "e2e_p50_s": 8.64517516642809,
       "e2e_p99_s": 11.369138952884823,
       "completion_tokens_median": 277.5,
       "chunks_per_token_median": 0.3011815602177048
      }
     },
     "billing": {
      "requests": 444,
      "prompt_delta_median": 9.0,
      "completion_ratio_median": 1.0115607365607366,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       1.0,
       1.0
      ]
     },
     "errors": {
      "requests": 445,
      "failed_attempts_by_status": {
       "404": 1
      },
      "final_failures_by_status": {
       "404": 1
      }
     }
    },
    "novita": {
     "provider": "Novita",
     "declared_quantization": "unknown",
     "price_per_m": {
      "input": 2.0,
      "output": 6.0,
      "cache_read": 0.25
     },
     "context_length": 1000000,
     "gsm8k": {
      "answered": 200,
      "correct": 198,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 99.0,
      "wilson95": [
       96.42775148038206,
       99.72533993962251
      ]
     },
     "cruxeval_o": {
      "answered": 150,
      "correct": 146,
      "failed": 0,
      "truncated": 0,
      "acc_pct": 97.33333333333333,
      "wilson95": [
       93.34450938998245,
       98.95822277088952
      ]
     },
     "cruxeval_o_800": null,
     "tool_calls": {
      "valid": 8,
      "unsupported_by_endpoint": 1,
      "schema_valid_other_tool": 1
     },
     "cache": {
      "hits": 3,
      "warm_ok": 3,
      "cached_share": [
       0.93,
       0.93,
       0.93
      ],
      "prompt_cost_discount": [
       0.814,
       0.814,
       0.814
      ],
      "expected_discount": 0.814
     },
     "context_probes": {
      "ctx29000": {
       "ok": true,
       "found": true,
       "prompt_tokens": 32464,
       "ref_prompt_tokens": 32455
      }
     },
     "latency": {
      "sequential": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1476.4517294242978,
       "ttft_p50_ci": [
        1387.559374794364,
        1601.3331254944205
       ],
       "ttft_p90_ms": 1960.6568877585232,
       "ttft_p99_ms": 2343.1137496978045,
       "ttft_max_ms": 2343.4942923486233,
       "decode_tok_s_p50": 38.75577905769019,
       "decode_p50_ci": [
        37.63320724546067,
        39.92843596450027
       ],
       "decode_tok_s_p10": 35.980148875130574,
       "tpot_p99_ms": 30.557460270812687,
       "e2e_p50_s": 6.689832874573767,
       "e2e_p99_s": 8.022091003973037,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.305
      },
      "concurrent": {
       "requests": 40,
       "ok": 40,
       "failed": 0,
       "retried": 0,
       "ttft_p50_ms": 1502.2076456807554,
       "ttft_p50_ci": [
        1347.646979149431,
        1649.5116455480456
       ],
       "ttft_p90_ms": 1850.9934043511748,
       "ttft_p99_ms": 2900.4005649220194,
       "ttft_max_ms": 3276.9780000671744,
       "decode_tok_s_p50": 39.8746982015416,
       "decode_p50_ci": [
        39.26629771355465,
        40.510038934207145
       ],
       "decode_tok_s_p10": 36.66889622023206,
       "tpot_p99_ms": 29.523230340921472,
       "e2e_p50_s": 6.576411520596594,
       "e2e_p99_s": 8.015059181507675,
       "completion_tokens_median": 200.0,
       "chunks_per_token_median": 0.3025
      }
     },
     "billing": {
      "requests": 449,
      "prompt_delta_median": 9,
      "completion_ratio_median": 1.0140352605141336,
      "cost_ratio_median": 1.0,
      "cost_ratio_minmax": [
       1.0,
       1.0
      ]
     },
     "errors": {
      "requests": 451,
      "failed_attempts_by_status": {
       "404": 2
      },
      "final_failures_by_status": {
       "404": 2
      }
     }
    }
   }
  }
 }
}