{ "arms": [ { "cpu_concurrent_batch_slots": 0, "cpu_online_delivered_output_credit": 0, "delivered_output_tokens_per_s": 3596.179812053888, "energy_kwh_per_million_output_tokens": 5.190693358574182, "evidence_class": "modeled_engineering_scenario_not_measured", "gpu_count": 64, "gpu_racks": 2, "id": "h100_native", "label": "H100 · native stack", "latency_percentile": "not-established", "limiting_resource": "gpu_service", "lookup_conditional_acceptance": 0.8, "lookup_eligible_share": 0.25, "lookup_proposal_width": null, "lookup_ready_round_fraction": null, "mean_accepted_draft_prefix": null, "modeled_it_ac_kw": 67.2, "residual_miss_fraction": 0.3, "sidecar_count": 0, "slo_goodput": "not-established", "stack": "Dynamo + native MTP / local suffix lookup", "verified_correction_or_bonus_tokens": null, "warm_dram_sessions": 6327 }, { "cpu_concurrent_batch_slots": 64, "cpu_online_delivered_output_credit": 0, "delivered_output_tokens_per_s": 7124.3644517044795, "energy_kwh_per_million_output_tokens": 2.698096420042545, "evidence_class": "modeled_engineering_scenario_not_measured", "gpu_count": 64, "gpu_racks": 2, "id": "h100_software", "label": "H100 · Manifold software", "latency_percentile": "not-established", "limiting_resource": "gpu_service", "lookup_conditional_acceptance": 0.954271052839686, "lookup_eligible_share": 0.7648534540781986, "lookup_proposal_width": 16, "lookup_ready_round_fraction": 0.5, "mean_accepted_draft_prefix": 11, "modeled_it_ac_kw": 69.2, "residual_miss_fraction": 0.14, "sidecar_count": 0, "slo_goodput": "not-established", "stack": "Manifold software on the same host estate", "verified_correction_or_bonus_tokens": 1, "warm_dram_sessions": 6245 }, { "cpu_concurrent_batch_slots": 96, "cpu_online_delivered_output_credit": 0, "delivered_output_tokens_per_s": 10855.423542554068, "energy_kwh_per_million_output_tokens": 2.2111543664531643, "evidence_class": "modeled_engineering_scenario_not_measured", "gpu_count": 64, "gpu_racks": 2, "id": "h100_full", "label": "H100 · Manifold rack", "latency_percentile": "not-established", "limiting_resource": "gpu_service", "lookup_conditional_acceptance": 0.954271052839686, "lookup_eligible_share": 0.9070459328060372, "lookup_proposal_width": 16, "lookup_ready_round_fraction": 0.75, "mean_accepted_draft_prefix": 11, "modeled_it_ac_kw": 86.41086179694086, "residual_miss_fraction": 0.08, "sidecar_count": 1, "slo_goodput": "not-established", "stack": "Manifold software + Manifold rack (CPU-only Turin)", "verified_correction_or_bonus_tokens": 1, "warm_dram_sessions": 31466 }, { "cpu_concurrent_batch_slots": 0, "cpu_online_delivered_output_credit": 0, "delivered_output_tokens_per_s": 8552.533741235238, "energy_kwh_per_million_output_tokens": 3.1179843860885152, "evidence_class": "modeled_engineering_scenario_not_measured", "gpu_count": 64, "gpu_racks": 2, "id": "b200_native", "label": "B200 FP8 · native stack", "latency_percentile": "not-established", "limiting_resource": "gpu_service", "lookup_conditional_acceptance": 0.8, "lookup_eligible_share": 0.25, "lookup_proposal_width": null, "lookup_ready_round_fraction": null, "mean_accepted_draft_prefix": null, "modeled_it_ac_kw": 96.0, "residual_miss_fraction": 0.3, "sidecar_count": 0, "slo_goodput": "not-established", "stack": "Dynamo + native MTP / local suffix lookup", "verified_correction_or_bonus_tokens": null, "warm_dram_sessions": 11266 } ], "as_of": "2026-09-05", "assumption_classes": { "derived": "state bytes, timing residuals, component power, topology closure, service and resource admission arithmetic", "not_established": [ "tail latency percentiles", "latency-SLO goodput", "qualified CPU drafter", "B200 TP4/FP8 measured speed", "complete Flash tier integration", "usable draft-round frequency" ], "planning_assumption": "all operating values not explicitly classified as published or derived, including miss, eligibility, acceptance, efficiency, transfer and CPU-achievement inputs", "published": "model geometry, engine layout, timing notes, hardware specifications and compatibility limits identified by P01-P13" }, "calculated_state": { "bf16_width_16_bytes": 1836900352, "bf16_width_5_bytes": 1818517504, "bf16_width_8_bytes": 1823531008, "components_width_16": { "compressed_index": 51904512, "convolution": 31752192, "fp32_kda": 142606336, "growing_mla": 1610612736, "tail": 24576 } }, "calibration": { "b200": { "additive_residual_seconds": 0.00809526010608561, "anchor_seconds": 0.0191614, "calibration_batch": 16, "calibration_context_tokens": 8192, "calibration_draft_width": 5, "calibration_kv_element_bytes": 2, "calibration_tp": 8, "calibration_verification_queries": 6, "derivation": "anchor_seconds - raw_verifier_seconds - raw_mtp_seconds; verifier queries=k+1", "evidence_class": "unmeasured_extrapolation", "raw_mtp_seconds": 0.0015345379876923076, "raw_seconds": 0.011066139893914388, "raw_verifier_seconds": 0.00953160190622208, "simulated_accepted_length": 2.98 }, "h100": { "additive_residual_seconds": 0.005387104730950717, "anchor_seconds": 0.030515200000000003, "calibration_batch": 16, "calibration_context_tokens": 8192, "calibration_draft_width": 5, "calibration_kv_element_bytes": 2, "calibration_tp": 8, "calibration_verification_queries": 6, "derivation": "anchor_seconds - raw_verifier_seconds - raw_mtp_seconds; verifier queries=k+1", "evidence_class": "derived_from_reported_timing", "raw_mtp_seconds": 0.003231732507921929, "raw_seconds": 0.025128095269049286, "raw_verifier_seconds": 0.021896362761127358, "simulated_accepted_length": 2.98 } }, "countercases": { "cold_bootstrap": { "arms": [ { "bootstrap": { "amortization_hours": 720, "amortized_bandwidth_bytes_per_s": 0.0, "amortized_compute_flops_per_s": 0.0, "amortized_external_transfer_bytes_per_s": 0.0, "bandwidth_bytes": 0.0, "compute_flops": 0, "recurring_admission_fraction": 0.0, "required_amortized_rates": { "bandwidth_bytes_per_s": 0.0, "compute_flops_per_s": 0.0, "external_transfer_bytes_per_s": 0.0 }, "shadow_sessions": 0, "time_seconds_at_25_percent_reservation": null, "time_seconds_at_full_reservation": null, "tokens": 0 }, "id": "h100_native" }, { "bootstrap": { "amortization_hours": 720, "amortized_bandwidth_bytes_per_s": 129846643.51851852, "amortized_compute_flops_per_s": 1422222222.2222223, "amortized_external_transfer_bytes_per_s": 62692.9012345679, "bandwidth_bytes": 336562500000000.0, "compute_flops": 3686400000000000, "recurring_admission_fraction": 1.0, "required_amortized_rates": { "bandwidth_bytes_per_s": 129846643.51851852, "compute_flops_per_s": 1422222222.2222223, "external_transfer_bytes_per_s": 62692.9012345679 }, "shadow_sessions": 625, "time_seconds_at_25_percent_reservation": 1051.7578125, "time_seconds_at_full_reservation": 262.939453125, "tokens": 2560000 }, "id": "h100_software" }, { "bootstrap": { "amortization_hours": 720, "amortized_bandwidth_bytes_per_s": 6232638888.888889, "amortized_compute_flops_per_s": 68266666666.666664, "amortized_external_transfer_bytes_per_s": 3009259.2592592593, "bandwidth_bytes": 1.6155e+16, "compute_flops": 176947200000000000, "recurring_admission_fraction": 1.0, "required_amortized_rates": { "bandwidth_bytes_per_s": 6232638888.888889, "compute_flops_per_s": 68266666666.666664, "external_transfer_bytes_per_s": 3009259.2592592593 }, "shadow_sessions": 30000, "time_seconds_at_25_percent_reservation": 14745.6, "time_seconds_at_full_reservation": 3686.4, "tokens": 122880000 }, "id": "h100_full" }, { "bootstrap": { "amortization_hours": 720, "amortized_bandwidth_bytes_per_s": 0.0, "amortized_compute_flops_per_s": 0.0, "amortized_external_transfer_bytes_per_s": 0.0, "bandwidth_bytes": 0.0, "compute_flops": 0, "recurring_admission_fraction": 0.0, "required_amortized_rates": { "bandwidth_bytes_per_s": 0.0, "compute_flops_per_s": 0.0, "external_transfer_bytes_per_s": 0.0 }, "shadow_sessions": 0, "time_seconds_at_25_percent_reservation": null, "time_seconds_at_full_reservation": null, "tokens": 0 }, "id": "b200_native" } ], "headline": null, "online_output_effect": 0 }, "cpu_supply_zero": { "arms": [ { "delivered_output_tokens_per_s": 3596.179812053888, "energy_kwh_per_million_output_tokens": 5.190693358574182, "id": "h100_native" }, { "delivered_output_tokens_per_s": 7124.3644517044795, "energy_kwh_per_million_output_tokens": 2.698096420042545, "id": "h100_software" }, { "delivered_output_tokens_per_s": 10855.423542554068, "energy_kwh_per_million_output_tokens": 2.2111543664531643, "id": "h100_full" }, { "delivered_output_tokens_per_s": 8552.533741235238, "energy_kwh_per_million_output_tokens": 3.1179843860885152, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 3.018598654652423, "energy_saving_fraction": 0.5740156056799818, "warm_dram_capacity_ratio": 4.973289078552236 } }, "equal_8_percent_misses": { "arms": [ { "delivered_output_tokens_per_s": 7645.955050732498, "energy_kwh_per_million_output_tokens": 2.4413780283574074, "id": "h100_native" }, { "delivered_output_tokens_per_s": 9980.478626862565, "energy_kwh_per_million_output_tokens": 1.9259820035569641, "id": "h100_software" }, { "delivered_output_tokens_per_s": 10855.423542554068, "energy_kwh_per_million_output_tokens": 2.2111543664531643, "id": "h100_full" }, { "delivered_output_tokens_per_s": 24376.815454743883, "energy_kwh_per_million_output_tokens": 1.0939356174794015, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 1.4197603138556374, "energy_saving_fraction": 0.09430070199293994, "warm_dram_capacity_ratio": 4.973289078552236 } }, "equal_lookup_profiles": { "arms": [ { "delivered_output_tokens_per_s": 3744.0820858725815, "energy_kwh_per_million_output_tokens": 4.98564567724115, "id": "h100_native" }, { "delivered_output_tokens_per_s": 6250.665669548049, "energy_kwh_per_million_output_tokens": 3.075228021851963, "id": "h100_software" }, { "delivered_output_tokens_per_s": 8481.93568184541, "energy_kwh_per_million_output_tokens": 2.8298985121040885, "id": "h100_full" }, { "delivered_output_tokens_per_s": 8645.719595543484, "energy_kwh_per_million_output_tokens": 3.084377925050015, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 2.265424605365895, "energy_saving_fraction": 0.43239076835680046, "warm_dram_capacity_ratio": 4.973289078552236 } }, "native_miss_10": { "arms": [ { "delivered_output_tokens_per_s": 6935.888670589221, "energy_kwh_per_million_output_tokens": 2.691315785649842, "id": "h100_native" }, { "delivered_output_tokens_per_s": 7124.3644517044795, "energy_kwh_per_million_output_tokens": 2.698096420042545, "id": "h100_software" }, { "delivered_output_tokens_per_s": 10855.423542554068, "energy_kwh_per_million_output_tokens": 2.2111543664531643, "id": "h100_full" }, { "delivered_output_tokens_per_s": 8552.533741235238, "energy_kwh_per_million_output_tokens": 3.1179843860885152, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 1.5651092539280729, "energy_saving_fraction": 0.17841140075680062, "warm_dram_capacity_ratio": 4.973289078552236 } }, "native_miss_20": { "arms": [ { "delivered_output_tokens_per_s": 4736.524996382602, "energy_kwh_per_million_output_tokens": 3.941004572112013, "id": "h100_native" }, { "delivered_output_tokens_per_s": 7124.3644517044795, "energy_kwh_per_million_output_tokens": 2.698096420042545, "id": "h100_software" }, { "delivered_output_tokens_per_s": 10855.423542554068, "energy_kwh_per_million_output_tokens": 2.2111543664531643, "id": "h100_full" }, { "delivered_output_tokens_per_s": 8552.533741235238, "energy_kwh_per_million_output_tokens": 3.1179843860885152, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 2.291853954290248, "energy_saving_fraction": 0.43893636102325284, "warm_dram_capacity_ratio": 4.973289078552236 } }, "native_miss_30": { "arms": [ { "delivered_output_tokens_per_s": 3596.179812053888, "energy_kwh_per_million_output_tokens": 5.190693358574182, "id": "h100_native" }, { "delivered_output_tokens_per_s": 7124.3644517044795, "energy_kwh_per_million_output_tokens": 2.698096420042545, "id": "h100_software" }, { "delivered_output_tokens_per_s": 10855.423542554068, "energy_kwh_per_million_output_tokens": 2.2111543664531643, "id": "h100_full" }, { "delivered_output_tokens_per_s": 8552.533741235238, "energy_kwh_per_million_output_tokens": 3.1179843860885152, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 3.018598654652423, "energy_saving_fraction": 0.5740156056799818, "warm_dram_capacity_ratio": 4.973289078552236 } }, "native_miss_40": { "arms": [ { "delivered_output_tokens_per_s": 2898.378736897343, "energy_kwh_per_million_output_tokens": 6.440382145036353, "id": "h100_native" }, { "delivered_output_tokens_per_s": 7124.3644517044795, "energy_kwh_per_million_output_tokens": 2.698096420042545, "id": "h100_software" }, { "delivered_output_tokens_per_s": 10855.423542554068, "energy_kwh_per_million_output_tokens": 2.2111543664531643, "id": "h100_full" }, { "delivered_output_tokens_per_s": 8552.533741235238, "energy_kwh_per_million_output_tokens": 3.1179843860885152, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 3.7453433550145983, "energy_saving_fraction": 0.6566734214432732, "warm_dram_capacity_ratio": 4.973289078552236 } }, "native_miss_50": { "arms": [ { "delivered_output_tokens_per_s": 2427.372495383367, "energy_kwh_per_million_output_tokens": 7.690070931498523, "id": "h100_native" }, { "delivered_output_tokens_per_s": 7124.3644517044795, "energy_kwh_per_million_output_tokens": 2.698096420042545, "id": "h100_software" }, { "delivered_output_tokens_per_s": 10855.423542554068, "energy_kwh_per_million_output_tokens": 2.2111543664531643, "id": "h100_full" }, { "delivered_output_tokens_per_s": 8552.533741235238, "energy_kwh_per_million_output_tokens": 3.1179843860885152, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 4.472088055376774, "energy_saving_fraction": 0.712466323633469, "warm_dram_capacity_ratio": 4.973289078552236 } }, "no_lookup": { "arms": [ { "delivered_output_tokens_per_s": 3563.8735004886403, "energy_kwh_per_million_output_tokens": 5.237746700074313, "id": "h100_native" }, { "delivered_output_tokens_per_s": 5764.075049785658, "energy_kwh_per_million_output_tokens": 3.3348320513170653, "id": "h100_software" }, { "delivered_output_tokens_per_s": 7610.174682172183, "energy_kwh_per_million_output_tokens": 3.1540691466711106, "id": "h100_full" }, { "delivered_output_tokens_per_s": 8536.167896411143, "energy_kwh_per_million_output_tokens": 3.123962296697341, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 2.1353661068858805, "energy_saving_fraction": 0.397819458007322, "warm_dram_capacity_ratio": 4.973289078552236 } }, "prefill_minus_50_percent": { "arms": [ { "delivered_output_tokens_per_s": 2058.126575309556, "energy_kwh_per_million_output_tokens": 9.06973695913677, "id": "h100_native" }, { "delivered_output_tokens_per_s": 4164.022042303043, "energy_kwh_per_million_output_tokens": 4.616263321120838, "id": "h100_software" }, { "delivered_output_tokens_per_s": 6604.908504984438, "energy_kwh_per_million_output_tokens": 3.634118042316994, "id": "h100_full" }, { "delivered_output_tokens_per_s": 4474.488336469389, "energy_kwh_per_million_output_tokens": 5.959713080335817, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 3.2091847917521865, "energy_saving_fraction": 0.5993138435336853, "warm_dram_capacity_ratio": 4.973289078552236 } }, "prefill_plus_50_percent": { "arms": [ { "delivered_output_tokens_per_s": 4789.175174383771, "energy_kwh_per_million_output_tokens": 3.89767882505332, "id": "h100_native" }, { "delivered_output_tokens_per_s": 9337.034358163688, "energy_kwh_per_million_output_tokens": 2.058707453016447, "id": "h100_software" }, { "delivered_output_tokens_per_s": 13819.990301263806, "energy_kwh_per_million_output_tokens": 1.7368331411652211, "id": "h100_full" }, { "delivered_output_tokens_per_s": 12284.588843953565, "energy_kwh_per_million_output_tokens": 2.1707414880060814, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 2.8856723335541887, "energy_saving_fraction": 0.554392955622386, "warm_dram_capacity_ratio": 4.973289078552236 } }, "timing_residual_minus_50_percent": { "arms": [ { "delivered_output_tokens_per_s": 3668.1608566202744, "energy_kwh_per_million_output_tokens": 5.08883535818152, "id": "h100_native" }, { "delivered_output_tokens_per_s": 7263.1725893580015, "energy_kwh_per_million_output_tokens": 2.6465324877983236, "id": "h100_software" }, { "delivered_output_tokens_per_s": 11111.229777684737, "energy_kwh_per_million_output_tokens": 2.160248473487914, "id": "h100_full" }, { "delivered_output_tokens_per_s": 8629.365920149794, "energy_kwh_per_million_output_tokens": 3.0902231882877174, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 3.0291010160122225, "energy_saving_fraction": 0.5754925594095321, "warm_dram_capacity_ratio": 4.973289078552236 } }, "timing_residual_plus_50_percent": { "arms": [ { "delivered_output_tokens_per_s": 3527.0758176322543, "energy_kwh_per_million_output_tokens": 5.292391667156649, "id": "h100_native" }, { "delivered_output_tokens_per_s": 6990.762406795437, "energy_kwh_per_million_output_tokens": 2.7496603522867664, "id": "h100_software" }, { "delivered_output_tokens_per_s": 10611.130745026298, "energy_kwh_per_million_output_tokens": 2.262060259418415, "id": "h100_full" }, { "delivered_output_tokens_per_s": 8477.377515137703, "energy_kwh_per_million_output_tokens": 3.145626889807503, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 3.0084782107546553, "energy_saving_fraction": 0.5725826050524123, "warm_dram_capacity_ratio": 4.973289078552236 } }, "zero_misses": { "arms": [ { "delivered_output_tokens_per_s": 12948.333152184976, "energy_kwh_per_million_output_tokens": 1.4416269991876711, "id": "h100_native" }, { "delivered_output_tokens_per_s": 21441.500863795907, "energy_kwh_per_million_output_tokens": 0.8964961149095235, "id": "h100_software" }, { "delivered_output_tokens_per_s": 26044.353225834195, "energy_kwh_per_million_output_tokens": 0.9216207811990346, "id": "h100_full" }, { "delivered_output_tokens_per_s": 74504.98536253652, "energy_kwh_per_million_output_tokens": 0.3579178834397237, "id": "b200_native" } ], "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 2.011405863575523, "energy_saving_fraction": 0.3607078795566745, "warm_dram_capacity_ratio": 4.973289078552236 } } }, "diagnostics": { "cpu_neural_online_credit": 0, "diagnostic_scope": "nominal arms; independent tests additionally exercise boundaries and countercases", "energy_boundary": "modeled IT AC excluding facility PUE", "finite_values": true, "latency_percentiles": "not-established", "resource_conservation_checked": true, "slo_goodput": "not-established" }, "equations": { "accepted_length": "sum(alpha^j for j=0..k); includes one correction/bonus output, with alpha conditional per-position draft survival", "blended_decode_seconds_per_output": "(1-eligible)/best_native_rate + eligible/max(best_native_rate,best_lookup_rate); eligible is output-token share, not verification-round share", "cpu_background": "forecasts=lambda*z*recall/precision; input/feedback and all candidates charge compute and bandwidth; beta=min(memory,compute,bandwidth,remaining transfer admission); required bootstrap work is separate from admitted recurring usage; online output credit=0", "energy_kwh_per_million": "IT_AC_kW*1e6/(3600*delivered_output_tokens_per_s)", "episode_service": "new_input + true_miss*retained_context prefill, decoded output, and exposed successful-restore service; capacity is min(GPU service, transfer admissions, offered load)", "group_capacity": "BW_eff=TP*bandwidth_per_GPU*efficiency; F_eff=TP*compute_per_GPU*efficiency; q=k+1 includes bonus-token query", "logical_state_bytes": "12*512*kvbytes*C + 12*132*ceil(C/4) + 34*64*128^2*4 + 34*3*64*128*(3+k)*2 + 12*4*2*128*2", "mtp_seconds": "k*(7.5e9+B*TP*2048*512*kvbytes)/BW_eff + B*TP*132*ceil(C/4)/BW_eff + mtp_fixed; k=0 is zero", "recurrent_query_bytes": "2*KDAbytes + TP*(11*2048*512*kvbytes + 11*132*ceil(C/4))", "sidecar_power": "u=active/64; component board DC -> fan/distribution branch -> PSU AC + switches + management", "target_output_share": "r*(D+1) / ((1-r)*A_native + r*(D+1)); r is per-sequence ready-round fraction and D excludes the verified correction/bonus token", "touched_weight_bytes_per_group": "Wfixed + (Wtotal-Wfixed) * (1-(1-active_experts/experts)^(B*q))", "verify_seconds": "max((W+B*q*R)/BW_eff, B*q*45e9/F_eff) + fixed + additive_calibration_residual", "warm_capacity": "largest integer n with n*target_state*1.1 + ceil(n*shadow_fraction)*260e6 <= usable_DRAM - suffix_store - packed_worker_weights; population occupancy is separate" }, "evidence_class": "modeled_engineering_scenario", "headline": { "comparison": "h100_full / h100_native", "delivered_output_ratio": 3.018598654652423, "energy_saving_fraction": 0.5740156056799818, "warm_dram_capacity_ratio": 4.973289078552236 }, "inputs": { "arms": [ { "hardware": "h100", "id": "h100_native", "label": "H100 · native stack", "lookup_eligible_share": 0.25, "manifold": false, "residual_miss": 0.3, "sidecar": false, "stack": "Dynamo + native MTP / local suffix lookup" }, { "hardware": "h100", "id": "h100_software", "label": "H100 · Manifold software", "lookup_proposal_width": 16, "lookup_ready_round_fraction": 0.5, "manifold": true, "mean_accepted_draft_prefix": 11, "residual_miss": 0.14, "sidecar": false, "stack": "Manifold software on the same host estate" }, { "hardware": "h100", "id": "h100_full", "label": "H100 · Manifold rack", "lookup_proposal_width": 16, "lookup_ready_round_fraction": 0.75, "manifold": true, "mean_accepted_draft_prefix": 11, "residual_miss": 0.08, "sidecar": true, "stack": "Manifold software + Manifold rack (CPU-only Turin)" }, { "hardware": "b200", "id": "b200_native", "label": "B200 FP8 · native stack", "lookup_eligible_share": 0.25, "manifold": false, "residual_miss": 0.3, "sidecar": false, "stack": "Dynamo + native MTP / local suffix lookup" } ], "as_of": "2026-09-05", "cpu": { "batch": 4, "bootstrap_amortization_hours": 720, "candidate_width": 8, "cold_shadow_fraction": 0.2, "context_tokens": 4096, "flops_per_token": 1440000000, "forecast_precision": 0.5, "forecast_recall": 0.5, "host_bandwidth_bytes_per_socket_s": 80000000000, "host_compute_flops_per_socket_s": 1000000000000, "host_increment_kw": 2.0, "host_maintained_episode_share": 0.1, "host_shadow_session_fraction": 0.1, "host_sockets": 16, "lookup_bytes_per_candidate": 512, "parameters": 600000000, "predicted_input_tokens": 256, "prefill_batch": 128, "sidecar_active_sockets": 24, "sidecar_bandwidth_bytes_per_socket_s": 200000000000, "sidecar_compute_flops_per_socket_s": 2000000000000, "sidecar_maintained_episode_share": 1.0, "sidecar_shadow_session_fraction": 1.0, "state_bytes_per_session": 260000000, "weight_bits": 2, "weight_packing_factor": 1.2 }, "evidence_class": "modeled_engineering_scenario", "gpu_model": { "active_experts": 8, "b200": { "bandwidth_bytes_per_gpu_s": 8000000000000, "base_estate_kw": 96.0, "batch": 64, "calibration_tpot_s": 0.00643, "fixed_s": 0.0012, "fp8_dense_flops_per_gpu_s": 4500000000000000, "hbm_bytes_per_gpu": 180000000000, "kv_element_bytes": 1, "max_node_kw": 14.3, "mtp_fixed_s": 0.0006, "prefill_tokens_per_group_s": 23108, "tp": 4 }, "calibration_accepted_length": 2.98, "calibration_batch": 16, "calibration_context_tokens": 8192, "compute_efficiency": 0.4, "experts": 288, "flops_per_query": 45000000000, "gpu_count": 64, "gpus_per_node": 8, "gpus_per_rack": 32, "h100": { "bandwidth_bytes_per_gpu_s": 3350000000000, "base_estate_kw": 67.2, "batch": 16, "calibration_tpot_s": 0.01024, "fixed_s": 0.002, "fp8_dense_flops_per_gpu_s": 1979000000000000, "hbm_bytes_per_gpu": 80000000000, "kv_element_bytes": 2, "max_node_kw": 10.2, "mtp_fixed_s": 0.001, "prefill_tokens_per_group_s": 23700, "tp": 8 }, "hbm_allocation_fraction": 0.9, "hbm_efficiency": 0.65, "lookup_batch_s": 0.0002, "lookup_max_width": 16, "mtp_max_width": 5, "mtp_sweep_bytes": 7500000000, "native_alpha": 0.8, "nonrouted_bytes": 9340000000, "resident_checkpoint_bytes": 328564998144, "trunk_sweep_bytes": 320000000000 }, "model_name": "GLM-5.3-Flash", "schema_version": 1, "sidecar": { "board": "Supermicro H13SSL-N R2.01", "board_count": 64, "board_other_w": 15.0, "breakout_cables": 72, "breakout_legs": 2, "cpu": "AMD EPYC 9355P", "cpu_active_w": 280.0, "cpu_park_w": 45.0, "cpu_vrm_efficiency": 0.92, "dimm_bytes": 96000000000, "dimms_per_board": 12, "distribution_efficiency": 0.95, "dram_active_w_per_board": 73.5, "dram_park_w_per_board": 31.5, "dual_homed_boards": 48, "external_host_count": 8, "external_ports_per_host": 2, "fabric_goodput_fraction": 0.7, "fan_active_fraction": 0.06, "fan_park_fraction": 0.032, "m_cells_w": 14.0, "management_w": 95.0, "memory_speed_mt_s": 6400, "nic": "Broadcom BCM957608-P2200GQF00", "nic_active_w": 15.0, "nic_count": 64, "nic_park_w": 14.0, "operating_socket_cap": 48, "port_bits_per_s": 200000000000, "psu_efficiency": 0.95, "psu_module_w": 3000, "psu_modules": 9, "single_homed_boards": 16, "switch": "Edgecore AS9716-32D", "switch_active_w": 400.0, "switch_cages_each": 32, "switch_count": 2, "switch_park_w": 350.0 }, "workload": { "context_tokens": 131072, "host_memory_bytes_per_node": 2000000000000, "host_restore_bytes_per_s": 50000000000, "new_input_tokens": 1024, "nvlink_broadcast_bytes_per_s": 450000000000, "nvme_cache_bytes_per_node": 8000000000000, "nvme_restore_bytes_per_s": 12000000000, "object_fixed_s": 0.02, "object_restore_bytes_per_s": 4000000000, "offered_output_tokens_per_s": 1000000, "output_tokens": 1024, "restore_exposed_fraction": 0.25, "restore_fixed_s": 0.002, "state_envelope": 1.1, "state_reserve_fraction": 0.2, "suffix_store_bytes": 15000000000, "working_sessions": 30000, "workspace_draft_width": 16 } }, "lineage": { "design": "Revision_J/planning/website_four_arm_2026-09-04/DESIGN.md", "model_source_sha256": "0284a6a3cef9b6857f16358f3809ea9dea2df5c22f554df7c08bff4fbf45a961", "scenario": "Revision_J/planning/website_four_arm_2026-09-04/scenario.json", "scenario_sha256": "0309e390be20dba1fddc09bdc695e032bb69e042f67bc9b29e2915a1e36220a0", "sources": "Revision_J/planning/website_four_arm_2026-09-04/SOURCES.md", "sources_sha256": "13f40558c03437af37b62debd70df6edd098464b1db73eb82a64a19ba119f832" }, "model_name": "GLM-5.3-Flash", "per_arm_accounting": { "b200_native": { "cpu_background": { "active_sockets": 0, "admitted_shadow_sessions": 0, "bandwidth_demand_bytes_per_s": 0.0, "beta_components": { "bandwidth": 1.0, "compute": 1.0, "memory": 1.0, "transfer": 1.0 }, "bootstrap": { "amortization_hours": 720, "amortized_bandwidth_bytes_per_s": 0.0, "amortized_compute_flops_per_s": 0.0, "amortized_external_transfer_bytes_per_s": 0.0, "bandwidth_bytes": 0.0, "compute_flops": 0, "recurring_admission_fraction": 0.0, "required_amortized_rates": { "bandwidth_bytes_per_s": 0.0, "compute_flops_per_s": 0.0, "external_transfer_bytes_per_s": 0.0 }, "shadow_sessions": 0, "time_seconds_at_25_percent_reservation": null, "time_seconds_at_full_reservation": null, "tokens": 0 }, "candidate_tokens_per_s": 0.0, "compute_demand_flops_per_s": 0.0, "concurrent_batch_slots": 0, "desired_maintained_episode_share": 0.0, "desired_shadow_sessions": 0, "effective_maintained_episode_share": 0.0, "enabled": false, "evidence_class": "unqualified_background_target_no_online_speed_credit", "external_transfer_bytes_per_s": 0.0, "external_transfer_capacity_bytes_per_s": 385556891544.92975, "failed_forecasts_per_s": 0.0, "forecasts_per_s": 0.0, "input_and_feedback_tokens_per_s": 0.0, "online_delivered_output_credit": 0, "packed_worker_weights_bytes": 180000000.0, "remaining_candidate_capacity_tokens_per_s": 0.0, "resource_admission_beta": 0.0, "stored_warm_sessions": 11266, "transfer_contract": "online service has priority; background coverage is reduced to fit remaining host PCIe and Full attach capacity" }, "decode": { "lookup": { "accepted_length": 4.161139200000001, "effective_rate": 7060.090739021996, "fallback_to_native": false, "rate": 7060.090739021996, "seconds": 0.03772089037440491, "width": 7 }, "lookup_widths_searched": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 ], "mtp_widths_searched": [ 0, 1, 2, 3, 4, 5 ], "native": { "accepted_length": 3.6892800000000006, "rate": 6410.740318423321, "seconds": 0.0368309911604828, "width": 5 }, "net_output_tokens_per_s": 6561.6161486148585 }, "episode_service": { "decode_seconds": 0.15605911360971092, "decoded_output_tokens": 1024, "exposed_restore_seconds": 0.01367238345753031, "new_input_tokens": 1024, "prefill_seconds": 1.7459581097455426, "total_seconds_per_group": 1.9156896068127838, "true_miss_recomputed_context_tokens": 39321.6 }, "hbm_admission": { "admitted": true, "allowed_bytes_per_rank": 162000000000.0, "batch": 64, "checkpoint_bytes_per_rank": 82141249536, "required_bytes_per_rank": 145559335731.2, "state_bytes_per_rank_with_envelope": 990907596.8000001 }, "mean_inter_token_seconds": 0.009753694600606932, "resource_admission": { "bytes_per_delivered_token_by_resource": { "gpu_service": 0.00011692441447831932, "host_h2d": 1688751.9993559504, "intra_node_broadcast": 1697995.91808, "nvme": 1032025.9694693908 }, "canonical_park_bytes_per_episode": 1031593984, "capacity_by_resource_tokens_per_s": { "gpu_service": 8552.533741235238, "host_h2d": 236861.30358545863, "intra_node_broadcast": 2120146.439498324, "nvme": 93020.91501569263 }, "limiting_output_tokens_per_s": 8552.533741235238, "limiting_resource": "gpu_service", "lookup_candidates_per_episode": 430.6512985674691, "lookup_rounds_per_episode": 61.521614081067014, "physical_host_count": 8, "resource_capacity_per_s": { "gpu_service": 1.0, "host_h2d": 400000000000, "intra_node_broadcast": 3600000000000, "nvme": 96000000000 }, "total_use_per_s_including_cpu": { "gpu_service": 1.0, "host_h2d": 14443108455.070234, "intra_node_broadcast": 14522167381.858906, "nvme": 8826436925.717974 } }, "restore": { "conditional_tier_mix": { "hbm": 0.034133333333333335, "host_dram": 0.37553333333333333, "nvme": 0.5903333333333334, "object": 0.0 }, "exposed_restore_seconds_per_episode": 0.01367238345753031, "independence_note": "conditional storage-tier mix is independent of the true residual-miss assumption", "nvme_can_retain_all_working_sessions": true, "tier_counts": { "hbm": 1024, "host_dram": 11266, "nvme": 17710, "object": 0 }, "tier_duration_seconds": { "hbm": 0.0, "host_dram": 0.02834678272, "nvme": 0.11431294805333334, "object": 0.30624527872 }, "true_residual_miss_fraction": 0.3, "weighted_successful_restore_seconds": 0.07812790547160178 }, "state": { "broadcast_bytes": 2571706368, "canonical_bytes": 1031593984, "components": { "compressed_index": 51904512, "convolution": 31752192, "fp32_kda": 142606336, "growing_mla": 805306368, "tail": 24576 }, "per_rank_bytes": 900825088, "replicated_bytes": 857235456, "serving_group_bytes": 3603300352, "sharded_bytes": 174358528 }, "warm_memory": { "complete_target_and_shadow_slot_bytes": 1134753382.4, "host_usable_bytes": 12800000000000.0, "occupied_paused_sessions": 11266, "packed_cpu_weight_reservation_bytes": 0.0, "scope": "physical DRAM capacity for complete canonical target states after worker weights and configured shadow reservations; independent of workload occupancy; HBM/NVMe/object excluded", "shadow_capacity_sessions": 0, "shadow_session_fraction": 0.0, "shadow_state_reservation_bytes": 0, "sidecar_usable_bytes": 0.0, "suffix_store_reservation_bytes": 15000000000, "target_slot_bytes": 1134753382.4, "usable_bytes_after_fixed_reservations": 12785000000000.0, "warm_stored_sessions": 11266 } }, "h100_full": { "cpu_background": { "active_sockets": 24, "admitted_shadow_sessions": 30000, "bandwidth_demand_bytes_per_s": 4382113983338.369, "beta_components": { "bandwidth": 1.0, "compute": 1.0, "memory": 1.0, "transfer": 1.0 }, "bootstrap": { "amortization_hours": 720, "amortized_bandwidth_bytes_per_s": 6232638888.888889, "amortized_compute_flops_per_s": 68266666666.666664, "amortized_external_transfer_bytes_per_s": 3009259.2592592593, "bandwidth_bytes": 1.6155e+16, "compute_flops": 176947200000000000, "recurring_admission_fraction": 1.0, "required_amortized_rates": { "bandwidth_bytes_per_s": 6232638888.888889, "compute_flops_per_s": 68266666666.666664, "external_transfer_bytes_per_s": 3009259.2592592593 }, "shadow_sessions": 30000, "time_seconds_at_25_percent_reservation": 14745.6, "time_seconds_at_full_reservation": 3686.4, "tokens": 122880000 }, "candidate_tokens_per_s": 84.80799642620366, "compute_demand_flops_per_s": 47799143713751.2, "concurrent_batch_slots": 96, "desired_maintained_episode_share": 1.0, "desired_shadow_sessions": 30000, "effective_maintained_episode_share": 1.0, "enabled": true, "evidence_class": "unqualified_background_target_no_online_speed_credit", "external_transfer_bytes_per_s": 5512652543.102443, "external_transfer_capacity_bytes_per_s": 242681594912.63483, "failed_forecasts_per_s": 5.300499776637729, "forecasts_per_s": 10.600999553275457, "input_and_feedback_tokens_per_s": 33109.04180478991, "online_delivered_output_credit": 0, "packed_worker_weights_bytes": 180000000.0, "remaining_candidate_capacity_tokens_per_s": 176.88412113601888, "resource_admission_beta": 1.0, "stored_warm_sessions": 31466, "transfer_contract": "online service has priority; background coverage is reduced to fit remaining host PCIe and Full attach capacity" }, "decode": { "lookup": { "accepted_length": 11.999999999999996, "effective_rate": 4894.627683209041, "fallback_to_native": false, "rate": 4894.627683209041, "seconds": 0.03922668125681011, "width": 16 }, "lookup_widths_searched": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 ], "mtp_widths_searched": [ 0, 1, 2, 3, 4, 5 ], "native": { "accepted_length": 3.6892800000000006, "rate": 1815.6169441590914, "seconds": 0.03251152738461539, "width": 5 }, "net_output_tokens_per_s": 4228.123290250738 }, "episode_service": { "decode_seconds": 0.24218782890299168, "decoded_output_tokens": 1024, "exposed_restore_seconds": 0.02681410917432434, "new_input_tokens": 1024, "prefill_seconds": 0.4856438818565401, "total_seconds_per_group": 0.7546458199338562, "true_miss_recomputed_context_tokens": 10485.76 }, "hbm_admission": { "admitted": true, "allowed_bytes_per_rank": 72000000000.0, "batch": 16, "checkpoint_bytes_per_rank": 41070624768, "required_bytes_per_rank": 70714949632.0, "state_bytes_per_rank_with_envelope": 1852770304.0000002 }, "mean_inter_token_seconds": 0.003784184826609245, "resource_admission": { "bytes_per_delivered_token_by_resource": { "full_external_fabric": 3437765.918674129, "gpu_service": 9.211985106614455e-05, "host_h2d": 3437765.918674129, "intra_node_broadcast": 10411217.901056001 }, "canonical_park_bytes_per_episode": 1836900352, "capacity_by_resource_tokens_per_s": { "full_external_fabric": 81448.24476821559, "gpu_service": 10855.423542554068, "host_h2d": 116354.63538316514, "intra_node_broadcast": 345780.8715764997 }, "limiting_output_tokens_per_s": 10855.423542554068, "limiting_resource": "gpu_service", "lookup_candidates_per_episode": 1238.4200469245097, "lookup_rounds_per_episode": 77.40125293278186, "physical_host_count": 8, "resource_capacity_per_s": { "full_external_fabric": 280000000000.0, "gpu_service": 1.0, "host_h2d": 400000000000, "intra_node_broadcast": 3600000000000 }, "total_use_per_s_including_cpu": { "full_external_fabric": 42834066889.72686, "gpu_service": 1.0, "host_h2d": 42834066889.72686, "intra_node_broadcast": 113018179909.78366 } }, "restore": { "conditional_tier_mix": { "full_dram": 0.9957333333333334, "hbm": 0.004266666666666667, "nvme": 0.0, "object": 0.0 }, "exposed_restore_seconds_per_episode": 0.02681410917432434, "independence_note": "conditional storage-tier mix is independent of the true residual-miss assumption", "nvme_can_retain_all_working_sessions": true, "tier_counts": { "full_dram": 29872, "hbm": 128, "nvme": 0, "object": 0 }, "tier_duration_seconds": { "full_dram": 0.11708263594666667, "hbm": 0.0, "nvme": 0.21767479808, "object": 0.5438248567466667 }, "true_residual_miss_fraction": 0.08, "weighted_successful_restore_seconds": 0.11658308336662755 }, "state": { "broadcast_bytes": 11637792768, "canonical_bytes": 1836900352, "components": { "compressed_index": 51904512, "convolution": 31752192, "fp32_kda": 142606336, "growing_mla": 1610612736, "tail": 24576 }, "per_rank_bytes": 1684336640, "replicated_bytes": 1662541824, "serving_group_bytes": 13474693120, "sharded_bytes": 174358528 }, "warm_memory": { "complete_target_and_shadow_slot_bytes": 2280590387.2, "host_usable_bytes": 12800000000000.0, "occupied_paused_sessions": 29872, "packed_cpu_weight_reservation_bytes": 4320000000.0, "scope": "physical DRAM capacity for complete canonical target states after worker weights and configured shadow reservations; independent of workload occupancy; HBM/NVMe/object excluded", "shadow_capacity_sessions": 31466, "shadow_session_fraction": 1.0, "shadow_state_reservation_bytes": 8181160000000, "sidecar_usable_bytes": 58982400000000.0, "suffix_store_reservation_bytes": 15000000000, "target_slot_bytes": 2020590387.2, "usable_bytes_after_fixed_reservations": 71763080000000.0, "warm_stored_sessions": 31466 } }, "h100_native": { "cpu_background": { "active_sockets": 0, "admitted_shadow_sessions": 0, "bandwidth_demand_bytes_per_s": 0.0, "beta_components": { "bandwidth": 1.0, "compute": 1.0, "memory": 1.0, "transfer": 1.0 }, "bootstrap": { "amortization_hours": 720, "amortized_bandwidth_bytes_per_s": 0.0, "amortized_compute_flops_per_s": 0.0, "amortized_external_transfer_bytes_per_s": 0.0, "bandwidth_bytes": 0.0, "compute_flops": 0, "recurring_admission_fraction": 0.0, "required_amortized_rates": { "bandwidth_bytes_per_s": 0.0, "compute_flops_per_s": 0.0, "external_transfer_bytes_per_s": 0.0 }, "shadow_sessions": 0, "time_seconds_at_25_percent_reservation": null, "time_seconds_at_full_reservation": null, "tokens": 0 }, "candidate_tokens_per_s": 0.0, "compute_demand_flops_per_s": 0.0, "concurrent_batch_slots": 0, "desired_maintained_episode_share": 0.0, "desired_shadow_sessions": 0, "effective_maintained_episode_share": 0.0, "enabled": false, "evidence_class": "unqualified_background_target_no_online_speed_credit", "external_transfer_bytes_per_s": 0.0, "external_transfer_capacity_bytes_per_s": 389051638821.1218, "failed_forecasts_per_s": 0.0, "forecasts_per_s": 0.0, "input_and_feedback_tokens_per_s": 0.0, "online_delivered_output_credit": 0, "packed_worker_weights_bytes": 180000000.0, "remaining_candidate_capacity_tokens_per_s": 0.0, "resource_admission_beta": 0.0, "stored_warm_sessions": 6327, "transfer_contract": "online service has priority; background coverage is reduced to fit remaining host PCIe and Full attach capacity" }, "decode": { "lookup": { "accepted_length": 4.4631290880000005, "effective_rate": 2127.14317965829, "fallback_to_native": false, "rate": 2127.14317965829, "seconds": 0.03357087858066588, "width": 9 }, "lookup_widths_searched": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 ], "mtp_widths_searched": [ 0, 1, 2, 3, 4, 5 ], "native": { "accepted_length": 3.6892800000000006, "rate": 1815.6169441590914, "seconds": 0.03251152738461539, "width": 5 }, "net_output_tokens_per_s": 1884.6189088162678 }, "episode_service": { "decode_seconds": 0.543345922727251, "decoded_output_tokens": 1024, "exposed_restore_seconds": 0.03228094250298027, "new_input_tokens": 1024, "prefill_seconds": 1.7023459915611814, "total_seconds_per_group": 2.2779728567914126, "true_miss_recomputed_context_tokens": 39321.6 }, "hbm_admission": { "admitted": true, "allowed_bytes_per_rank": 72000000000.0, "batch": 16, "checkpoint_bytes_per_rank": 41070624768, "required_bytes_per_rank": 70714949632.0, "state_bytes_per_rank_with_envelope": 1852770304.0000002 }, "mean_inter_token_seconds": 0.008489780042613297, "resource_admission": { "bytes_per_delivered_token_by_resource": { "gpu_service": 0.0002780728584950455, "host_h2d": 3044442.0888468414, "intra_node_broadcast": 7921578.83776, "nvme": 2399414.557686199 }, "canonical_park_bytes_per_episode": 1836900352, "capacity_by_resource_tokens_per_s": { "gpu_service": 3596.179812053888, "host_h2d": 131386.96297274946, "intra_node_broadcast": 454454.85978625686, "nvme": 40009.75975263509 }, "limiting_output_tokens_per_s": 3596.179812053888, "limiting_resource": "gpu_service", "lookup_candidates_per_episode": 516.2297470164501, "lookup_rounds_per_episode": 57.35886077960557, "physical_host_count": 8, "resource_capacity_per_s": { "gpu_service": 1.0, "host_h2d": 400000000000, "intra_node_broadcast": 3600000000000, "nvme": 96000000000 }, "total_use_per_s_including_cpu": { "gpu_service": 1.0, "host_h2d": 10948361178.87818, "intra_node_broadcast": 28487421895.945812, "nvme": 8628726193.099318 } }, "restore": { "conditional_tier_mix": { "hbm": 0.004266666666666667, "host_dram": 0.2109, "nvme": 0.7848333333333334, "object": 0.0 }, "exposed_restore_seconds_per_episode": 0.03228094250298027, "independence_note": "conditional storage-tier mix is independent of the true residual-miss assumption", "nvme_can_retain_all_working_sessions": true, "tier_counts": { "hbm": 128, "host_dram": 6327, "nvme": 23545, "object": 0 }, "tier_duration_seconds": { "hbm": 0.0, "host_dram": 0.06459976874666667, "nvme": 0.21767479808, "object": 0.5438248567466667 }, "true_residual_miss_fraction": 0.3, "weighted_successful_restore_seconds": 0.18446252858845869 }, "state": { "broadcast_bytes": 11637792768, "canonical_bytes": 1836900352, "components": { "compressed_index": 51904512, "convolution": 31752192, "fp32_kda": 142606336, "growing_mla": 1610612736, "tail": 24576 }, "per_rank_bytes": 1684336640, "replicated_bytes": 1662541824, "serving_group_bytes": 13474693120, "sharded_bytes": 174358528 }, "warm_memory": { "complete_target_and_shadow_slot_bytes": 2020590387.2, "host_usable_bytes": 12800000000000.0, "occupied_paused_sessions": 6327, "packed_cpu_weight_reservation_bytes": 0.0, "scope": "physical DRAM capacity for complete canonical target states after worker weights and configured shadow reservations; independent of workload occupancy; HBM/NVMe/object excluded", "shadow_capacity_sessions": 0, "shadow_session_fraction": 0.0, "shadow_state_reservation_bytes": 0, "sidecar_usable_bytes": 0.0, "suffix_store_reservation_bytes": 15000000000, "target_slot_bytes": 2020590387.2, "usable_bytes_after_fixed_reservations": 12785000000000.0, "warm_stored_sessions": 6327 } }, "h100_software": { "cpu_background": { "active_sockets": 16, "admitted_shadow_sessions": 625, "bandwidth_demand_bytes_per_s": 59915858656.2536, "beta_components": { "bandwidth": 1.0, "compute": 1.0, "memory": 0.20833333333333334, "transfer": 1.0 }, "bootstrap": { "amortization_hours": 720, "amortized_bandwidth_bytes_per_s": 129846643.51851852, "amortized_compute_flops_per_s": 1422222222.2222223, "amortized_external_transfer_bytes_per_s": 62692.9012345679, "bandwidth_bytes": 336562500000000.0, "compute_flops": 3686400000000000, "recurring_admission_fraction": 1.0, "required_amortized_rates": { "bandwidth_bytes_per_s": 129846643.51851852, "compute_flops_per_s": 1422222222.2222223, "external_transfer_bytes_per_s": 62692.9012345679 }, "shadow_sessions": 625, "time_seconds_at_25_percent_reservation": 1051.7578125, "time_seconds_at_full_reservation": 262.939453125, "tokens": 2560000 }, "candidate_tokens_per_s": 1.1595645266446095, "compute_demand_flops_per_s": 653549120249.3281, "concurrent_batch_slots": 64, "desired_maintained_episode_share": 0.1, "desired_shadow_sessions": 3000, "effective_maintained_episode_share": 0.020833333333333336, "enabled": true, "evidence_class": "unqualified_background_target_no_online_speed_credit", "external_transfer_bytes_per_s": 75373509.64612252, "external_transfer_capacity_bytes_per_s": 376272324186.7261, "failed_forecasts_per_s": 0.0724727829152881, "forecasts_per_s": 0.1449455658305762, "input_and_feedback_tokens_per_s": 452.6939912020555, "online_delivered_output_credit": 0, "packed_worker_weights_bytes": 180000000.0, "remaining_candidate_capacity_tokens_per_s": 4001.0097110847687, "resource_admission_beta": 0.20833333333333334, "stored_warm_sessions": 6245, "transfer_contract": "online service has priority; background coverage is reduced to fit remaining host PCIe and Full attach capacity" }, "decode": { "lookup": { "accepted_length": 11.999999999999996, "effective_rate": 4894.627683209041, "fallback_to_native": false, "rate": 4894.627683209041, "seconds": 0.03922668125681011, "width": 16 }, "lookup_widths_searched": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 ], "mtp_widths_searched": [ 0, 1, 2, 3, 4, 5 ], "native": { "accepted_length": 3.6892800000000006, "rate": 1815.6169441590914, "seconds": 0.03251152738461539, "width": 5 }, "net_output_tokens_per_s": 3499.2298351738136 }, "episode_service": { "decode_seconds": 0.2926358222334761, "decoded_output_tokens": 1024, "exposed_restore_seconds": 0.03974940073875684, "new_input_tokens": 1024, "prefill_seconds": 0.8174717299578059, "total_seconds_per_group": 1.1498569529300389, "true_miss_recomputed_context_tokens": 18350.08 }, "hbm_admission": { "admitted": true, "allowed_bytes_per_rank": 72000000000.0, "batch": 16, "checkpoint_bytes_per_rank": 41070624768, "required_bytes_per_rank": 70714949632.0, "state_bytes_per_rank_with_envelope": 1852770304.0000002 }, "mean_inter_token_seconds": 0.004572434722398064, "resource_admission": { "bytes_per_delivered_token_by_resource": { "gpu_service": 0.00014036339757446764, "host_h2d": 3330497.1936966507, "intra_node_broadcast": 9732225.429248, "nvme": 2633814.9636677653 }, "canonical_park_bytes_per_episode": 1836900352, "capacity_by_resource_tokens_per_s": { "gpu_service": 7124.3644517044795, "host_h2d": 120102.18797272853, "intra_node_broadcast": 369905.1184306742, "nvme": 36449.02976263508 }, "limiting_output_tokens_per_s": 7124.3644517044795, "limiting_resource": "gpu_service", "lookup_candidates_per_episode": 1044.2799159681008, "lookup_rounds_per_episode": 65.2674947480063, "physical_host_count": 8, "resource_capacity_per_s": { "gpu_service": 1.0, "host_h2d": 400000000000, "intra_node_broadcast": 3600000000000, "nvme": 96000000000 }, "total_use_per_s_including_cpu": { "gpu_service": 1.0, "host_h2d": 23803112015.8213, "intra_node_broadcast": 69335920884.10881, "nvme": 18764257699.521954 } }, "restore": { "conditional_tier_mix": { "hbm": 0.004266666666666667, "host_dram": 0.20816666666666667, "nvme": 0.7875666666666666, "object": 0.0 }, "exposed_restore_seconds_per_episode": 0.03974940073875684, "independence_note": "conditional storage-tier mix is independent of the true residual-miss assumption", "nvme_can_retain_all_working_sessions": true, "tier_counts": { "hbm": 128, "host_dram": 6245, "nvme": 23627, "object": 0 }, "tier_duration_seconds": { "hbm": 0.0, "host_dram": 0.06459976874666667, "nvme": 0.21767479808, "object": 0.5438248567466667 }, "true_residual_miss_fraction": 0.14, "weighted_successful_restore_seconds": 0.18488093366863645 }, "state": { "broadcast_bytes": 11637792768, "canonical_bytes": 1836900352, "components": { "compressed_index": 51904512, "convolution": 31752192, "fp32_kda": 142606336, "growing_mla": 1610612736, "tail": 24576 }, "per_rank_bytes": 1684336640, "replicated_bytes": 1662541824, "serving_group_bytes": 13474693120, "sharded_bytes": 174358528 }, "warm_memory": { "complete_target_and_shadow_slot_bytes": 2046590387.2, "host_usable_bytes": 12800000000000.0, "occupied_paused_sessions": 6245, "packed_cpu_weight_reservation_bytes": 2880000000.0, "scope": "physical DRAM capacity for complete canonical target states after worker weights and configured shadow reservations; independent of workload occupancy; HBM/NVMe/object excluded", "shadow_capacity_sessions": 625, "shadow_session_fraction": 0.1, "shadow_state_reservation_bytes": 162500000000, "sidecar_usable_bytes": 0.0, "suffix_store_reservation_bytes": 15000000000, "target_slot_bytes": 2020590387.2, "usable_bytes_after_fixed_reservations": 12782120000000.0, "warm_stored_sessions": 6245 } } }, "schema_version": 1, "sensitivity": { "classification": "chosen target sensitivity, not a confidence interval", "full_residual_miss_fraction": [ 0.08, 0.09, 0.1, 0.11, 0.12 ], "full_rows": [ { "delivered_output_tokens_per_s": 10855.423542554068, "energy_kwh_per_million_output_tokens": 2.2111543664531643, "residual_miss": 0.08 }, { "delivered_output_tokens_per_s": 10117.83968784654, "energy_kwh_per_million_output_tokens": 2.3723460646099306, "residual_miss": 0.09 }, { "delivered_output_tokens_per_s": 9474.110675818352, "energy_kwh_per_million_output_tokens": 2.533537762766697, "residual_miss": 0.1 }, { "delivered_output_tokens_per_s": 8907.394049713346, "energy_kwh_per_million_output_tokens": 2.6947294609234627, "residual_miss": 0.11 }, { "delivered_output_tokens_per_s": 8404.64978856323, "energy_kwh_per_million_output_tokens": 2.8559211590802294, "residual_miss": 0.12 } ], "rack_lower_readiness_case": { "arm_id": "h100_full", "classification": "separate lower-readiness case; not part of the residual-miss stripe", "delivered_output_tokens_per_s": 10175.211714501498, "energy_kwh_per_million_output_tokens": 2.3589698022310737, "lookup_output_share": 0.7648534540781986, "lookup_proposal_width": 16, "lookup_ready_round_fraction": 0.5, "mean_accepted_draft_prefix": 11, "residual_miss": 0.08, "verified_correction_or_bonus_tokens": 1 } }, "sidecar_power": { "24": { "active_sockets": 24, "admitted": true, "conservative_it_ac_kw": 17.210861796940865, "conservative_shelf_dc_w": 15559.443707093822, "direct_12v_it_ac_kw": 16.391943707093823, "direct_12v_shelf_dc_w": 14781.47152173913, "evidence_class": "component_arithmetic_from_operating_assumptions", "n_plus_one_dc_capacity_w": 24000, "operating_socket_cap": 48, "per_board_dc_w": 221.32608695652172, "utilization_fraction": 0.375 }, "48": { "active_sockets": 48, "admitted": true, "conservative_it_ac_kw": 25.770158978682407, "conservative_shelf_dc_w": 23655.151029748286, "direct_12v_it_ac_kw": 24.525151029748287, "direct_12v_shelf_dc_w": 22472.39347826087, "evidence_class": "component_arithmetic_from_operating_assumptions", "n_plus_one_dc_capacity_w": 24000, "operating_socket_cap": 48, "per_board_dc_w": 333.2391304347826, "utilization_fraction": 0.75 }, "64": { "active_sockets": 64, "admitted": false, "conservative_it_ac_kw": 31.568946284475498, "conservative_shelf_dc_w": 29140.24897025172, "direct_12v_it_ac_kw": 30.03524897025172, "direct_12v_shelf_dc_w": 27683.23652173913, "evidence_class": "component_arithmetic_from_operating_assumptions", "n_plus_one_dc_capacity_w": 24000, "operating_socket_cap": 48, "per_board_dc_w": 407.8478260869565, "utilization_fraction": 1.0 } }, "sources": [ { "evidence_class": "primary_source", "id": "P01", "supports": "model geometry", "url": "https://huggingface.co/zai-org/GLM-5.3-Flash/blob/690b705278a3a58e538fcb37c2ca8b5f9511213c/config.json" }, { "evidence_class": "primary_source", "id": "P02", "supports": "KDA and convolution state layout", "url": "https://github.com/vllm-project/vllm/blob/6cbb3c154ef1449d2b3c9131a237f36faa695734/vllm/model_executor/layers/mamba/mamba_utils.py" }, { "evidence_class": "primary_source", "id": "P03", "supports": "MTP state lineage", "url": "https://github.com/vllm-project/vllm/blob/6cbb3c154ef1449d2b3c9131a237f36faa695734/vllm/models/glm5next/nvidia/mtp.py" }, { "evidence_class": "primary_source", "id": "P04", "supports": "serving precision recipe", "url": "https://recipes.vllm.ai/zai-org/GLM-5.3-Flash" }, { "evidence_class": "primary_source", "id": "P05", "supports": "reported 8K timing anchors; simulated acceptance", "url": "https://docs.sglang.io/cookbook/autoregressive/GLM/GLM-5.3-Flash.md" }, { "evidence_class": "primary_source", "id": "P06", "supports": "GPU memory and bandwidth specifications", "url": "https://docs.nvidia.com/enterprise-reference-architectures/hgx-ai-factory-h100-h200-b200/latest/components.html" }, { "evidence_class": "primary_source", "id": "P08", "supports": "native host, disk, object and broadcast tiers", "url": "https://docs.nvidia.com/dynamo/dev/reference/components/kvbm-configuration" }, { "evidence_class": "primary_source", "id": "P09", "supports": "CPU socket specification, not measured power", "url": "https://www.amd.com/en/products/processors/server/epyc/9005-series/amd-epyc-9355p.html" }, { "evidence_class": "primary_source_unavailable_live", "id": "P10", "supports": "NIC lineage only; live PDF returned 404", "url": "https://docs.broadcom.com/doc/957608-P2200GQF00-DS" }, { "evidence_class": "primary_source", "id": "P11", "supports": "CPU worker geometry reference, not qualified software", "url": "https://huggingface.co/Qwen/Qwen3-0.6B/blob/main/config.json" }, { "evidence_class": "primary_source", "id": "P12", "supports": "k candidate drafts require k+1 target verification queries, including bonus-token logit", "url": "https://github.com/vllm-project/vllm/blob/6cbb3c154ef1449d2b3c9131a237f36faa695734/vllm/v1/worker/gpu_model_runner.py#L2917-L2945" }, { "evidence_class": "primary_source", "id": "P13", "supports": "KVBM hybrid limitation and transition to framework-native offload; complete Flash tier integration remains a model assumption", "url": "https://github.com/ai-dynamo/dynamo/issues/13755#issuecomment-5481330975" }, { "evidence_class": "primary_source", "id": "P14", "supports": "measured B200 versus H200 single-request 8K TTFT and concurrency-256 throughput on the sibling GLM-5.3 model with vendor-recommended stacks; basis for the 1.95x per-GPU B200 prefill ratio", "url": "https://docs.sglang.io/cookbook/autoregressive/GLM/GLM-5.3.md" }, { "evidence_class": "local_planning_lineage", "id": "L01-L07", "supports": "BOM, power, topology and development assumptions", "url": "Revision_J/planning/website_four_arm_2026-09-04/SOURCES.md" } ], "status": "modeled service capacity; not measured and not latency-SLO goodput", "topology": { "admitted": true, "board_dac_trunks": 56, "board_endpoints": 112, "dual_homed_boards": 48, "external_raw_bytes_per_s": 400000000000.0, "external_usable_bytes_per_s": 280000000000.0, "failure_reason": null, "host_endpoints": 16, "host_trunks": 8, "per_switch_board_cages": 28, "per_switch_host_cages": 4, "reduced_plane_failure_coverage_boards": 16, "single_homed_boards": 16, "spare_cables": 8, "total_switch_cages": 64, "used_cables": 64 } }