pub mod ab;
pub mod bootstrap;
pub mod drain;
pub mod join;
pub mod metrics;
pub mod protocol;
pub mod receipt;
pub mod replicate;
pub mod samples;
pub mod window;
pub mod witness;
#[cfg(test)]
mod join_fixture;
pub use ab::{AbRecord, AbReplicate, Arm, ArmId, ConfigDiff, DeltaKind};
pub use bootstrap::{
bootstrap_agg_tok_s_ci, bootstrap_ci, itl_p95_ms, median_decode_tok_s, paired_ratio_lcb,
ttft_p50_ms, BootstrapCi, SplitMix64, Statistic,
};
pub use drain::{
percentile, AdmissionCap, BandContext, BandInput, BandStatus, ComparatorStatus, DerivedBand,
Lane, LaneConfig, MeasuredJoin, Outcome, RequestOutcome, SampleRow, StreamMode, StreamVerdict,
StreamWitness, StreamWitnessSource, DRAIN_SUSPECT_FRACTION, REQUEST_TIMEOUT_MS, SCHEMA_VERSION,
};
pub use join::{BandRatios, JoinKey, Ratio, RatioBound, RatioMethod};
pub use metrics::{agg_tok_s, aggregate_terms, BandMetrics, RequestSample};
pub use protocol::{
min_sampled_requests, warmup_requests, BandConfig, ClientModel, ProtocolParams, ProtocolSource,
Sampler, BOOTSTRAP_RESAMPLES, BOOTSTRAP_SEED, COOLDOWN, INTERLEAVED, MIN_WALL_CLOCK, N_PREDICT,
QUIESCE, REPLICATES, REQUEST_TIMEOUT,
};
pub use receipt::{
sha256_file, ClientIdentity, ComparatorIdentity, ComputeClass, KvBlock, Ladder, ModelIdentity,
Provenance, Receipt, ReceiptBand, ReceiptInput, Roofline, RunId, SlotsAdmitted,
SubjectIdentity, TokenCountingMethod, TokenizationBlock, Workload,
CLOCK_SOURCE_SYSTEM_REALTIME, SERVER_ONLY_FIELDS, SPEC_ID,
};
pub use replicate::{log_ratio_lcb, t_lower_one_sided_95, ArmOrder, ReplicatePair, MIN_REPLICATES};
pub use samples::{read_samples_gz, write_samples_gz, SamplesFile};
pub use window::{WindowController, WindowReport};
pub use witness::{BatchInvariance, BatchInvarianceWitness};
#[cfg(test)]
mod conformance_tests {
use super::*;
#[test]
fn replicates_is_at_least_five() {
assert!(REPLICATES >= 5, "REPLICATES={REPLICATES}");
assert_eq!(REPLICATES, MIN_REPLICATES, "one floor, two spellings");
assert!(INTERLEAVED, "ยง4.3 makes interleaving mandatory");
}
#[test]
fn every_declared_band_has_a_conformant_config() {
for (c, want_samples) in [(1_usize, 30_usize), (4, 32), (8, 64), (16, 128)] {
let cfg = BandConfig::conformant(c);
assert!(
cfg.is_conformant(),
"c={c}: {:?}",
cfg.conformance_violations()
);
assert_eq!(cfg.min_samples, want_samples, "c={c}");
assert_eq!(cfg.warmup_requests, 2 * c, "c={c}");
assert_eq!(cfg.client_model, ClientModel::ClosedLoop);
}
}
#[test]
fn protocol_composes_end_to_end() {
let cfg = BandConfig::conformant(4);
let mut w = WindowController::new(&cfg);
let mut samples = Vec::new();
let mut now = 0.0_f64;
while let Some(index) = w.try_admit(now) {
let start = now;
let end = now + 1.0;
let drained = w.complete(end);
samples.push(RequestSample {
index,
worker: index % 4,
start_s: start,
end_s: end,
token_times_s: (1..=8).map(|k| start + f64::from(k) * 0.125).collect(),
generated_tokens: 8,
prompt_tokens: 512,
outcome: Outcome::Completed,
in_flight_at_start: 4,
drained,
});
now += 0.25;
}
let report = w.report();
assert!(report.requested >= cfg.min_samples, "{report:?}");
assert!(report.window_ms >= 60_000.0, "{report:?}");
let m = BandMetrics::from_samples(cfg.concurrency, &samples);
assert_eq!(m.requested, samples.len());
assert_eq!(m.completed, samples.len());
assert_eq!(m.timeouts, 0);
assert!(m.agg_tok_s > 0.0);
let ci = bootstrap_agg_tok_s_ci(&samples, 0.95).expect("n >= 2");
assert_eq!(ci.seed, BOOTSTRAP_SEED);
assert_eq!(ci.resamples, BOOTSTRAP_RESAMPLES);
assert!(ci.lower <= ci.point && ci.point <= ci.upper, "{ci:?}");
}
}
#[cfg(test)]
mod gate_conformance_tests {
#![allow(non_snake_case)]
use super::*;
use serde_json::Value;
use std::path::{Path, PathBuf};
const RECEIPT_SCOPE_UNPRODUCIBLE: &[(&str, &str)] = &[
(
"signature",
"ยง4.9.1's signature is applied by `scripts/perf_receipt_sign.sh` on the PRODUCING \
HOST, with a key that lives only there (forjar-deployed). A renderer able to sign \
its own output would be a renderer able to forge one, which is precisely the \
property the arm exists to deny โ so `render()` emits the payload and never the \
attestation over it. The receipt is legal unsigned at merge phase and fails \
ArmC-sig at release phase, which is the ยง4.5 Arm C table exactly (PERF-007).",
),
(
"kv",
"Arm D's memory block is server-reported (ยง4.4.9). Supplied only via \
KvBlock::from_server_report; absent otherwise.",
),
(
"itl",
"Arm E's itl.p95_w1_ms / p95_w2_ms is a ratio ACROSS two workloads. A single \
host x workload receipt cannot hold it (PERF-020).",
),
(
"injector",
"Arm E's out-of-band injector is a W2 construct (ยง4.3.2) this producer does not \
model (PERF-020).",
),
];
const BAND_SCOPE_UNPRODUCIBLE: &[(&str, &str)] = &[
(
"agg_ratio",
"The v2.2 BARE SCALAR. A ratio is representable only inside `ratios`, only beside a \
`baseline` that itself passes every receipt rule, and only when the two lanes share \
a run_id (PP-3) and a join key (PP-22). `perf_gate.sh` still reads the scalar for \
historical v2 receipts; a v3 receipt never writes one.",
),
("decode_ratio", "As agg_ratio (PP-3, PP-22, PP-25)."),
(
"comparator_requested",
"The COMPARATOR lane's request counter as a v2.2 top-level scalar. At v3 the whole \
comparator band travels in `baseline`, counters included, so a reader takes \
`band.baseline.requested` instead. `scripts/perf-receipt-fields.yaml` marks the \
legacy field `required: conditional` and `perf_gate.sh:88` reads it defensively.",
),
(
"comparator_completed",
"As comparator_requested; at v3 it is `band.baseline.completed`.",
),
];
fn repo_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("..")
.canonicalize()
.expect("workspace root is two levels above this crate")
}
fn gate_source() -> String {
let path = repo_root().join("scripts").join("perf_gate.sh");
std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("reading {}: {e}", path.display()))
}
fn gate_field_reads(source: &str) -> Vec<(String, String)> {
let re =
regex::Regex::new(r#"([A-Za-z_][A-Za-z0-9_]*(?:\[[a-z]\])?)\.get\("([a-z0-9_]+)""#)
.expect("static regex");
re.captures_iter(source)
.map(|c| (c[1].to_string(), c[2].to_string()))
.collect()
}
fn band(concurrency: u32, requests: Vec<RequestOutcome>) -> BandInput {
BandInput::new(
concurrency,
60_000.0,
requests,
ComparatorStatus::unmeasured(
"perf-gate",
"no same-client comparator lane on this cell yet (PP-25)",
),
)
}
fn passing_witness(m: u32) -> BatchInvarianceWitness {
let tokens: Vec<u32> = (0..128).collect();
BatchInvarianceWitness::compare(&tokens, &tokens, 64)
.formed_at(m, "scripts/perf041_batched_parity_probe.py")
}
fn synthetic_band(concurrency: u32) -> BandInput {
let window_ms = 60_000.0;
let n = 30.max(8 * concurrency as usize);
let step = (window_ms - 1_000.0) / n as f64;
let mut requests: Vec<RequestOutcome> = (0..n)
.map(|i| {
let issued = i as f64 * step;
let dur = 400.0 + (i % 7) as f64 * 13.0;
let ttft = 40.0 + (i % 5) as f64;
RequestOutcome::completed(issued, issued + dur, 128)
.streamed(
ttft,
(0..128)
.map(|k| issued + ttft + f64::from(k) * (dur - ttft) / 128.0)
.collect(),
)
.server_prefill(512, 30.0 + (i % 3) as f64)
.in_flight(concurrency)
})
.collect();
let last = requests.last_mut().expect("n >= 30");
last.issued_ms = window_ms - 100.0;
last.settled_ms = window_ms + 250.0;
last.ttft_ms = Some(40.0);
last.token_times_ms = (0..128)
.map(|k| window_ms - 60.0 + k as f64 * 310.0 / 128.0)
.collect();
let mut band = band(concurrency, requests)
.n_predict(128)
.stream_mode(StreamMode::Live)
.lane(LaneConfig {
n_ctx_slot: Some(1024),
kv_type: Some("f16".to_string()),
fa: Some(true),
n_batch: Some(2048),
});
if concurrency > 1 {
band = band.witness(passing_witness(concurrency));
}
band
}
fn provenance() -> Provenance {
Provenance {
binary_path: "/opt/clean-room/bin/apr".to_string(),
binary_sha256: "a".repeat(64),
resolution: "scripts/apr_bin.sh".to_string(),
compute_class: ComputeClass::Cuda,
host: "lambda".to_string(),
accelerator: "rtx-4090".to_string(),
model: "qwen2.5-coder-7b-apache-q4k-v1".to_string(),
quantization: "Q4_K_M".to_string(),
feature_set: vec!["inference".to_string(), "cuda".to_string()],
started_utc: "2026-09-02T10:11:12.345Z".to_string(),
clock_source: CLOCK_SOURCE_SYSTEM_REALTIME.to_string(),
subject: SubjectIdentity {
path: "/opt/clean-room/bin/apr".to_string(),
sha256: "5".repeat(64),
commit: "62d23d8d1".to_string(),
feature_set: vec!["inference".to_string(), "cuda".to_string()],
},
client: ClientIdentity {
path: "/opt/clean-room/bin/apr".to_string(),
sha256: "a".repeat(64),
commit: "62d23d8d1".to_string(),
pid: 4242,
},
comparator: None,
server_config: None,
model_file: None,
}
}
fn ladder() -> Ladder {
Ladder::derive(
&[1, 4, 8, 16],
SlotsAdmitted {
apr: Some(16),
llama: Some(16),
},
)
}
fn run_id() -> RunId {
RunId::derive("2026-09-02T10:11:12.345Z", "lambda", &"a".repeat(64), 4242)
}
fn receipt_input(kv: Option<KvBlock>) -> ReceiptInput {
ReceiptInput {
kv,
..ReceiptInput::new(
run_id(),
provenance(),
TokenizationBlock::ClientTokenizer {
tokenizer_sha256: "b".repeat(64),
counts_special_tokens: false,
counts_prompt_echo: false,
},
Workload::W1,
ProtocolParams::spec_fallback(),
"62d23d8d1",
ladder(),
vec![
synthetic_band(1),
synthetic_band(4),
synthetic_band(8),
synthetic_band(16),
],
)
}
}
fn rendered() -> Value {
receipt_input(None).render().expect("valid receipt")
}
fn unproducible(table: &[(&str, &str)], field: &str) -> bool {
table.iter().any(|(name, _)| *name == field)
}
#[test]
fn every_field_the_gate_reads_is_produced_or_declared_unproducible() {
let receipt = rendered();
let bands = receipt["bands"].as_array().expect("bands array");
let mut checked = 0_usize;
for (receiver, field) in gate_field_reads(&gate_source()) {
match receiver.as_str() {
"r" => {
checked += 1;
assert!(
receipt.get(&field).is_some()
|| unproducible(RECEIPT_SCOPE_UNPRODUCIBLE, &field),
"perf_gate.sh reads receipt field `{field}` and the producer neither \
emits it nor declares it unproducible"
);
}
"b" | "bands[c]" => {
checked += 1;
let missing: Vec<u64> = bands
.iter()
.filter(|b| b.get(&field).is_none())
.filter_map(|b| b["concurrency"].as_u64())
.collect();
assert!(
missing.is_empty() || unproducible(BAND_SCOPE_UNPRODUCIBLE, &field),
"perf_gate.sh reads band field `{field}`, absent at bands {missing:?} \
and not declared unproducible"
);
}
_ => {}
}
}
assert!(
checked >= 10,
"only {checked} field reads found in perf_gate.sh โ the extractor stopped matching, \
so this test is no longer checking anything"
);
}
#[test]
fn the_field_ratchet_rejects_an_unclassified_new_requirement() {
let receipt = rendered();
assert!(
receipt.get("max_in_flight").is_none()
&& !unproducible(RECEIPT_SCOPE_UNPRODUCIBLE, "max_in_flight"),
"control: max_in_flight is neither emitted nor classified, so were the gate to start \
reading it the test above would go red"
);
}
#[test]
fn a_produced_receipt_satisfies_every_arm_c_rule() {
let r = rendered();
assert_eq!(
r["requested"], r["completed"],
"Arm C: completed == requested"
);
assert_eq!(
r["timeouts"], 0,
"I-5: a timeout is fatal to this host's ratio"
);
assert!(
r["tokenization"]["method"]
.as_str()
.is_some_and(|m| !m.is_empty()),
"I-13: tokenization.method has no default"
);
assert!(
r["drain_ms"].as_f64().is_some(),
"ยง4.4.7: drain_ms is recorded โ the field with zero producers on 62d23d8d1"
);
for b in r["bands"].as_array().expect("bands") {
assert_ne!(
b["tokens_total"], 0,
"Arm C: a zero-token response is a failure, not a fast request"
);
}
}
#[test]
fn a_produced_receipt_carries_every_declared_band_including_the_arm_a_denominator() {
let r = rendered();
let seen: Vec<u64> = r["bands"]
.as_array()
.expect("bands")
.iter()
.filter_map(|b| b["concurrency"].as_u64())
.collect();
assert_eq!(seen, vec![1, 4, 8, 16], "ยง4.5: all four bands");
let base = r["bands"][0]["aggregate_tok_per_sec"]
.as_f64()
.expect("agg(1)");
assert!(base > 0.0, "Arm A is undefined without a non-zero agg(1)");
}
#[test]
fn a_produced_receipt_satisfies_the_receipt_validators_rules() {
let r = rendered();
let prov = &r["provenance"];
for key in [
"binary_path",
"binary_sha256",
"resolution",
"compute_class",
] {
assert!(prov.get(key).is_some(), "provenance.{key} is required");
}
let sha = prov["binary_sha256"].as_str().expect("digest");
assert_eq!(sha.len(), 64);
assert!(sha
.bytes()
.all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()));
let samples = r["samples_ms"].as_array().expect("samples_ms");
assert!(
!samples.is_empty(),
"I-4: summary-only receipts are rejected"
);
assert_eq!(r["n"].as_u64(), Some(samples.len() as u64));
let distinct: std::collections::BTreeSet<String> = samples
.iter()
.map(std::string::ToString::to_string)
.collect();
assert!(
distinct.len() > 1,
"F12: a constant sample set is the fabricated-measurement shape"
);
}
#[test]
fn the_receipt_names_what_a_client_cannot_produce() {
let r = rendered();
assert!(r.get("kv").is_none(), "no invented Arm D block");
let notes = r["unproduced_fields"]
.as_array()
.expect("unproduced_fields")
.iter()
.filter_map(Value::as_str)
.collect::<Vec<_>>()
.join("\n");
assert!(notes.contains("PP-13"), "{notes}");
assert!(notes.contains("Arm D `kv` block"), "{notes}");
}
#[test]
fn the_kv_block_appears_only_when_the_server_reported_one() {
let r = receipt_input(Some(KvBlock::from_server_report(50, 100, Some(0), Some(0))))
.render()
.expect("valid receipt");
assert_eq!(r["kv"]["bytes_used"], 50);
assert_eq!(r["kv"]["bytes_reserved"], 100);
}
#[test]
fn the_emitted_drain_ms_tracks_the_measurement() {
let straggling = rendered();
let mut input = receipt_input(None);
for b in &mut input.bands {
let window = b.window_ms;
let last = b.requests.last_mut().expect("non-empty band");
last.settled_ms = window - 10.0;
last.ttft_ms = Some(20.0);
last.token_times_ms = vec![window - 80.0, window - 20.0];
last.generated_tokens = 2;
}
let quiet = input.render().expect("valid receipt");
assert_eq!(straggling["drain_ms"], serde_json::json!(250.0));
assert_eq!(quiet["drain_ms"], serde_json::json!(0.0));
assert_eq!(straggling["bands"][0]["drain_ms"], serde_json::json!(250.0));
}
#[test]
fn the_receipt_drain_ms_is_the_worst_band_not_the_mean() {
let mut input = receipt_input(None);
for b in input.bands.iter_mut().take(3) {
let window = b.window_ms;
let last = b.requests.last_mut().expect("non-empty band");
last.settled_ms = window - 10.0;
last.ttft_ms = Some(20.0);
last.token_times_ms = vec![window - 80.0, window - 20.0];
last.generated_tokens = 2;
}
let r = input.render().expect("valid receipt");
assert_eq!(r["bands"][0]["drain_ms"], serde_json::json!(0.0));
assert_eq!(r["bands"][3]["drain_ms"], serde_json::json!(250.0));
assert_eq!(r["drain_ms"], serde_json::json!(250.0));
}
#[test]
fn a_non_streaming_band_is_nonconformant_at_v3_and_legal_at_v2() {
let plain = RequestOutcome::completed(0.0, 500.0, 128).with_prompt_tokens(512);
let other = RequestOutcome::completed(600.0, 1_250.0, 128).with_prompt_tokens(512);
let input = ReceiptInput {
bands: vec![band(1, vec![plain, other]).n_predict(128)],
ladder: Ladder::derive(
&[1],
SlotsAdmitted {
apr: Some(16),
llama: Some(16),
},
),
..receipt_input(None)
};
let v3 = input.render().expect("the evidence still renders");
assert!(
v3["bands"][0].get("ttft_p95_ms").is_none(),
"no invented p95"
);
assert_eq!(
v3["bands"][0]["status"], "NONCONFORMANT-VALID",
"PP-27: a band that never streamed is a record, not a measurement"
);
let notes = v3["unproduced_fields"].to_string();
assert!(notes.contains("PP-27"), "{notes}");
let historical = ReceiptInput {
schema_version: 2,
..input
};
let v2 = historical.render().expect("a v2 receipt still renders");
assert_ne!(
v2["bands"][0]["status"], "NONCONFORMANT-VALID",
"the v2.2 rule stands for a v2-dated receipt"
);
assert_eq!(v2["schema_version"], 2);
}
#[test]
fn an_empty_resolution_is_refused_rather_than_defaulted() {
let mut input = receipt_input(None);
input.provenance.resolution = String::new();
let err = input.render().expect_err("empty resolution");
assert!(err.contains("no default"), "{err}");
}
#[test]
fn a_compute_class_the_build_cannot_reach_is_refused() {
let mut input = receipt_input(None);
input.provenance.subject.feature_set = vec!["inference".to_string()];
let err = input.render().expect_err("cuda without the feature");
assert!(err.contains("PP-2"), "{err}");
assert!(err.contains("subject.feature_set"), "{err}");
}
#[test]
fn a_client_tokenizer_without_a_digest_is_refused() {
let input = ReceiptInput {
tokenization: TokenizationBlock::ClientTokenizer {
tokenizer_sha256: "not-a-digest".to_string(),
counts_special_tokens: false,
counts_prompt_echo: false,
},
..receipt_input(None)
};
let err = input.render().expect_err("bad digest");
assert!(err.contains("client_tokenizer"), "{err}");
}
#[test]
fn a_receipt_with_no_bands_is_refused() {
let input = ReceiptInput {
bands: Vec::new(),
..receipt_input(None)
};
let err = input.render().expect_err("no bands");
assert!(err.contains("vacuous"), "{err}");
}
#[test]
fn the_receipt_renders_to_parseable_json() {
let text = receipt_input(None).render_string().expect("renders");
let back: Value = serde_json::from_str(&text).expect("valid JSON");
assert_eq!(back["workload"], "W1");
assert_eq!(back["client_model"], "closed_loop");
assert_eq!(back["spec"], SPEC_ID);
assert_eq!(back["schema_version"], 3);
}
#[test]
fn timestamp_ok__a_receipt_carries_its_start_instant_and_clock() {
let r = rendered();
assert_eq!(r["provenance"]["started_utc"], "2026-09-02T10:11:12.345Z");
assert_eq!(
r["provenance"]["clock_source"],
CLOCK_SOURCE_SYSTEM_REALTIME
);
assert_eq!(r["run_id"], run_id().as_str());
let recomputed = RunId::derive(
r["provenance"]["started_utc"]
.as_str()
.expect("started_utc"),
r["provenance"]["host"].as_str().expect("host"),
r["provenance"]["client"]["sha256"]
.as_str()
.expect("client sha"),
4242,
);
assert_eq!(r["run_id"], recomputed.as_str());
}
#[test]
fn timestamp_absent__an_unparseable_start_instant_is_refused() {
for bad in ["", "2026-09-02", "2026-09-02T10:11:12+00:00"] {
let mut input = receipt_input(None);
input.provenance.started_utc = bad.to_string();
let err = input.render().expect_err("{bad} must be refused");
assert!(err.contains("started_utc"), "{bad}: {err}");
}
}
#[test]
fn subject_and_client_are_distinct_identities() {
let r = rendered();
let subject = r["provenance"]["subject"]["sha256"]
.as_str()
.expect("subject sha");
let client = r["provenance"]["client"]["sha256"]
.as_str()
.expect("client sha");
assert_ne!(subject, client, "two binaries, two digests");
assert_eq!(
client, r["provenance"]["binary_sha256"],
"the legacy field is the client's, and the new one says so"
);
assert!(r["provenance"]["subject"]["commit"].is_string());
assert!(r["provenance"]["client"]["commit"].is_string());
}
#[test]
fn every_digest_in_the_split_provenance_is_checked() {
for (name, mutate) in [
(
"binary_sha256",
Box::new(|p: &mut Provenance| p.binary_sha256 = "nope".to_string())
as Box<dyn Fn(&mut Provenance)>,
),
(
"subject.sha256",
Box::new(|p: &mut Provenance| p.subject.sha256 = "A".repeat(64)),
),
(
"client.sha256",
Box::new(|p: &mut Provenance| p.client.sha256 = "z".repeat(64)),
),
(
"model_file.sha256",
Box::new(|p: &mut Provenance| {
p.model_file = Some(ModelIdentity {
path: "/models/qwen.gguf".to_string(),
sha256: "short".to_string(),
bytes: 4_683_073_440,
});
}),
),
(
"comparator.sha256",
Box::new(|p: &mut Provenance| {
p.comparator = Some(ComparatorIdentity {
commit: "39173bcac".to_string(),
cmake: "cmake -DGGML_CUDA=ON".to_string(),
sha256: "0123".to_string(),
pin_expiry: "2026-12-01T00:00:00.000Z".to_string(),
props: serde_json::json!({}),
});
}),
),
] {
let mut input = receipt_input(None);
mutate(&mut input.provenance);
let err = input.render().expect_err("{name} must be checked");
assert!(err.contains(name), "expected {name} in: {err}");
assert!(err.contains("64 lowercase hex"), "{err}");
}
let mut ok = receipt_input(None);
ok.provenance.model_file = Some(ModelIdentity {
path: "/models/qwen.gguf".to_string(),
sha256: "9".repeat(64),
bytes: 4_683_073_440,
});
let r = ok.render().expect("a well-formed model file renders");
assert_eq!(r["provenance"]["model_file"]["bytes"], 4_683_073_440_u64);
}
#[test]
fn prefill_is_emitted_with_its_source() {
let r = rendered();
for b in r["bands"].as_array().expect("bands") {
assert!(b["prefill_tok_per_sec"].as_f64().expect("prefill") > 0.0);
assert_eq!(b["prefill_source"], "server");
}
let mut input = receipt_input(None);
for band in &mut input.bands {
for req in &mut band.requests {
req.prefill_ms = None;
}
}
let without = input.render().expect("the evidence still renders");
let b = &without["bands"][0];
assert!(b.get("prefill_tok_per_sec").is_none());
assert!(b.get("prefill_source").is_none());
assert_eq!(b["status"], "NONCONFORMANT-VALID");
assert!(without["unproduced_fields"].to_string().contains("PP-13"));
}
#[test]
fn a_stale_comparator_pin_marks_every_band() {
let mut input = receipt_input(None);
input.provenance.comparator = Some(ComparatorIdentity {
commit: "39173bcac".to_string(),
cmake: "cmake -DGGML_CUDA=ON".to_string(),
sha256: "e".repeat(64),
pin_expiry: "2026-08-01T00:00:00.000Z".to_string(),
props: serde_json::json!({"n_ctx": 4096, "total_slots": 4}),
});
assert!(input.provenance.comparator_is_stale());
let r = input.render().expect("renders");
for b in r["bands"].as_array().expect("bands") {
assert_eq!(b["status"], "COMPARATOR_STALE", "{b}");
}
let mut fresh = input;
if let Some(c) = fresh.provenance.comparator.as_mut() {
c.pin_expiry = "2026-12-01T00:00:00.000Z".to_string();
}
assert!(!fresh.provenance.comparator_is_stale());
let r = fresh.render().expect("renders");
assert_ne!(r["bands"][0]["status"], "COMPARATOR_STALE");
let mut at_the_instant = fresh;
if let Some(c) = at_the_instant.provenance.comparator.as_mut() {
c.pin_expiry = at_the_instant.provenance.started_utc.clone();
}
assert!(
!at_the_instant.provenance.comparator_is_stale(),
"expiry == started_utc is not `expiry < started_utc`"
);
}
#[test]
fn every_band_carries_its_own_samples() {
let r = rendered();
for b in r["bands"].as_array().expect("bands") {
let samples = b["samples"].as_array().expect("samples[]");
assert_eq!(
samples.len(),
b["requested"].as_u64().expect("requested") as usize
);
for row in samples {
for key in [
"index",
"issued_ms",
"settled_ms",
"outcome",
"generated_tokens",
"prompt_tokens",
"ttft_ms",
"in_flight_at_start",
] {
assert!(row.get(key).is_some(), "samples row lacks {key}: {row}");
}
assert!(
row.get("token_times_ms").is_none(),
"token times stay in the gz side file"
);
}
assert!(
b.get("samples_file").is_some(),
"the key is present even when no file was written"
);
}
let mut input = receipt_input(None);
input.bands[0] = input.bands[0].clone().samples_file(SamplesFile {
path: std::path::PathBuf::from("samples.c1.r1.jsonl.gz"),
sha256: "f".repeat(64),
bytes: 4_096,
rows: 30,
});
let r = input.render().expect("renders");
assert_eq!(
r["bands"][0]["samples_file"]["sha256"],
"f".repeat(64).as_str()
);
}
#[test]
fn a_band_above_the_derived_ladder_is_refused_unless_capped() {
let capped = Ladder::derive(
&[1, 4, 8, 16],
SlotsAdmitted {
apr: Some(11),
llama: Some(16),
},
);
assert_eq!(capped.derived, vec![1, 4, 8]);
let input = ReceiptInput {
ladder: capped.clone(),
..receipt_input(None)
};
let err = input.render().expect_err("c=16 above an 11-slot subject");
assert!(err.contains("PP-24"), "{err}");
assert!(err.contains("c=16"), "{err}");
let mut allowed = ReceiptInput {
ladder: capped,
..receipt_input(None)
};
allowed.bands[3].comparator = ComparatorStatus::Unmeasured {
owner: "perf-gate".to_string(),
reason: "the subject admitted 11 slots; c=16 was not run".to_string(),
admission_capped: Some(AdmissionCap {
lane: Lane::Apr,
cap: 11,
}),
};
let r = allowed.render().expect("an admission cap is an answer");
assert_eq!(r["bands"][3]["comparator_admission_capped"]["cap"], 11);
assert_eq!(r["bands"][3]["comparator_admission_capped"]["lane"], "apr");
assert_eq!(Lane::Apr.wire_token(), "apr");
assert_eq!(Lane::Llama.wire_token(), "llama");
}
#[test]
fn a_server_reported_budget_ceiling_yields_na() {
let mut input = ReceiptInput {
ladder: Ladder::derive(
&[1, 4, 8, 16],
SlotsAdmitted {
apr: Some(11),
llama: Some(16),
},
),
..receipt_input(None)
};
input.bands[3].comparator = ComparatorStatus::NotApplicable {
decided_by: "spec-owner".to_string(),
reason: "KV budget admits 11 slots by design".to_string(),
budget: Some("kv_per_slot 469.8 MB, reserve 3.5 GB => 11 slots".to_string()),
};
let r = input.render().expect("a decided ceiling is an answer");
assert_eq!(r["bands"][3]["status"], "NA");
assert_eq!(r["bands"][3]["comparator_status"], "NOT_APPLICABLE");
assert!(r["bands"][3]["comparator_budget"].is_string());
assert_eq!(r["ladder"]["derived"], serde_json::json!([1, 4, 8]));
assert_eq!(r["ladder"]["slots_admitted"]["apr"], 11);
}
#[test]
fn scaling_efficiency_is_reported_not_gated() {
let base = rendered();
let se = base["bands"][1]["scaling_efficiency"]
.as_f64()
.expect("c=4 reports one");
assert!(se > 0.0, "{se}");
assert!(
base["bands"][0]["scaling_efficiency"].is_null(),
"null at c=1, where it is 1 by construction"
);
assert!(
base["bands"][0]["overhead_share"].as_f64().is_some(),
"overhead_share is the c=1 figure"
);
assert!(
base["bands"][1]["overhead_share"].is_null(),
"โฆand only the c=1 figure"
);
let mut input = receipt_input(None);
for r in &mut input.bands[0].requests {
let dur = r.settled_ms - r.issued_ms;
r.settled_ms = r.issued_ms + dur / 1.2;
}
let faster = input.render().expect("a faster c=1 is not an error");
let agg1_before = base["bands"][0]["aggregate_tok_per_sec"]
.as_f64()
.expect("agg(1)");
let agg1_after = faster["bands"][0]["aggregate_tok_per_sec"]
.as_f64()
.expect("agg(1)");
assert!(agg1_after > agg1_before, "{agg1_after} vs {agg1_before}");
assert!(
faster["bands"][1]["scaling_efficiency"]
.as_f64()
.expect("se")
< se,
"SE falls when agg(1) rises โ and that is REPORTED, not a failure"
);
}
#[test]
fn a_decode_above_the_roofline_is_refused() {
let mut input = receipt_input(None);
input.roofline = Some(Roofline {
bandwidth_bytes_per_sec: 1.0,
model_bytes: 1,
});
let err = input.render().expect_err("dec(1) is far above 1 tok/s");
assert!(err.contains("PP-23"), "{err}");
assert!(err.contains("c=1"), "{err}");
let dec1 = rendered()["bands"][0]["decode_tok_per_sec"]
.as_f64()
.expect("dec(1)");
let mut exact = receipt_input(None);
exact.roofline = Some(Roofline {
bandwidth_bytes_per_sec: dec1,
model_bytes: 1,
});
let r = exact.render().expect("at the ceiling is not above it");
assert_eq!(r["bands"][0]["roofline_tok_per_sec"], dec1);
}
#[test]
fn an_aggregate_above_the_roofline_is_not_compared() {
let slow: Vec<RequestOutcome> = (0..30)
.map(|i| {
let issued = f64::from(i) * 1_900.0;
RequestOutcome::completed(issued, issued + 400.0 + f64::from(i % 5), 4)
.streamed(
40.0,
(0..4)
.map(|k| issued + 40.0 + f64::from(k) * 120.0)
.collect(),
)
.server_prefill(512, 30.0)
})
.collect();
let mut input = receipt_input(None);
input.bands[0] = band(1, slow).n_predict(4).stream_mode(StreamMode::Live);
input.roofline = Some(Roofline {
bandwidth_bytes_per_sec: 10.0,
model_bytes: 1,
});
let r = input
.render()
.expect("an aggregate above the ceiling is fine");
let dec1 = r["bands"][0]["decode_tok_per_sec"]
.as_f64()
.expect("dec(1)");
let agg8 = r["bands"][2]["aggregate_tok_per_sec"]
.as_f64()
.expect("agg(8)");
assert!(dec1 <= 10.0, "dec(1)={dec1} must sit under the ceiling");
assert!(
agg8 > 10.0,
"agg(8)={agg8} must sit ABOVE it โ that is the case under test"
);
assert_eq!(r["bands"][2]["roofline_tok_per_sec"], 10.0);
}
#[test]
fn ratio_bare__a_scalar_ratio_is_unrepresentable() {
let r = rendered();
let text = r.to_string();
for scalar in ["agg_ratio", "decode_ratio", "prefill_ratio"] {
assert!(
!text.contains(scalar),
"a v3 receipt must not write `{scalar}` anywhere: {text}"
);
}
for b in r["bands"].as_array().expect("bands") {
assert!(b["ratios"].is_null(), "no comparator lane, no ratios");
assert!(b["baseline"].is_null());
assert_eq!(b["comparator_status"], "UNMEASURED");
}
}
#[test]
fn a_measured_band_renders_its_baseline_and_ratios() {
let mut input = receipt_input(None);
let subject = input.bands[0].clone();
let comparator = synthetic_band(1);
let key = input.join_key(&subject);
let comparator_key = input.join_key(&comparator);
let id = run_id();
input.bands[0].comparator =
BandInput::join_status(&subject, &comparator, &key, &comparator_key, (&id, &id))
.expect("a same-run join");
let r = input.render().expect("renders");
let b = &r["bands"][0];
assert_eq!(b["status"], "MEASURED");
assert_eq!(b["comparator_status"], "MEASURED");
assert_eq!(b["baseline"]["concurrency"], 1);
assert!(b["baseline"]["aggregate_tok_per_sec"].as_f64().is_some());
assert_eq!(b["baseline"]["run_id"], id.as_str());
assert!(
b["baseline"].get("baseline").is_none() && b["baseline"].get("ratios").is_none(),
"a baseline is one comparator lane, not a chain of them"
);
assert_eq!(b["ratios"]["agg"]["method"], "replicate_t_lower");
assert_eq!(b["ratios"]["dec"]["method"], "paired_percentile_bootstrap");
assert!(
(b["ratios"]["agg"]["point"].as_f64().expect("agg point") - 1.0).abs() < 1e-9,
"identical lanes are parity"
);
assert!(
!r.to_string().contains("agg_ratio"),
"and still no bare scalar"
);
let parsed = Receipt::parse(&r.to_string()).expect("a joined receipt parses");
parsed.validate().expect("and validates");
let band = &parsed.bands[0];
assert!(band.ratios.is_some() && band.baseline.is_some());
assert_eq!(
band.baseline.as_ref().expect("baseline").run_id.as_ref(),
Some(&id)
);
}
#[test]
fn a_baselines_per_lane_figures_come_from_the_comparator_lane() {
let halved = |c: u32| -> BandInput {
let full = synthetic_band(c);
let last = full.requests.len() - 1;
let requests = full
.requests
.iter()
.enumerate()
.filter(|(i, _)| i % 2 == 0 || *i == last)
.map(|(_, r)| r.clone())
.collect();
BandInput { requests, ..full }
};
let mut input = receipt_input(None);
let id = run_id();
for i in [0_usize, 1] {
let subject = input.bands[i].clone();
let comparator = halved(subject.concurrency);
let key = input.join_key(&subject);
let ckey = input.join_key(&comparator);
input.bands[i].comparator =
BandInput::join_status(&subject, &comparator, &key, &ckey, (&id, &id))
.expect("a same-run join");
}
let r = input.render().expect("renders");
let subject_agg1 = r["bands"][0]["aggregate_tok_per_sec"]
.as_f64()
.expect("agg(1) subject");
let baseline_agg1 = r["bands"][0]["baseline"]["aggregate_tok_per_sec"]
.as_f64()
.expect("agg(1) comparator");
assert!(
(subject_agg1 / baseline_agg1) > 1.5,
"the fixture's two lanes must differ, or this proves nothing: {subject_agg1} vs \
{baseline_agg1}"
);
let baseline_c4 = &r["bands"][1]["baseline"];
let baseline_agg4 = baseline_c4["aggregate_tok_per_sec"]
.as_f64()
.expect("agg(4) comparator");
let want = baseline_agg4 / (4.0 * baseline_agg1);
let got = baseline_c4["scaling_efficiency"]
.as_f64()
.expect("the baseline reports one");
assert!(
(got - want).abs() < 1e-9,
"the baseline's scaling_efficiency must divide by the COMPARATOR's agg(1): got {got}, \
comparator-lane {want}, subject-lane {}",
baseline_agg4 / (4.0 * subject_agg1)
);
let baseline_dec1 = r["bands"][0]["baseline"]["decode_tok_per_sec"]
.as_f64()
.expect("dec(1) comparator");
let overhead = r["bands"][0]["baseline"]["overhead_share"]
.as_f64()
.expect("the baseline reports one");
assert!(
(overhead - baseline_agg1 / baseline_dec1).abs() < 1e-9,
"overhead_share on the baseline is the comparator's agg(1)/dec(1): {overhead}"
);
assert!(
(overhead - subject_agg1 / baseline_dec1).abs() > 1e-6,
"โฆand the subject's agg(1) must give a DIFFERENT answer, or the fixture is degenerate"
);
}
#[test]
fn a_stale_pin_does_not_relabel_an_unwitnessed_band() {
let expired = ComparatorIdentity {
commit: "39173bcac0123456789abcdef0123456789abcde".to_string(),
cmake: "cmake -B build -DGGML_CUDA=ON".to_string(),
sha256: "d".repeat(64),
pin_expiry: "2026-01-01T00:00:00.000Z".to_string(),
props: Value::Null,
};
let stale_provenance = Provenance {
comparator: Some(expired),
..provenance()
};
assert!(
stale_provenance.comparator_is_stale(),
"the fixture's pin must be expired"
);
let unwitnessed = BandInput {
witness: None,
..synthetic_band(4)
};
let input = ReceiptInput {
provenance: stale_provenance.clone(),
bands: vec![synthetic_band(1), unwitnessed],
..receipt_input(None)
};
let r = input.render().expect("renders");
assert_eq!(
r["bands"][1]["status"], "INVALID-CORRECTNESS",
"correctness first: a fresher pin would not give this band a throughput"
);
assert!(
r["bands"][1].get("aggregate_tok_per_sec").is_none(),
"and it still reports none: {}",
r["bands"][1]
);
assert_eq!(r["bands"][0]["status"], "COMPARATOR_STALE");
}
#[test]
fn admission_unequal__a_hand_written_derived_ladder_is_refused() {
let honest = Ladder::derive(
&[1, 4, 8, 16],
SlotsAdmitted {
apr: Some(4),
llama: Some(16),
},
);
assert_eq!(honest.derived, vec![1, 4], "the fixture's real ceiling");
let forged = Ladder {
derived: vec![1, 4, 8, 16],
..honest.clone()
};
let input = ReceiptInput {
ladder: forged,
..receipt_input(None)
};
let err = input
.render()
.expect_err("a ladder that disagrees with its own inputs must be refused");
assert!(err.contains("PP-24"), "{err}");
assert!(
err.contains("[1, 4]"),
"the error names the real ladder: {err}"
);
let narrowed = ReceiptInput {
ladder: honest,
bands: vec![synthetic_band(1), synthetic_band(4)],
..receipt_input(None)
};
narrowed.render().expect("the derived ladder renders");
}
#[test]
fn a_run_id_its_own_provenance_does_not_reproduce_is_refused() {
let text = receipt_input(None).render_string().expect("renders");
let good = Receipt::parse(&text).expect("parses");
good.validate().expect("its own id reproduces");
assert_eq!(good.provenance.client.pid, 4242, "pid is on the wire");
let mut value: Value = serde_json::from_str(&text).expect("json");
value["provenance"]["client"]["pid"] = serde_json::json!(4243);
let err = Receipt::parse(&value.to_string())
.expect("still parses")
.validate()
.expect_err("the id no longer reproduces");
assert!(err.contains("PP-3"), "{err}");
assert!(err.contains("DERIVED"), "{err}");
for (pointer, replacement) in [
("/provenance/host", serde_json::json!("gx10")),
(
"/provenance/started_utc",
serde_json::json!("2026-09-02T10:11:13.345Z"),
),
(
"/provenance/client/sha256",
serde_json::json!("c".repeat(64)),
),
] {
let mut v: Value = serde_json::from_str(&text).expect("json");
*v.pointer_mut(pointer).expect("field exists") = replacement;
let e = Receipt::parse(&v.to_string())
.expect("parses")
.validate()
.expect_err("changing a run_id input must break the id");
assert!(e.contains("run_id"), "{pointer}: {e}");
}
}
#[test]
fn a_partial_kv_block_keeps_the_figures_the_server_did_report() {
let uncounted = KvBlock::from_server_report(50, 100, None, None);
assert_eq!(
uncounted.uncounted_fields(),
vec!["kv.admission_rejected", "kv.preempted_swap"]
);
let r = receipt_input(Some(uncounted)).render().expect("renders");
assert_eq!(r["kv"]["bytes_used"], 50, "the reported figures survive");
assert_eq!(r["kv"]["bytes_reserved"], 100);
assert!(
r["kv"]["admission_rejected"].is_null(),
"null, not 0: `not counted` and `counted none` are different facts"
);
let notes = r["unproduced_fields"].as_array().expect("array");
assert!(
notes.iter().any(|n| n
.as_str()
.is_some_and(|t| t.contains("kv.admission_rejected"))),
"the missing counters are NAMED: {notes:?}"
);
let complete = KvBlock::from_server_report(50, 100, Some(0), Some(3));
assert!(complete.uncounted_fields().is_empty());
let full = receipt_input(Some(complete)).render().expect("renders");
assert_eq!(full["kv"]["admission_rejected"], 0);
assert_eq!(full["kv"]["preempted_swap"], 3);
assert!(
!full["unproduced_fields"]
.as_array()
.expect("array")
.iter()
.any(|n| n
.as_str()
.is_some_and(|t| t.contains("kv.admission_rejected"))),
"a complete block has nothing to name"
);
}
#[test]
fn join_ok__the_receipt_builds_its_own_join_key() {
let input = receipt_input(None);
let key = input.join_key(&input.bands[1]);
assert_eq!(key.host, "lambda");
assert_eq!(key.band, 4);
assert_eq!(key.window_ms, 60_000);
assert_eq!(key.replicates, 5);
assert!(key.interleaved);
assert_eq!(key.n_predict, 128);
assert_eq!(key.n_batch, Some(2048));
key.refuse_mismatch(&input.join_key(&input.bands[1]))
.expect("a key joins itself");
let elsewhere = ReceiptInput {
provenance: Provenance {
host: "gx10".to_string(),
..provenance()
},
..receipt_input(None)
};
let err = key
.refuse_mismatch(&elsewhere.join_key(&elsewhere.bands[1]))
.expect_err("two hosts");
assert!(err.contains("host: lambda != gx10"), "{err}");
let r = input.render().expect("renders");
assert_eq!(r["bands"][1]["join_key"]["band"], 4);
assert_eq!(r["bands"][1]["join_key"]["window_ms"], 60_000);
assert_eq!(r["bands"][1]["join_key"]["interleaved"], true);
}
#[test]
fn a_receipt_round_trips_through_its_own_type() {
let text = receipt_input(None).render_string().expect("renders");
let parsed = Receipt::parse(&text).expect("parses");
parsed.validate().expect("and satisfies its own L1 rules");
assert_eq!(parsed.spec, SPEC_ID);
assert_eq!(parsed.schema_version, 3);
assert_eq!(parsed.run_id, run_id());
assert_eq!(parsed.bands.len(), 4);
assert_eq!(parsed.bands[0].status, "UNMEASURED");
assert_eq!(parsed.protocol, ProtocolParams::spec_fallback());
let again = serde_json::to_string(&parsed).expect("serialises");
let twice = Receipt::parse(&again).expect("re-parses");
assert_eq!(parsed, twice, "the type is its own fixed point");
}
#[test]
fn a_signed_receipt_parses_and_an_unsigned_one_carries_no_signature_key() {
let unsigned = rendered();
assert!(
unsigned.get("signature").is_none(),
"the renderer must never sign"
);
let mut signed = unsigned.clone();
signed.as_object_mut().expect("object").insert(
"signature".to_string(),
serde_json::json!({"alg": "hmac-sha256", "key_id": "lambda-1",
"signed_at": "2026-09-02T00:00:00Z", "commit": "0".repeat(40),
"host": "lambda", "body_sha256": "0".repeat(64), "value": "00"}),
);
let parsed = Receipt::parse(&signed.to_string()).expect("a signed receipt parses");
assert_eq!(
parsed.signature.as_ref().and_then(|s| s["key_id"].as_str()),
Some("lambda-1")
);
let back = serde_json::to_value(&parsed).expect("serialises");
assert_eq!(back["signature"]["alg"], "hmac-sha256");
let plain = Receipt::parse(&unsigned.to_string()).expect("unsigned parses");
assert!(plain.signature.is_none());
}
#[test]
fn an_unknown_field_is_refused() {
let mut value = rendered();
value
.as_object_mut()
.expect("object")
.insert("agg_ratio".to_string(), serde_json::json!(1.42));
let err = Receipt::parse(&value.to_string()).expect_err("unknown top-level field");
assert!(err.contains("agg_ratio"), "{err}");
let mut banded = rendered();
banded["bands"][0]
.as_object_mut()
.expect("object")
.insert("agg_ratio".to_string(), serde_json::json!(1.42));
assert!(
Receipt::parse(&banded.to_string()).is_err(),
"and the same inside a band"
);
}
#[test]
fn claim_bandless__ratios_without_a_baseline_are_refused_by_the_reader() {
let mut value = rendered();
value["bands"][0].as_object_mut().expect("object").insert(
"ratios".to_string(),
serde_json::json!({
"agg": {"point": 1.42, "lcb95": null, "method": "replicate_t_lower", "n": 1},
"dec": null,
"prefill": null
}),
);
let parsed = Receipt::parse(&value.to_string()).expect("the shape parses");
let err = parsed.validate().expect_err("but does not validate");
assert!(err.contains("PP-3"), "{err}");
assert!(err.contains("baseline"), "{err}");
}
#[test]
fn a_status_outside_the_vocabulary_is_refused_by_the_reader() {
let mut value = rendered();
value["bands"][0]["status"] = serde_json::json!("SKIP");
let parsed = Receipt::parse(&value.to_string()).expect("a string parses");
let err = parsed.validate().expect_err("SKIP is not a status");
assert!(err.contains("ยง7.4"), "{err}");
}
#[test]
fn the_protocol_block_is_on_the_wire() {
let r = rendered();
let p = &r["protocol"];
assert_eq!(p["window_ms"], 60_000);
assert_eq!(p["warmup_requests_per_worker"], 2);
assert_eq!(p["quiesce_ms"], 5_000);
assert_eq!(p["cooldown_ms"], 10_000);
assert_eq!(p["n_predict"], 128);
assert_eq!(p["replicates"], 5);
assert_eq!(p["interleaved"], true);
assert_eq!(p["sampler"]["temperature"], 0.0);
assert_eq!(p["sampler"]["ignore_eos"], true);
assert!(p["sampler"]["seed"].is_u64());
}
#[test]
fn sampler_pinned__a_conformant_receipt_reports_no_short_samples() {
let r = rendered();
assert_eq!(r["short_of_n_predict"], 0);
for b in r["bands"].as_array().expect("bands") {
assert_eq!(b["short_of_n_predict"], 0);
assert_eq!(b["stream_mode"], "live");
assert_eq!(b["stream_witness"]["verdict"], "live");
}
let mut input = receipt_input(None);
input.bands[0].requests[3].generated_tokens = 67;
input.bands[2].requests[5].generated_tokens = 120;
let short = input.render().expect("the evidence still renders");
assert_eq!(short["short_of_n_predict"], 2);
assert_eq!(short["bands"][0]["short_of_n_predict"], 1);
assert_eq!(short["bands"][0]["status"], "NONCONFORMANT-VALID");
assert_eq!(short["bands"][1]["short_of_n_predict"], 0);
}
}