use super::*;
#[derive(Debug, Clone, serde::Serialize)]
pub(crate) struct Finding {
pub pairing_key: PairingKey,
pub scenario: String,
pub topology: String,
pub work_type: String,
pub metric: &'static MetricDef,
pub val_a: f64,
pub val_b: f64,
pub delta: f64,
pub kind: FindingKind,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
pub(crate) enum FindingKind {
Regression,
Improvement,
Informational,
}
#[derive(Debug, Clone, serde::Serialize)]
pub(crate) struct CoverageDiff {
pub pairing_key: PairingKey,
pub scenario: String,
pub topology: String,
pub work_type: String,
pub metric: &'static MetricDef,
pub present_side: ComparePartition,
pub value: f64,
}
#[derive(Debug, Clone, Default, serde::Serialize)]
pub(crate) struct CompareReport {
pub regressions: u32,
pub improvements: u32,
pub informational: u32,
pub unchanged: u32,
pub excluded_pairs: u32,
pub new_in_b: u32,
pub removed_from_a: u32,
pub findings: Vec<Finding>,
pub coverage_diffs: Vec<CoverageDiff>,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Serialize)]
pub(crate) enum ComparePartition {
A,
B,
}
impl ComparePartition {
pub fn as_str(self) -> &'static str {
match self {
Self::A => "A",
Self::B => "B",
}
}
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct ComparisonPolicy {
pub default_percent: Option<f64>,
pub per_metric_percent: BTreeMap<String, f64>,
}
#[derive(Debug, Default, Clone)]
pub struct PhaseDisplayOptions {
pub no_phases: bool,
pub phases_only: bool,
pub steps_only: bool,
pub phase: Option<u16>,
pub phase_threshold: Option<f64>,
}
impl PhaseDisplayOptions {
pub fn rel_threshold(
&self,
policy: &ComparisonPolicy,
metric_name: &str,
default_rel: f64,
) -> f64 {
match self.phase_threshold {
Some(pct) => pct / 100.0,
None => policy.rel_threshold(metric_name, default_rel),
}
}
pub fn matches_phase(&self, step_index: u16) -> bool {
if let Some(want) = self.phase
&& step_index != want
{
return false;
}
if self.steps_only && step_index == 0 {
return false;
}
true
}
pub(crate) fn passes_noise_spread_threshold(&self, verdict: &NoiseVerdict) -> bool {
let Some(pct) = self.phase_threshold else {
return true;
};
let a = verdict.a.mean.abs();
let delta = (verdict.b.mean - verdict.a.mean).abs();
let rel = if a > ZERO_MEAN_EPS {
delta / a
} else if delta > ZERO_MEAN_EPS {
f64::INFINITY
} else {
0.0
};
rel >= pct / 100.0
}
}
impl ComparisonPolicy {
pub fn new() -> Self {
Self::default()
}
pub fn uniform(percent: f64) -> Self {
Self {
default_percent: Some(percent),
per_metric_percent: BTreeMap::new(),
}
}
pub fn load_json(path: &std::path::Path) -> anyhow::Result<Self> {
use anyhow::Context;
let data = std::fs::read_to_string(path)
.with_context(|| format!("read comparison policy from {}", path.display()))?;
let policy: ComparisonPolicy = serde_json::from_str(&data)
.with_context(|| format!("parse comparison policy from {}", path.display()))?;
policy
.validate()
.with_context(|| format!("validate comparison policy from {}", path.display()))?;
Ok(policy)
}
pub fn validate(&self) -> anyhow::Result<()> {
if let Some(p) = self.default_percent
&& p < 0.0
{
anyhow::bail!(
"ComparisonPolicy: default_percent must be non-negative; got {p}. \
Thresholds are absolute-value comparisons — a negative value \
would invert the dual-gate logic and silently classify every \
delta as significant."
);
}
for (name, p) in &self.per_metric_percent {
if !METRICS.iter().any(|m| m.name == name) {
let known: Vec<&str> = METRICS.iter().map(|m| m.name).collect();
anyhow::bail!(
"ComparisonPolicy: per_metric_percent contains unknown \
metric `{name}`. A typo in the key would silently fall \
through to default_percent. Registered metrics: {}",
known.join(", "),
);
}
if *p < 0.0 {
anyhow::bail!(
"ComparisonPolicy: per_metric_percent[{name:?}] must be \
non-negative; got {p}",
);
}
}
Ok(())
}
pub fn from_cli_flags(
threshold: Option<f64>,
policy: Option<&std::path::Path>,
) -> anyhow::Result<Self> {
match (threshold, policy) {
(Some(t), None) => {
let p = Self::uniform(t);
p.validate()?;
Ok(p)
}
(None, Some(path)) => Self::load_json(path),
(None, None) => Ok(Self::default()),
(Some(_), Some(_)) => anyhow::bail!(
"--threshold and --policy are mutually exclusive; use --policy \
for per-metric overrides"
),
}
}
pub fn rel_threshold(&self, metric_name: &str, default_rel: f64) -> f64 {
if let Some(p) = self.per_metric_percent.get(metric_name) {
p / 100.0
} else if let Some(p) = self.default_percent {
p / 100.0
} else {
default_rel
}
}
}
pub(crate) fn compare_rows_by(
rows_a: &[GauntletRow],
rows_b: &[GauntletRow],
pairing_dims: &[Dimension],
filter: Option<&str>,
policy: &ComparisonPolicy,
) -> CompareReport {
let mut report = CompareReport::default();
let mut a_by_key: HashMap<PairingKey, &GauntletRow> = HashMap::with_capacity(rows_a.len());
for row_a in rows_a {
let key = PairingKey::from_row(row_a, pairing_dims);
a_by_key.entry(key).or_insert(row_a);
}
let rel_thresholds: Vec<f64> = METRICS
.iter()
.map(|m| policy.rel_threshold(m.name, m.default_rel))
.collect();
let suppressed: Vec<bool> = METRICS
.iter()
.map(|m| is_render_suppressed_component(m.name))
.collect();
for row_b in rows_b {
let key_b = PairingKey::from_row(row_b, pairing_dims);
if let Some(f) = filter {
let joined = format!(
"{} {} {} {}",
row_b.scenario, row_b.topology, row_b.scheduler, row_b.work_type,
);
if !joined.contains(f) {
continue;
}
}
let Some(&row_a) = a_by_key.get(&key_b) else {
report.new_in_b += 1;
continue;
};
if row_a.is_fail()
|| row_b.is_fail()
|| row_a.is_inconclusive()
|| row_b.is_inconclusive()
|| row_a.is_skip()
|| row_b.is_skip()
|| row_a.expected_failure
|| row_b.expected_failure
{
report.excluded_pairs += 1;
continue;
}
push_scalar_findings(
&mut report,
row_a,
row_b,
&key_b,
&rel_thresholds,
&suppressed,
);
}
let b_keys: HashSet<PairingKey> = rows_b
.iter()
.map(|r| PairingKey::from_row(r, pairing_dims))
.collect();
for row_a in rows_a {
let key_a = PairingKey::from_row(row_a, pairing_dims);
if let Some(f) = filter {
let joined = format!(
"{} {} {} {}",
row_a.scenario, row_a.topology, row_a.scheduler, row_a.work_type,
);
if !joined.contains(f) {
continue;
}
}
if !b_keys.contains(&key_a) {
report.removed_from_a += 1;
}
}
report
}
fn push_scalar_findings(
report: &mut CompareReport,
row_a: &GauntletRow,
row_b: &GauntletRow,
key_b: &PairingKey,
rel_thresholds: &[f64],
suppressed: &[bool],
) {
for (i, m) in METRICS.iter().enumerate() {
if suppressed[i] {
continue;
}
let (val_a, val_b) = match (m.read(row_a), m.read(row_b)) {
(Some(a), Some(b)) => (a, b),
(None, None) => continue,
(Some(a), None) => {
report.coverage_diffs.push(CoverageDiff {
pairing_key: key_b.clone(),
scenario: row_b.scenario.clone(),
topology: row_b.topology.clone(),
work_type: row_b.work_type.clone(),
metric: m,
present_side: ComparePartition::A,
value: a,
});
continue;
}
(None, Some(b)) => {
report.coverage_diffs.push(CoverageDiff {
pairing_key: key_b.clone(),
scenario: row_b.scenario.clone(),
topology: row_b.topology.clone(),
work_type: row_b.work_type.clone(),
metric: m,
present_side: ComparePartition::B,
value: b,
});
continue;
}
};
if val_a.abs() < ZERO_MEAN_EPS && val_b.abs() < ZERO_MEAN_EPS {
continue;
}
let rel_thresh = rel_thresholds[i];
let delta = val_b - val_a;
let rel_delta = if val_a.abs() > ZERO_MEAN_EPS {
(delta / val_a).abs()
} else {
f64::INFINITY
};
if delta.abs() < m.default_abs || rel_delta < rel_thresh {
report.unchanged += 1;
continue;
}
let kind = match m.classify_direction() {
None => {
report.informational += 1;
FindingKind::Informational
}
Some(higher_is_worse) => {
let is_regression = if higher_is_worse {
delta > 0.0
} else {
delta < 0.0
};
if is_regression {
report.regressions += 1;
FindingKind::Regression
} else {
report.improvements += 1;
FindingKind::Improvement
}
}
};
report.findings.push(Finding {
pairing_key: key_b.clone(),
scenario: row_b.scenario.clone(),
topology: row_b.topology.clone(),
work_type: row_b.work_type.clone(),
metric: m,
val_a,
val_b,
delta,
kind,
});
}
}
fn warn_on_dirty_builds(rows_a: &[GauntletRow], rows_b: &[GauntletRow]) {
if let Some(text) = render_dirty_warning(rows_a, rows_b) {
eprint!("{text}");
}
}
fn warn_on_overcommit(rows_a: &[GauntletRow], rows_b: &[GauntletRow], pairing_dims: &[Dimension]) {
if let Some(text) = render_overcommit_warning(rows_a, rows_b, pairing_dims) {
eprint!("{text}");
}
}
pub(crate) fn render_overcommit_warning(
rows_a: &[GauntletRow],
rows_b: &[GauntletRow],
pairing_dims: &[Dimension],
) -> Option<String> {
use std::collections::BTreeSet;
use std::fmt::Write;
let overcommitted = |rows: &[GauntletRow]| -> BTreeSet<(u32, u32)> {
let mut over = BTreeSet::new();
for r in rows {
if let (Some(b), Some(v)) = (r.cpu_budget, r.vcpus)
&& b < v
{
over.insert((b, v));
}
}
over
};
let cpu_budget_is_pairing = pairing_dims.contains(&Dimension::CpuBudget);
let mixed_folded = |rows: &[GauntletRow]| -> BTreeSet<u32> {
let mut folded = BTreeSet::new();
if cpu_budget_is_pairing {
return folded;
}
let mut by_key: std::collections::HashMap<PairingKey, BTreeSet<u32>> =
std::collections::HashMap::new();
for r in rows {
if let Some(b) = r.cpu_budget {
by_key
.entry(PairingKey::from_row(r, pairing_dims))
.or_default()
.insert(b);
}
}
for budgets in by_key.values() {
if budgets.len() > 1 {
folded.extend(budgets.iter().copied());
}
}
folded
};
let over_a = overcommitted(rows_a);
let over_b = overcommitted(rows_b);
let mixed_a = mixed_folded(rows_a);
let mixed_b = mixed_folded(rows_b);
if over_a.is_empty() && over_b.is_empty() && mixed_a.is_empty() && mixed_b.is_empty() {
return None;
}
let any_overcommit = !over_a.is_empty() || !over_b.is_empty();
let mut out = String::new();
if any_overcommit {
let _ = writeln!(
out,
"ktstr: WARNING: CPU-budget hazard in this comparison — a run was \
host-overcommitted, so its guest-scheduler timing metrics \
(wake-latency / off-CPU / run-delay) are host-contention-confounded. \
Compare the overcommit-invariant worst_iterations_per_cpu_sec metric \
instead of raw \
timing."
);
} else {
let _ = writeln!(
out,
"ktstr: WARNING: CPU-budget hazard in this comparison — runs of \
different CPU budgets share a pairing group, mixing two measurement \
conditions. Slice with --cpu-budget, or compare the budget-invariant \
worst_iterations_per_cpu_sec metric."
);
}
let mut emit_side = |label: &str, over: &BTreeSet<(u32, u32)>, mixed: &BTreeSet<u32>| {
if !over.is_empty() {
let list = over
.iter()
.map(|(b, v)| format!("{b}/{v}"))
.collect::<Vec<_>>()
.join(", ");
let _ = writeln!(
out,
" side {label}: host-overcommitted run(s) [budget/vcpus]: {list}"
);
}
if !mixed.is_empty() {
let list = mixed
.iter()
.map(|b| b.to_string())
.collect::<Vec<_>>()
.join(", ");
let _ = writeln!(
out,
" side {label}: CPU budgets [{list}] share a pairing group — \
the average fold collapses them into one mean; slice with --cpu-budget so cross-budget runs are \
not compared under one key"
);
}
};
emit_side("A", &over_a, &mixed_a);
emit_side("B", &over_b, &mixed_b);
Some(out)
}
pub(crate) fn render_dirty_warning(
rows_a: &[GauntletRow],
rows_b: &[GauntletRow],
) -> Option<String> {
use std::collections::BTreeSet;
use std::fmt::Write;
let mut dirty_kernel: BTreeSet<&str> = BTreeSet::new();
let mut dirty_project: BTreeSet<&str> = BTreeSet::new();
for row in rows_a.iter().chain(rows_b.iter()) {
if let Some(c) = row.kernel_commit.as_deref()
&& c.ends_with("-dirty")
{
dirty_kernel.insert(c);
}
if let Some(c) = row.commit.as_deref()
&& c.ends_with("-dirty")
{
dirty_project.insert(c);
}
}
if dirty_kernel.is_empty() && dirty_project.is_empty() {
return None;
}
let mut out = String::new();
writeln!(out, "warning: comparison includes dirty builds:").unwrap();
for v in &dirty_kernel {
writeln!(
out,
" - kernel source: {v} (working tree may have changed since this run)"
)
.unwrap();
}
for v in &dirty_project {
writeln!(
out,
" - project: {v} (working tree may have changed since this run)"
)
.unwrap();
}
writeln!(
out,
" Dirty runs overwrite previous results with the same HEAD."
)
.unwrap();
writeln!(out, " Commit changes for reproducible-ish comparisons.").unwrap();
Some(out)
}
pub(crate) fn zero_match_diagnostic(
side: &str,
filter: &RowFilter,
rows: &[GauntletRow],
pool_len: usize,
) -> String {
let mut msg = format!(
"perf-delta: {side} side filter matched 0 sidecars in \
pool ({pool_len} pooled). Check the per-side filters or \
confirm the runs exist with `cargo ktstr stats list`."
);
let mut dirty_hints: Vec<String> = Vec::new();
for want in &filter.project_commits {
let dirty = format!("{want}-dirty");
let found = rows
.iter()
.any(|r| r.commit.as_deref() == Some(dirty.as_str()));
if found {
dirty_hints.push(format!(
"no rows match `--project-commit {want}` but `{dirty}` exists in the pool — \
did you mean `--project-commit {dirty}`?"
));
}
}
for want in &filter.kernel_commits {
let dirty = format!("{want}-dirty");
let found = rows
.iter()
.any(|r| r.kernel_commit.as_deref() == Some(dirty.as_str()));
if found {
dirty_hints.push(format!(
"no rows match `--kernel-commit {want}` but `{dirty}` exists in the pool — \
did you mean `--kernel-commit {dirty}`?"
));
}
}
for hint in dirty_hints {
msg.push_str("\nhint: ");
msg.push_str(&hint);
}
if !filter.run_sources.is_empty() {
let pool_run_sources: std::collections::BTreeSet<&str> = rows
.iter()
.filter_map(|r| r.run_source.as_deref())
.collect();
let unknowns: Vec<&str> = filter
.run_sources
.iter()
.map(String::as_str)
.filter(|want| !pool_run_sources.contains(*want))
.collect();
if !unknowns.is_empty() {
let mut present: Vec<&str> = pool_run_sources.iter().copied().collect();
present.sort_unstable();
let unknown_list = unknowns
.iter()
.map(|s| format!("`{s}`"))
.collect::<Vec<_>>()
.join(", ");
let present_list = if present.is_empty() {
"(none — every row has `run_source: null`)".to_string()
} else {
present
.iter()
.map(|s| format!("`{s}`"))
.collect::<Vec<_>>()
.join(", ")
};
msg.push_str(&format!(
"\nhint: --run-source {unknown_list} not found in pool; \
distinct values present: {present_list}. Values are \
case-sensitive (`ci` ≠ `CI`)."
));
}
}
if !filter.resolve_sources.is_empty() {
let pool_resolve_sources: std::collections::BTreeSet<&str> = rows
.iter()
.filter_map(|r| r.resolve_source.as_deref())
.collect();
let unknowns: Vec<&str> = filter
.resolve_sources
.iter()
.map(String::as_str)
.filter(|want| !pool_resolve_sources.contains(*want))
.collect();
if !unknowns.is_empty() {
let mut present: Vec<&str> = pool_resolve_sources.iter().copied().collect();
present.sort_unstable();
let unknown_list = unknowns
.iter()
.map(|s| format!("`{s}`"))
.collect::<Vec<_>>()
.join(", ");
let present_list = if present.is_empty() {
"(none — every row has `resolve_source: null`)".to_string()
} else {
present
.iter()
.map(|s| format!("`{s}`"))
.collect::<Vec<_>>()
.join(", ")
};
msg.push_str(&format!(
"\nhint: --resolve-source {unknown_list} not found in pool; \
distinct values present: {present_list}. Values are \
case-sensitive (`auto_built` \u{2260} `Auto_Built`)."
));
}
}
if !filter.cpu_budgets.is_empty() {
let pool_budgets: std::collections::BTreeSet<u32> =
rows.iter().filter_map(|r| r.cpu_budget).collect();
let present_strs: std::collections::BTreeSet<String> =
pool_budgets.iter().map(|b| b.to_string()).collect();
let unknowns: Vec<&str> = filter
.cpu_budgets
.iter()
.map(String::as_str)
.filter(|want| !present_strs.contains(*want))
.collect();
if !unknowns.is_empty() {
let unknown_list = unknowns
.iter()
.map(|s| format!("`{s}`"))
.collect::<Vec<_>>()
.join(", ");
let present_list = if pool_budgets.is_empty() {
"(none — every row is a skip with no recorded budget)".to_string()
} else {
pool_budgets
.iter()
.map(|b| format!("`{b}`"))
.collect::<Vec<_>>()
.join(", ")
};
msg.push_str(&format!(
"\nhint: --cpu-budget {unknown_list} not found in pool; \
distinct budgets present: {present_list}."
));
}
}
let touched_commit_dim =
!filter.project_commits.is_empty() || !filter.kernel_commits.is_empty();
if touched_commit_dim {
msg.push_str(
"\nhint: run `cargo ktstr stats list-values` to see every \
distinct commit value present in the pool — the specific \
value the filter expected may not have a sidecar yet, or \
may differ from what was recorded by \
`detect_project_commit` / `detect_kernel_commit`.",
);
}
msg
}
struct PartitionedComparison {
slicing_dims: Vec<Dimension>,
pairing_dims: Vec<Dimension>,
pool: Vec<crate::test_support::SidecarResult>,
rows: Vec<GauntletRow>,
rows_a_for_compare: Vec<GauntletRow>,
rows_b_for_compare: Vec<GauntletRow>,
avg_a: Option<Vec<AveragedGroup>>,
avg_b: Option<Vec<AveragedGroup>>,
pre_agg_a: usize,
pre_agg_b: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RowPrep {
Averaged,
PerRunPooled,
}
fn prepare_partitioned_comparison(
filter_a: &RowFilter,
filter_b: &RowFilter,
dir: Option<&std::path::Path>,
prep: RowPrep,
) -> anyhow::Result<PartitionedComparison> {
let slicing_dims = derive_slicing_dims(filter_a, filter_b);
if slicing_dims.is_empty() {
anyhow::bail!(
"perf-delta: A and B select identical rows. \
Specify at least one per-side filter (e.g. \
--a-kernel 6.14 --b-kernel 6.15) to define what \
dimension separates the two sides."
);
}
if slicing_dims.len() > 1 {
let dim_names: Vec<&str> = slicing_dims.iter().map(|d| d.name()).collect();
eprintln!(
"warning: perf-delta: slicing on {n} dimensions [{dims}]; \
results compress multiple axes into a single A/B contrast.",
n = slicing_dims.len(),
dims = dim_names.join(", "),
);
}
let pairing_dims = Dimension::pairing_dims(&slicing_dims);
let (root, override_archive) = match dir {
Some(d) => (d.to_path_buf(), true),
None => (crate::test_support::runs_root(), false),
};
let mut pool = crate::test_support::collect_pool(&root);
if override_archive {
crate::test_support::apply_archive_source_override(&mut pool);
}
if pool.is_empty() {
anyhow::bail!(
"perf-delta: no sidecar data found under {}. \
Run `cargo ktstr test` to generate runs, or pass \
--dir to point at an archived sidecar tree.",
root.display(),
);
}
let rows: Vec<GauntletRow> = pool.iter().map(sidecar_to_row).collect();
let rows_a = apply_row_filters(&rows, filter_a);
let rows_b = apply_row_filters(&rows, filter_b);
if rows_a.is_empty() {
anyhow::bail!(
"{}",
zero_match_diagnostic("A", filter_a, &rows, pool.len()),
);
}
if rows_b.is_empty() {
anyhow::bail!(
"{}",
zero_match_diagnostic("B", filter_b, &rows, pool.len()),
);
}
warn_on_dirty_builds(&rows_a, &rows_b);
warn_on_overcommit(&rows_a, &rows_b, &pairing_dims);
let pre_agg_a = rows_a.len();
let pre_agg_b = rows_b.len();
let (rows_a_for_compare, rows_b_for_compare, avg_a, avg_b) = match prep {
RowPrep::Averaged => {
let avg_a = group_and_average_by(&rows_a, &pairing_dims);
let avg_b = group_and_average_by(&rows_b, &pairing_dims);
let a_rows: Vec<GauntletRow> = avg_a.iter().map(|r| r.row.clone()).collect();
let b_rows: Vec<GauntletRow> = avg_b.iter().map(|r| r.row.clone()).collect();
(a_rows, b_rows, Some(avg_a), Some(avg_b))
}
RowPrep::PerRunPooled => {
(rows_a, rows_b, None, None)
}
};
Ok(PartitionedComparison {
slicing_dims,
pairing_dims,
pool,
rows,
rows_a_for_compare,
rows_b_for_compare,
avg_a,
avg_b,
pre_agg_a,
pre_agg_b,
})
}
pub(crate) fn scalar_declared_gate_warning(rows_b: &[GauntletRow]) -> Option<String> {
let n = rows_b
.iter()
.filter(|r| !r.perf_delta_assertions.is_empty())
.map(|r| r.scenario.as_str())
.collect::<std::collections::BTreeSet<_>>()
.len();
(n > 0).then(|| {
format!(
"NOTE — {n} compared test(s) declare perf_delta_assertions gate(s) that this \
scalar comparison does NOT evaluate. Declared gates are enforced only under \
`perf-delta --noise-adjust N` (single-run scalar gating would flip CI on \
noise); re-run with --noise-adjust to gate on them."
)
})
}
pub fn compare_partitions(
filter_a: &RowFilter,
filter_b: &RowFilter,
filter: Option<&str>,
policy: &ComparisonPolicy,
dir: Option<&std::path::Path>,
gate: &GateOptions,
) -> anyhow::Result<i32> {
let prepared = prepare_partitioned_comparison(filter_a, filter_b, dir, RowPrep::Averaged)?;
let PartitionedComparison {
slicing_dims,
pairing_dims,
pool,
rows,
rows_a_for_compare,
rows_b_for_compare,
avg_a,
avg_b,
pre_agg_a,
pre_agg_b,
} = &prepared;
let report = compare_rows_by(
rows_a_for_compare,
rows_b_for_compare,
pairing_dims,
filter,
policy,
);
let label_a = render_side_label(filter_a, slicing_dims, "A");
let label_b = render_side_label(filter_b, slicing_dims, "B");
let slice_names: Vec<&str> = slicing_dims.iter().map(|d| d.name()).collect();
let pair_names: Vec<&str> = pairing_dims.iter().map(|d| d.name()).collect();
println!("slicing dimensions: {}", slice_names.join(", "));
println!(
"pairing on: scenario{}{}",
if pair_names.is_empty() { "" } else { ", " },
pair_names.join(", "),
);
if let Some(warning) = scalar_declared_gate_warning(rows_b_for_compare) {
println!("{warning}");
}
println!(
"{}",
format_average_header(*pre_agg_a, *pre_agg_b, &label_a, &label_b)
);
print_scalar_findings_table(&report, &label_a, &label_b);
print_summary_block(&report, avg_a, avg_b, &label_a, &label_b);
print_host_context_delta(pool, rows, filter_a, filter_b, &label_a, &label_b);
let regressing: Vec<&str> = report
.findings
.iter()
.filter(|f| f.kind == FindingKind::Regression)
.map(|f| f.metric.name)
.collect();
Ok(if gate_fails(®ressing, gate) { 1 } else { 0 })
}
#[derive(Debug, Clone, Default)]
pub struct GateOptions {
pub fail_threshold: Option<usize>,
pub must_fail: Vec<String>,
pub show_all: bool,
}
pub(crate) fn gate_fails(regressing_metrics: &[&str], gate: &GateOptions) -> bool {
let n = gate.fail_threshold.unwrap_or(5);
let fail_on_count = n >= 1 && regressing_metrics.len() >= n;
let fail_on_must = !gate.must_fail.is_empty()
&& regressing_metrics
.iter()
.any(|m| gate.must_fail.iter().any(|w| w.as_str() == *m));
fail_on_count || fail_on_must
}
#[cfg(test)]
mod gate_option_tests {
use super::*;
#[test]
fn default_gate_fails_only_at_five_regressions() {
let g = GateOptions::default();
assert!(!gate_fails(&[], &g), "0 regressions passes");
assert!(
!gate_fails(&["a", "b", "c", "d"], &g),
"4 < 5 passes under the default gate"
);
assert!(
gate_fails(&["a", "b", "c", "d", "e"], &g),
"5 >= 5 fails under the default gate"
);
}
#[test]
fn count_threshold_requires_n() {
let g = GateOptions {
fail_threshold: Some(3),
..Default::default()
};
assert!(!gate_fails(&["a", "b"], &g), "2 < 3 passes");
assert!(gate_fails(&["a", "b", "c"], &g), "3 >= 3 fails");
}
#[test]
fn zero_threshold_disables_count_gate() {
let g = GateOptions {
fail_threshold: Some(0),
..Default::default()
};
assert!(
!gate_fails(&["a", "b", "c"], &g),
"N=0 never fails on the count"
);
}
#[test]
fn must_fail_fails_regardless_of_count() {
let g = GateOptions {
fail_threshold: Some(0),
must_fail: vec!["worst_p99_wake_latency_us".to_string()],
..Default::default()
};
assert!(
gate_fails(&["worst_p99_wake_latency_us"], &g),
"a must-fail metric regressing fails even with the count gate off"
);
assert!(
!gate_fails(&["some_other_metric"], &g),
"a non-must-fail regression does not fail with the count gate off"
);
}
#[test]
fn must_fail_is_ored_above_the_count() {
let g = GateOptions {
fail_threshold: Some(10),
must_fail: vec!["avg_dsq_depth".to_string()],
..Default::default()
};
assert!(
gate_fails(&["avg_dsq_depth"], &g),
"must-fail fires even below the count threshold"
);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum NoiseKind {
Regression,
Improvement,
Noisy,
Informational,
Stable,
}
pub(crate) struct NoiseFinding {
pub pairing_label: String,
pub metric: &'static MetricDef,
pub verdict: NoiseVerdict,
pub kind: NoiseKind,
pub gated_by_assertion: bool,
}
pub(crate) struct NoisePhaseFinding {
pub pairing_label: String,
pub step_index: u16,
pub label: String,
pub metric: &'static MetricDef,
pub verdict: NoiseVerdict,
pub kind: NoiseKind,
pub gated_by_assertion: bool,
}
pub(crate) struct NoisePhaseCoverage {
pub pairing_label: String,
pub step_index: u16,
pub label: String,
pub metric: Option<&'static MetricDef>,
pub present_side: ComparePartition,
pub value: Option<f64>,
}
pub(crate) struct NoiseAssertionCoverage {
pub pairing_label: String,
pub assertion: crate::test_support::PerfDeltaAssertionRecord,
}
pub(crate) struct NoiseReport {
pub findings: Vec<NoiseFinding>,
pub phase_findings: Vec<NoisePhaseFinding>,
pub phase_coverage: Vec<NoisePhaseCoverage>,
pub assertion_coverage: Vec<NoiseAssertionCoverage>,
pub paired_scenarios: usize,
}
impl NoiseReport {
pub fn regressions(&self) -> usize {
self.findings
.iter()
.filter(|f| f.kind == NoiseKind::Regression)
.count()
}
pub fn declared_regressions(&self) -> usize {
self.findings
.iter()
.filter(|f| f.kind == NoiseKind::Regression && f.gated_by_assertion)
.count()
}
pub fn noisy(&self) -> usize {
self.findings
.iter()
.filter(|f| f.kind == NoiseKind::Noisy)
.count()
}
pub fn improvements(&self) -> usize {
self.findings
.iter()
.filter(|f| f.kind == NoiseKind::Improvement)
.count()
}
pub fn stable(&self) -> usize {
self.findings
.iter()
.filter(|f| f.kind == NoiseKind::Stable)
.count()
}
pub fn informational(&self) -> usize {
self.findings
.iter()
.filter(|f| f.kind == NoiseKind::Informational)
.count()
}
pub fn phase_regressions(&self) -> usize {
self.phase_findings
.iter()
.filter(|f| f.kind == NoiseKind::Regression)
.count()
}
pub fn declared_phase_regressions(&self) -> usize {
self.phase_findings
.iter()
.filter(|f| f.kind == NoiseKind::Regression && f.gated_by_assertion)
.count()
}
pub fn phase_noisy(&self) -> usize {
self.phase_findings
.iter()
.filter(|f| f.kind == NoiseKind::Noisy)
.count()
}
}
pub(crate) fn noise_findings(
rows_a: &[GauntletRow],
rows_b: &[GauntletRow],
pairing_dims: &[Dimension],
spread_threshold_pct: f64,
include_stable: bool,
) -> NoiseReport {
use std::collections::{BTreeMap, BTreeSet};
let group = |rows: &[GauntletRow]| {
let mut by_key: BTreeMap<PairingKey, Vec<GauntletRow>> = BTreeMap::new();
for r in rows {
if r.is_fail() || r.is_inconclusive() || r.is_skip() || r.expected_failure {
continue;
}
let mut row = r.clone();
crate::stats::metric::derive_rate_metrics(&mut row.ext_metrics);
by_key
.entry(PairingKey::from_row(&row, pairing_dims))
.or_default()
.push(row);
}
by_key
};
let a_by_key = group(rows_a);
let b_by_key = group(rows_b);
let mut findings = Vec::new();
let mut phase_findings = Vec::new();
let mut phase_coverage = Vec::new();
let mut assertion_coverage = Vec::new();
let mut paired_scenarios = 0usize;
for (key, a_rows) in &a_by_key {
let Some(b_rows) = b_by_key.get(key) else {
continue;
};
paired_scenarios += 1;
let mut consulted: BTreeSet<&'static str> = BTreeSet::new();
let mut consulted_phase: BTreeSet<(u16, &'static str)> = BTreeSet::new();
let pairing_label = key.0.join("/");
for m in METRICS {
if is_render_suppressed_component(m.name) {
continue;
}
let a_vals: Vec<f64> = a_rows.iter().filter_map(|r| m.read(r)).collect();
let b_vals: Vec<f64> = b_rows.iter().filter_map(|r| m.read(r)).collect();
if a_vals.is_empty() || b_vals.is_empty() {
continue;
}
let verdict = match m.kind {
MetricKind::Rate {
numerator,
denominator,
} => {
let pooled = |rows: &[GauntletRow]| -> Option<f64> {
let (num, den) = rows.iter().fold((0.0, 0.0), |(sn, sd), r| {
match (r.ext_metrics.get(numerator), r.ext_metrics.get(denominator)) {
(Some(n), Some(d)) => (sn + n, sd + d),
_ => (sn, sd),
}
});
(den != 0.0).then(|| num / den)
};
let (Some(a_pooled), Some(b_pooled)) = (pooled(a_rows), pooled(b_rows)) else {
continue;
};
noise_verdict_from(
SideSummary::of(&a_vals).with_pooled_mean(a_pooled),
SideSummary::of(&b_vals).with_pooled_mean(b_pooled),
spread_threshold_pct,
)
}
_ => noise_verdict(&a_vals, &b_vals, spread_threshold_pct),
};
consulted.insert(m.name);
let assertion = b_rows.first().and_then(|r| {
r.perf_delta_assertions
.iter()
.find(|x| x.metric == m.name && x.phase.is_none())
});
let gated_by_assertion = assertion.is_some();
let Some(kind) = classify_noise(&verdict, m, assertion, include_stable) else {
continue;
};
findings.push(NoiseFinding {
pairing_label: pairing_label.clone(),
metric: m,
verdict,
kind,
gated_by_assertion,
});
}
noise_phase_findings(
a_rows,
b_rows,
&pairing_label,
spread_threshold_pct,
include_stable,
&mut phase_findings,
&mut phase_coverage,
&mut consulted_phase,
);
if let Some(first) = b_rows.first() {
for a in &first.perf_delta_assertions {
let matched = match a.phase {
None => consulted.contains(a.metric.as_str()),
Some(step) => consulted_phase.contains(&(step, a.metric.as_str())),
};
if !matched {
assertion_coverage.push(NoiseAssertionCoverage {
pairing_label: pairing_label.clone(),
assertion: a.clone(),
});
}
}
}
}
NoiseReport {
findings,
phase_findings,
phase_coverage,
assertion_coverage,
paired_scenarios,
}
}
fn classify_noise(
verdict: &NoiseVerdict,
m: &MetricDef,
assertion: Option<&crate::test_support::PerfDeltaAssertionRecord>,
include_stable: bool,
) -> Option<NoiseKind> {
if verdict.a.mean.abs() < ZERO_MEAN_EPS && verdict.b.mean.abs() < ZERO_MEAN_EPS {
return None;
}
if verdict.insufficient_samples {
return Some(NoiseKind::Noisy);
}
let a = verdict.a.mean;
let b = verdict.b.mean;
let delta = b - a;
let rel_delta = if a.abs() > ZERO_MEAN_EPS {
(delta / a).abs()
} else if delta.abs() > ZERO_MEAN_EPS {
f64::INFINITY
} else {
0.0
};
let abs_gate = assertion
.and_then(|x| x.min_abs)
.filter(|v| v.is_finite() && *v >= 0.0)
.unwrap_or(m.default_abs);
let rel_gate = assertion
.and_then(|x| x.max_regression_pct)
.filter(|v| v.is_finite() && *v >= 0.0)
.map(|pct| pct / 100.0)
.unwrap_or(m.default_rel);
let material = delta.abs() >= abs_gate && rel_delta >= rel_gate;
let classify = match assertion.and_then(|x| x.direction) {
Some(crate::test_support::Polarity::TargetValue(_)) | None => m.classify_direction(),
Some(p) => p.classify_direction(),
};
Some(if verdict.separated && material {
match classify {
None => NoiseKind::Informational,
Some(higher_is_worse) => {
let worsened = if higher_is_worse { b > a } else { b < a };
if worsened {
NoiseKind::Regression
} else {
NoiseKind::Improvement
}
}
}
} else if include_stable {
NoiseKind::Stable
} else {
return None; })
}
fn push_noise_phase_coverage(
coverage: &mut Vec<NoisePhaseCoverage>,
pairing_label: &str,
step_index: u16,
label: &str,
metric: &'static MetricDef,
present_side: ComparePartition,
vals: &[f64],
) {
if vals.is_empty() {
return;
}
let value = vals.iter().sum::<f64>() / vals.len() as f64;
coverage.push(NoisePhaseCoverage {
pairing_label: pairing_label.to_string(),
step_index,
label: label.to_string(),
metric: Some(metric),
present_side,
value: Some(value),
});
}
fn push_noise_unpaired_step(
coverage: &mut Vec<NoisePhaseCoverage>,
pairing_label: &str,
step_index: u16,
side: ComparePartition,
buckets: &[&crate::assert::PhaseBucket],
) {
let label = buckets[0].label.clone();
let names: std::collections::BTreeSet<&str> = buckets
.iter()
.flat_map(|p| p.metrics.keys())
.map(String::as_str)
.collect();
let before = coverage.len();
for name in names {
if is_render_suppressed_component(name) {
continue;
}
let Some(m) = metric_def(name) else {
continue;
};
let vals: Vec<f64> = buckets
.iter()
.filter_map(|p| p.metrics.get(name).copied())
.collect();
push_noise_phase_coverage(coverage, pairing_label, step_index, &label, m, side, &vals);
}
if coverage.len() == before {
coverage.push(NoisePhaseCoverage {
pairing_label: pairing_label.to_string(),
step_index,
label,
metric: None,
present_side: side,
value: None,
});
}
}
#[allow(clippy::too_many_arguments)]
fn noise_phase_findings(
a_rows: &[GauntletRow],
b_rows: &[GauntletRow],
pairing_label: &str,
spread_threshold_pct: f64,
include_stable: bool,
findings: &mut Vec<NoisePhaseFinding>,
coverage: &mut Vec<NoisePhaseCoverage>,
consulted_phase: &mut std::collections::BTreeSet<(u16, &'static str)>,
) {
use std::collections::BTreeSet;
let has_phases = |rows: &[GauntletRow]| rows.iter().any(|r| !r.phases.is_empty());
if !has_phases(a_rows) || !has_phases(b_rows) {
return;
}
fn by_step(
rows: &[GauntletRow],
) -> std::collections::BTreeMap<u16, Vec<&crate::assert::PhaseBucket>> {
let mut m: std::collections::BTreeMap<u16, Vec<&crate::assert::PhaseBucket>> =
std::collections::BTreeMap::new();
for r in rows {
for p in &r.phases {
m.entry(p.step_index).or_default().push(p);
}
}
m
}
let a_by_step = by_step(a_rows);
let b_by_step = by_step(b_rows);
let steps: BTreeSet<u16> = a_by_step.keys().chain(b_by_step.keys()).copied().collect();
for step_index in steps {
match (a_by_step.get(&step_index), b_by_step.get(&step_index)) {
(Some(a_buckets), Some(b_buckets)) => {
let label = a_buckets[0].label.clone();
let names: BTreeSet<&str> = a_buckets
.iter()
.chain(b_buckets.iter())
.flat_map(|p| p.metrics.keys())
.map(String::as_str)
.collect();
for name in names {
if is_render_suppressed_component(name) {
continue;
}
let Some(m) = metric_def(name) else {
continue;
};
let a_vals: Vec<f64> = a_buckets
.iter()
.filter_map(|p| p.metrics.get(name).copied())
.collect();
let b_vals: Vec<f64> = b_buckets
.iter()
.filter_map(|p| p.metrics.get(name).copied())
.collect();
match (a_vals.is_empty(), b_vals.is_empty()) {
(true, true) => continue,
(false, true) => {
push_noise_phase_coverage(
coverage,
pairing_label,
step_index,
&label,
m,
ComparePartition::A,
&a_vals,
);
continue;
}
(true, false) => {
push_noise_phase_coverage(
coverage,
pairing_label,
step_index,
&label,
m,
ComparePartition::B,
&b_vals,
);
continue;
}
(false, false) => {}
}
let verdict = match m.kind {
MetricKind::Rate {
numerator,
denominator,
} => {
let pooled = |buckets: &[&crate::assert::PhaseBucket]| -> Option<f64> {
let (num, den) = buckets.iter().fold((0.0, 0.0), |(sn, sd), p| {
match (p.metrics.get(numerator), p.metrics.get(denominator)) {
(Some(n), Some(d)) => (sn + n, sd + d),
_ => (sn, sd),
}
});
(den != 0.0).then(|| num / den)
};
let (Some(a_pooled), Some(b_pooled)) =
(pooled(a_buckets), pooled(b_buckets))
else {
continue;
};
noise_verdict_from(
SideSummary::of(&a_vals).with_pooled_mean(a_pooled),
SideSummary::of(&b_vals).with_pooled_mean(b_pooled),
spread_threshold_pct,
)
}
_ => noise_verdict(&a_vals, &b_vals, spread_threshold_pct),
};
consulted_phase.insert((step_index, m.name));
let assertion = b_rows.first().and_then(|r| {
r.perf_delta_assertions
.iter()
.find(|x| x.metric == m.name && x.phase == Some(step_index))
});
let gated_by_assertion = assertion.is_some();
let Some(kind) = classify_noise(&verdict, m, assertion, include_stable) else {
continue;
};
findings.push(NoisePhaseFinding {
pairing_label: pairing_label.to_string(),
step_index,
label: label.clone(),
metric: m,
verdict,
kind,
gated_by_assertion,
});
}
}
(Some(a_buckets), None) => {
push_noise_unpaired_step(
coverage,
pairing_label,
step_index,
ComparePartition::A,
a_buckets,
);
}
(None, Some(b_buckets)) => {
push_noise_unpaired_step(
coverage,
pairing_label,
step_index,
ComparePartition::B,
b_buckets,
);
}
(None, None) => {}
}
}
}
pub(crate) fn summarize_side_runs(rows: &[GauntletRow]) -> (usize, String) {
let (mut skipped, mut failed, mut inconclusive, mut xfail, mut comparable) =
(0usize, 0usize, 0usize, 0usize, 0usize);
for r in rows {
if r.is_skip() {
skipped += 1;
} else if r.is_fail() {
failed += 1;
} else if r.is_inconclusive() {
inconclusive += 1;
} else if r.expected_failure {
xfail += 1;
} else {
comparable += 1;
}
}
let mut excluded = Vec::new();
if skipped > 0 {
excluded.push(format!("{skipped} skipped"));
}
if failed > 0 {
excluded.push(format!("{failed} failed"));
}
if inconclusive > 0 {
excluded.push(format!("{inconclusive} inconclusive"));
}
if xfail > 0 {
excluded.push(format!("{xfail} expected-failure"));
}
let breakdown = if excluded.is_empty() {
"none excluded".to_string()
} else {
excluded.join(", ")
};
(
comparable,
format!(
"{} run(s): {comparable} comparable ({breakdown})",
rows.len()
),
)
}
pub fn compare_partitions_noise(
filter_a: &RowFilter,
filter_b: &RowFilter,
dir: Option<&std::path::Path>,
spread_threshold_pct: f64,
phase_opts: &PhaseDisplayOptions,
gate: &GateOptions,
) -> anyhow::Result<i32> {
let prepared = prepare_partitioned_comparison(filter_a, filter_b, dir, RowPrep::PerRunPooled)?;
let label_a = render_side_label(filter_a, &prepared.slicing_dims, "A");
let label_b = render_side_label(filter_b, &prepared.slicing_dims, "B");
let report = noise_findings(
&prepared.rows_a_for_compare,
&prepared.rows_b_for_compare,
&prepared.pairing_dims,
spread_threshold_pct,
true,
);
println!(
"perf-delta --noise-adjust: {label_b} vs {label_a} (advisory noisy-spread threshold {spread_threshold_pct:.2}%)"
);
if !phase_opts.phases_only {
if report.findings.is_empty() {
if report.paired_scenarios == 0 {
let (a_ok, a_desc) = summarize_side_runs(&prepared.rows_a_for_compare);
let (b_ok, b_desc) = summarize_side_runs(&prepared.rows_b_for_compare);
println!(
"perf-delta --noise-adjust: no comparable runs to pair across the two runs."
);
println!(" {label_a} — {a_desc}");
println!(" {label_b} — {b_desc}");
if a_ok == 0 || b_ok == 0 {
println!(
" A skipped run carries no metrics: perf-delta runs with \
KTSTR_PERF_ONLY, which skips any test not marked \
#[ktstr_test(performance_mode = true)]; host-gated skips land here \
too. A failed / inconclusive run is excluded from the spread math."
);
} else {
println!(
" Both sides produced comparable runs but share no scenario / topology \
/ work_type — the two selections have no common test to contrast."
);
}
} else {
println!(
"perf-delta --noise-adjust: no metric to display — every compared metric \
was unchanged at zero, present on only one side, or render-suppressed"
);
}
} else {
print!(
"{}",
format_noise_findings_table(&report.findings, &label_a, &label_b, gate.show_all)
);
}
}
let phase_lines = format_noise_phase_findings_lines(
&report.phase_findings,
&report.phase_coverage,
phase_opts,
&label_a,
&label_b,
gate.show_all,
);
if phase_lines.is_empty() && phase_opts.phases_only {
println!(
"perf-delta --noise-adjust: no per-phase noise data to show (no matched \
multi-phase scenario at the selected step, or every per-phase row was \
filtered by --phase / --steps-only / --phase-threshold)"
);
}
for line in phase_lines {
println!("{line}");
}
for line in format_noise_assertion_coverage_lines(&report.assertion_coverage) {
println!("{line}");
}
let regressions = report.regressions();
let declared_phase_regressions = report.declared_phase_regressions();
if !phase_opts.phases_only {
let noisy = report.noisy();
let improvements = report.improvements();
let stable = report.stable();
let informational = report.informational();
let info_clause = if informational > 0 {
format!(", {informational} informational")
} else {
String::new()
};
let verdict = overall_verdict(&report, gate);
let stable_note = if verdict == "STABLE" && regressions + improvements > 0 {
" (moves are below the significance cutoff, more likely noise than signal)"
} else {
""
};
let has_phase_data = !report.phase_findings.is_empty() || !report.phase_coverage.is_empty();
let show_phase_footer = has_phase_data
&& !phase_opts.no_phases
&& phase_opts.phase.is_none()
&& !phase_opts.steps_only
&& phase_opts.phase_threshold.is_none();
let verdict_source =
if verdict.starts_with("REGRESSED") && regressions == 0 && !show_phase_footer {
" (regression is a declared per-phase gate)"
} else {
""
};
let phase_footer = if show_phase_footer {
let declared_note = if declared_phase_regressions > 0 {
format!(" ({declared_phase_regressions} declared-gated, exit-affecting)")
} else {
" (render-only)".to_string()
};
format!(
"; {} per-phase regression(s){declared_note}, {} per-phase under-sampled (<2 runs)",
report.phase_regressions(),
report.phase_noisy(),
)
} else {
String::new()
};
let unevaluated_gates = report.assertion_coverage.len();
let gate_footer = if unevaluated_gates > 0 {
format!("; {unevaluated_gates} declared gate(s) not evaluated")
} else {
String::new()
};
println!(
"perf-delta --noise-adjust: {} paired scenario(s); overall {verdict}: \
{regressions} regressed, {improvements} improved, {stable} stable{info_clause}, \
{noisy} under-sampled (<2 runs){stable_note}{verdict_source}{phase_footer}{gate_footer}",
report.paired_scenarios,
);
}
if !phase_opts.phases_only {
let changed: Vec<&NoiseFinding> = report
.findings
.iter()
.filter(|f| f.kind != NoiseKind::Stable)
.collect();
if !changed.is_empty() && changed.iter().all(|f| f.kind == NoiseKind::Noisy) {
println!(
"perf-delta --noise-adjust: NOTE -- every changed metric had a side with <2 usable \
runs; raise --noise-adjust N (or investigate why per-side runs failed) for a \
trustworthy verdict"
);
}
}
Ok(noise_exit_code(&report, gate))
}
pub(crate) fn noise_exit_code(report: &NoiseReport, gate: &GateOptions) -> i32 {
let regressing: Vec<&str> = report
.findings
.iter()
.filter(|f| f.kind == NoiseKind::Regression)
.map(|f| f.metric.name)
.collect();
if gate_fails(®ressing, gate)
|| report.declared_regressions() > 0
|| report.declared_phase_regressions() > 0
{
1
} else {
0
}
}
pub(crate) fn overall_verdict(report: &NoiseReport, gate: &GateOptions) -> &'static str {
verdict_label(
noise_exit_code(report, gate) == 1,
report.improvements(),
gate.fail_threshold,
)
}
pub(crate) fn verdict_label(
regressed: bool,
improvements: usize,
fail_threshold: Option<usize>,
) -> &'static str {
let n = fail_threshold.unwrap_or(5);
let improved = n >= 1 && improvements >= n;
match (regressed, improved) {
(true, true) => "REGRESSED + IMPROVED",
(true, false) => "REGRESSED",
(false, true) => "IMPROVED",
(false, false) => "STABLE",
}
}
fn compose_noise_verdict_text(
base: &str,
high_spread: bool,
kind: NoiseKind,
gated_by_assertion: bool,
) -> String {
let mut annotations: Vec<&str> = Vec::new();
if high_spread && kind != NoiseKind::Noisy {
annotations.push("noisy spread");
}
if gated_by_assertion {
annotations.push("declared gate");
}
if annotations.is_empty() {
base.to_string()
} else {
format!("{base} ({})", annotations.join(", "))
}
}
pub(crate) fn format_noise_findings_table(
findings: &[NoiseFinding],
label_a: &str,
label_b: &str,
show_all: bool,
) -> String {
use comfy_table::{Cell, Color};
let visible: Vec<&NoiseFinding> = findings
.iter()
.filter(|f| show_all || !matches!(f.kind, NoiseKind::Stable | NoiseKind::Noisy))
.collect();
if visible.is_empty() {
if findings.is_empty() {
return String::new();
}
let noisy = findings
.iter()
.filter(|f| f.kind == NoiseKind::Noisy)
.count();
return format!(
"perf-delta --noise-adjust: {} metric(s) compared, none meaningfully changed \
({noisy} under-sampled); re-run with --all-metrics to see them\n",
findings.len(),
);
}
let mut table = crate::cli::new_table();
table.set_header(vec![
"TEST / METRIC".to_string(),
format!("{label_a} (A: mean [min-max] spread%)"),
format!("{label_b} (B: mean [min-max] spread%)"),
"VERDICT".to_string(),
]);
for f in visible {
let (base, color) = match f.kind {
NoiseKind::Regression => ("REGRESSION", Color::Red),
NoiseKind::Improvement => ("improvement", Color::Green),
NoiseKind::Noisy => ("NOISY (<2 runs)", Color::Yellow),
NoiseKind::Informational => ("informational", Color::Blue),
NoiseKind::Stable => ("stable", Color::Grey),
};
let verdict_text =
compose_noise_verdict_text(base, f.verdict.high_spread, f.kind, f.gated_by_assertion);
let v = &f.verdict;
table.add_row(vec![
Cell::new(format!("{} / {}", f.pairing_label, f.metric.name)),
Cell::new(format!(
"{:.1} [{:.1}-{:.1}] {:.2}%",
v.a.mean, v.a.min, v.a.max, v.a.spread_pct
)),
Cell::new(format!(
"{:.1} [{:.1}-{:.1}] {:.2}%",
v.b.mean, v.b.min, v.b.max, v.b.spread_pct
)),
Cell::new(verdict_text).fg(color),
]);
}
format!("{table}\n")
}
fn describe_declared_gate(a: &crate::test_support::PerfDeltaAssertionRecord) -> String {
let mut parts: Vec<String> = Vec::new();
if let Some(pct) = a.max_regression_pct {
parts.push(format!("max_regression_pct={pct}"));
}
if let Some(abs) = a.min_abs {
parts.push(format!("min_abs={abs}"));
}
if let Some(dir) = a.direction {
parts.push(format!("direction={dir:?}"));
}
if parts.is_empty() {
"registry defaults".to_string()
} else {
parts.join(", ")
}
}
pub(crate) fn format_noise_assertion_coverage_lines(
coverage: &[NoiseAssertionCoverage],
) -> Vec<String> {
use comfy_table::{Cell, Color};
let mut lines = Vec::new();
if coverage.is_empty() {
return lines;
}
let mut rows: Vec<&NoiseAssertionCoverage> = coverage.iter().collect();
rows.sort_by(|a, b| {
a.pairing_label
.cmp(&b.pairing_label)
.then_with(|| a.assertion.metric.cmp(&b.assertion.metric))
.then_with(|| a.assertion.phase.cmp(&b.assertion.phase))
});
lines.push(String::new());
lines.push(
"declared perf gate(s) NOT evaluated — the metric was absent from the compared \
data (workload no longer emits it, a one-sided/failed run, or a Rate with no \
samples), so the declared gate silently did not fire:"
.to_string(),
);
let mut table = crate::cli::new_table();
table.set_header(vec![
"TEST".to_string(),
"METRIC".to_string(),
"PHASE".to_string(),
"DECLARED GATE".to_string(),
]);
for c in rows {
let phase = match c.assertion.phase {
None => "aggregate".to_string(),
Some(k) => k.to_string(),
};
table.add_row(vec![
Cell::new(&c.pairing_label),
Cell::new(&c.assertion.metric).fg(Color::Yellow),
Cell::new(phase),
Cell::new(describe_declared_gate(&c.assertion)),
]);
}
lines.push(format!("{table}"));
lines
}
pub(crate) fn format_noise_phase_findings_lines(
phase_findings: &[NoisePhaseFinding],
phase_coverage: &[NoisePhaseCoverage],
phase_opts: &PhaseDisplayOptions,
label_a: &str,
label_b: &str,
show_all: bool,
) -> Vec<String> {
use comfy_table::{Cell, Color};
let mut lines = Vec::new();
if phase_opts.no_phases {
return lines;
}
let phase_filtered: Vec<&NoisePhaseFinding> = phase_findings
.iter()
.filter(|f| phase_opts.matches_phase(f.step_index))
.filter(|f| phase_opts.passes_noise_spread_threshold(&f.verdict))
.collect();
let mut findings: Vec<&NoisePhaseFinding> = phase_filtered
.iter()
.copied()
.filter(|f| show_all || !matches!(f.kind, NoiseKind::Stable | NoiseKind::Noisy))
.collect();
let mut coverage: Vec<&NoisePhaseCoverage> = phase_coverage
.iter()
.filter(|c| phase_opts.matches_phase(c.step_index))
.collect();
let had_findings = !findings.is_empty();
let suppressed_hint: Option<String> =
if findings.is_empty() && !phase_filtered.is_empty() && !show_all {
let noisy = phase_filtered
.iter()
.filter(|f| f.kind == NoiseKind::Noisy)
.count();
Some(format!(
"perf-delta --noise-adjust: {} per-phase metric(s) compared, none meaningfully \
changed ({noisy} under-sampled); re-run with --all-metrics to see them",
phase_filtered.len(),
))
} else {
None
};
if findings.is_empty() && coverage.is_empty() {
lines.extend(suppressed_hint);
return lines;
}
lines.push(String::new());
if had_findings {
lines.push("per-phase spread:".to_string());
findings.sort_by(|a, b| {
a.step_index
.cmp(&b.step_index)
.then_with(|| a.pairing_label.cmp(&b.pairing_label))
.then_with(|| a.metric.name.cmp(b.metric.name))
});
let mut table = crate::cli::new_table();
table.set_header(vec![
"PHASE".to_string(),
"TEST / METRIC".to_string(),
format!("{label_a} (A: mean [min-max] spread%)"),
format!("{label_b} (B: mean [min-max] spread%)"),
"VERDICT".to_string(),
]);
for f in findings {
let (base, color) = match f.kind {
NoiseKind::Regression => ("REGRESSION", Color::Red),
NoiseKind::Improvement => ("improvement", Color::Green),
NoiseKind::Noisy => ("NOISY (<2 runs)", Color::Yellow),
NoiseKind::Informational => ("informational", Color::Blue),
NoiseKind::Stable => ("stable", Color::Grey),
};
let verdict_text = compose_noise_verdict_text(
base,
f.verdict.high_spread,
f.kind,
f.gated_by_assertion,
);
let v = &f.verdict;
table.add_row(vec![
Cell::new(format!("{}: {}", f.step_index, f.label)),
Cell::new(format!("{} / {}", f.pairing_label, f.metric.name)),
Cell::new(format!(
"{:.1} [{:.1}-{:.1}] {:.2}%",
v.a.mean, v.a.min, v.a.max, v.a.spread_pct
)),
Cell::new(format!(
"{:.1} [{:.1}-{:.1}] {:.2}%",
v.b.mean, v.b.min, v.b.max, v.b.spread_pct
)),
Cell::new(verdict_text).fg(color),
]);
}
lines.push(table.to_string());
} else {
lines.extend(suppressed_hint);
}
if !coverage.is_empty() {
if had_findings {
lines.push(String::new());
}
lines.push("per-phase coverage asymmetry (one-sided metrics):".to_string());
coverage.sort_by(|a, b| {
a.step_index
.cmp(&b.step_index)
.then_with(|| a.present_side.as_str().cmp(b.present_side.as_str()))
.then_with(|| a.pairing_label.cmp(&b.pairing_label))
.then_with(|| a.metric.map(|m| m.name).cmp(&b.metric.map(|m| m.name)))
});
let mut table = crate::cli::new_table();
table.set_header(vec!["SIDE", "TEST", "PHASE", "METRIC", "VALUE"]);
for c in coverage {
let metric_cell = c.metric.map(|m| m.name).unwrap_or("—");
let value_cell = match c.value {
Some(v) => format!("{v:.2}"),
None => "—".to_string(),
};
table.add_row(vec![
Cell::new(c.present_side.as_str()),
Cell::new(c.pairing_label.as_str()),
Cell::new(format!("{}: {}", c.step_index, c.label)),
Cell::new(metric_cell),
Cell::new(value_cell),
]);
}
lines.push(table.to_string());
}
lines
}
fn print_scalar_findings_table(report: &CompareReport, label_a: &str, label_b: &str) {
use comfy_table::{Cell, Color};
let mut table = crate::cli::new_table();
table.set_header(vec!["TEST", "METRIC", label_a, label_b, "DELTA", "VERDICT"]);
for f in &report.findings {
let (verdict_text, verdict_color) = match f.kind {
FindingKind::Regression => ("REGRESSION", Color::Red),
FindingKind::Improvement => ("improvement", Color::Green),
FindingKind::Informational => ("informational", Color::Blue),
};
let label = f.pairing_key.0.join("/");
table.add_row(vec![
Cell::new(label),
Cell::new(f.metric.name),
Cell::new(format!("{:.2}", f.val_a)),
Cell::new(format!("{:.2}", f.val_b)),
Cell::new(format!("{:+.2}{}", f.delta, f.metric.display_unit)),
Cell::new(verdict_text).fg(verdict_color),
]);
}
println!("{table}");
}
fn print_summary_block(
report: &CompareReport,
avg_a: &Option<Vec<AveragedGroup>>,
avg_b: &Option<Vec<AveragedGroup>>,
label_a: &str,
label_b: &str,
) {
println!();
println!(
"summary: {} regressions, {} improvements, {} informational, {} unchanged",
report.regressions, report.improvements, report.informational, report.unchanged,
);
if report.excluded_pairs > 0 {
println!(
" {} pairing-key row pair(s) excluded from regression math because one \
or both sides was excluded (failed, inconclusive, skipped, or an inverted expected-failure run)",
report.excluded_pairs,
);
}
if let (Some(avg_a), Some(avg_b)) = (avg_a, avg_b) {
let block = format_per_group_pass_counts(avg_a, avg_b, label_a, label_b);
if !block.is_empty() {
print!("{block}");
}
}
if report.new_in_b > 0 {
println!(
" {} row(s) new in '{}' (no matching key in '{}')",
report.new_in_b, label_b, label_a,
);
}
if report.removed_from_a > 0 {
println!(
" {} row(s) removed from '{}' (no matching key in '{}')",
report.removed_from_a, label_a, label_b,
);
}
for line in format_coverage_diff_lines(report, label_a, label_b) {
println!("{line}");
}
}
pub(crate) fn format_coverage_diff_lines(
report: &CompareReport,
label_a: &str,
label_b: &str,
) -> Vec<String> {
if report.coverage_diffs.is_empty() {
return Vec::new();
}
let mut lines = vec![format!(
" {} metric(s) present on only one side (coverage difference, \
not a regression):",
report.coverage_diffs.len(),
)];
for cd in &report.coverage_diffs {
let (present, absent) = match cd.present_side {
ComparePartition::A => (label_a, label_b),
ComparePartition::B => (label_b, label_a),
};
lines.push(format!(
" {} / {} = {:.2} in '{}', absent in '{}'",
cd.pairing_key.0.join("/"),
cd.metric.name,
cd.value,
present,
absent,
));
}
lines
}
fn print_host_context_delta(
pool: &[crate::test_support::SidecarResult],
rows: &[GauntletRow],
filter_a: &RowFilter,
filter_b: &RowFilter,
label_a: &str,
label_b: &str,
) {
let sidecars_a: Vec<&crate::test_support::SidecarResult> = pool
.iter()
.zip(rows.iter())
.filter(|(_, r)| filter_a.matches(r))
.map(|(s, _)| s)
.collect();
let sidecars_b: Vec<&crate::test_support::SidecarResult> = pool
.iter()
.zip(rows.iter())
.filter(|(_, r)| filter_b.matches(r))
.map(|(s, _)| s)
.collect();
let host_a = sidecars_a.iter().find_map(|s| s.host.as_ref());
let host_b = sidecars_b.iter().find_map(|s| s.host.as_ref());
print!("{}", format_host_delta(host_a, host_b, label_a, label_b));
}
pub(crate) fn format_average_header(
pre_agg_a: usize,
pre_agg_b: usize,
a: &str,
b: &str,
) -> String {
format!("averaged across {pre_agg_a} runs ({a}) and {pre_agg_b} runs ({b})")
}
pub(crate) fn format_per_group_pass_counts(
avg_a: &[AveragedGroup],
avg_b: &[AveragedGroup],
a: &str,
b: &str,
) -> String {
type SummaryKey<'a> = (&'a str, &'a str, &'a str);
type SummaryValue<'a> = (Option<&'a AveragedGroup>, Option<&'a AveragedGroup>);
let mut keys: BTreeMap<SummaryKey<'_>, SummaryValue<'_>> = BTreeMap::new();
for ar in avg_a {
let k = (
ar.row.scenario.as_str(),
ar.row.topology.as_str(),
ar.row.work_type.as_str(),
);
keys.entry(k).or_insert((None, None)).0 = Some(ar);
}
for br in avg_b {
let k = (
br.row.scenario.as_str(),
br.row.topology.as_str(),
br.row.work_type.as_str(),
);
keys.entry(k).or_insert((None, None)).1 = Some(br);
}
if keys.is_empty() {
return String::new();
}
let mut out = String::new();
out.push('\n');
out.push_str(
"per-group pass counts (passes/total + skip/inconc/fail breakdown when non-zero):\n",
);
for ((scn, topo, wt), (ka, kb)) in keys.into_iter() {
let fmt_side = |r: Option<&AveragedGroup>| -> String {
let Some(x) = r else {
return "-".to_string();
};
let mut s = format!("{}/{}", x.passes_observed, x.total_observed);
let mut extras: Vec<String> = Vec::with_capacity(3);
if x.skips_observed > 0 {
extras.push(format!("{} skip", x.skips_observed));
}
if x.inconclusives_observed > 0 {
extras.push(format!("{} inc", x.inconclusives_observed));
}
if x.failures_observed > 0 {
extras.push(format!("{} fail", x.failures_observed));
}
if !extras.is_empty() {
s.push_str(&format!(" ({})", extras.join(", ")));
}
s
};
out.push_str(&format!(
" {scn}/{topo}/{wt}: {a}={pa} {b}={pb}\n",
pa = fmt_side(ka),
pb = fmt_side(kb),
));
}
out
}
pub(crate) fn format_host_delta(
host_a: Option<&crate::host_context::HostContext>,
host_b: Option<&crate::host_context::HostContext>,
a: &str,
b: &str,
) -> String {
match (host_a, host_b) {
(Some(ha), Some(hb)) => {
let delta = ha.diff(hb);
if delta.is_empty() {
match (ha.arch.as_deref(), hb.arch.as_deref()) {
(Some(arch_a), Some(arch_b)) if arch_a == arch_b => {
format!("\nhost: identical between '{a}' and '{b}' (arch: {arch_a})\n",)
}
_ => format!("\nhost: identical between '{a}' and '{b}'\n"),
}
} else {
format!("\nhost delta ('{a}' → '{b}'):\n{delta}")
}
}
(Some(_), None) => {
format!("\nhost: captured in '{a}' only, delta unavailable\n")
}
(None, Some(_)) => {
format!("\nhost: captured in '{b}' only, delta unavailable\n")
}
(None, None) => String::new(),
}
}