use super::*;
fn cmp_row(scenario: &str, topo: &str, passed: bool, spread: f64, iters: u64) -> GauntletRow {
let mut r = make_row(scenario, topo, passed, spread);
r.gap_ms = 0;
r.migrations = 0;
r.imbalance_ratio = 0.0;
r.max_dsq_depth = 0;
r.total_iterations = iters;
r
}
#[test]
fn compare_rows_dual_gate_both_must_trigger() {
let rows_a = vec![cmp_row("test_a", "tiny-1llc", true, 10.0, 0)];
let rows_b = vec![cmp_row("test_a", "tiny-1llc", true, 12.0, 0)];
let res = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(res.regressions, 0, "abs gate must block 2.0 < 5.0");
assert_eq!(res.improvements, 0);
assert_eq!(
res.unchanged, 1,
"worst_spread should be classified unchanged"
);
assert!(res.findings.is_empty());
let rows_b2 = vec![cmp_row("test_a", "tiny-1llc", true, 14.0, 0)];
let res2 = compare_rows_by(
&rows_a,
&rows_b2,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(
res2.regressions, 0,
"rel-only is insufficient: abs gate must also fire"
);
assert_eq!(res2.unchanged, 1);
}
#[test]
fn compare_rows_subinteger_stuck_count_difference_is_unchanged() {
let mut a = make_row("test_a", "tiny-1llc", true, 10.0);
a.stuck_count = 1.4;
let mut b = make_row("test_a", "tiny-1llc", true, 10.0);
b.stuck_count = 1.6;
let res = compare_rows_by(
&[a],
&[b],
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(
res.regressions, 0,
"a 0.2 sub-integer stuck_count delta must NOT be a regression",
);
assert_eq!(res.improvements, 0);
assert!(
res.findings.iter().all(|f| f.metric.name != "stuck_count"),
"stuck_count must not be a finding for a sub-abs delta; got {:?}",
res.findings
.iter()
.map(|f| f.metric.name)
.collect::<Vec<_>>(),
);
}
#[test]
fn compare_rows_genuine_stuck_count_regression_is_flagged() {
let mut a = make_row("test_a", "tiny-1llc", true, 10.0);
a.stuck_count = 1.0;
let mut b = make_row("test_a", "tiny-1llc", true, 10.0);
b.stuck_count = 2.5;
let res = compare_rows_by(
&[a],
&[b],
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(
res.regressions, 1,
"a 1.5 stuck_count delta clears both gates and must be a regression",
);
assert!(
res.findings
.iter()
.any(|f| f.metric.name == "stuck_count" && f.is_regression),
"stuck_count must be the flagged regression",
);
}
#[test]
fn compare_rows_synthetic_regression_and_improvement() {
let rows_a = vec![cmp_row("test1", "tiny-1llc", true, 10.0, 1000)];
let rows_b = vec![cmp_row("test1", "tiny-1llc", true, 30.0, 500)];
let res = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::uniform(10.0),
);
assert_eq!(
res.regressions, 2,
"spread up + iterations down both regress"
);
assert_eq!(res.improvements, 0);
assert_eq!(res.excluded_pairs, 0);
let metrics: Vec<&str> = res.findings.iter().map(|d| d.metric.name).collect();
assert!(metrics.contains(&"worst_spread"));
assert!(metrics.contains(&"total_iterations"));
for d in &res.findings {
assert!(d.is_regression, "all reported deltas should be regressions");
assert_eq!(d.scenario, "test1");
assert_eq!(d.topology, "tiny-1llc");
}
let res_imp = compare_rows_by(
&rows_b,
&rows_a,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::uniform(10.0),
);
assert_eq!(res_imp.regressions, 0);
assert_eq!(res_imp.improvements, 2);
for d in &res_imp.findings {
assert!(!d.is_regression);
}
}
#[test]
fn compare_rows_suppresses_rate_components_not_the_rate() {
let mut a = cmp_row("t", "tiny-1llc", true, 0.0, 1000);
a.ext_metrics
.insert("total_iterations_pooled".to_string(), 1000.0);
a.ext_metrics
.insert("iterations_per_cpu_sec".to_string(), 500.0);
let mut b = cmp_row("t", "tiny-1llc", true, 0.0, 1000);
b.ext_metrics
.insert("total_iterations_pooled".to_string(), 2000.0);
b.ext_metrics
.insert("iterations_per_cpu_sec".to_string(), 1000.0);
let res = compare_rows_by(
&[a],
&[b],
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
let names: Vec<&str> = res.findings.iter().map(|d| d.metric.name).collect();
assert!(
!names.contains(&"total_iterations_pooled"),
"the Rate component must be suppressed from compare findings; got {names:?}",
);
assert!(
names.contains(&"iterations_per_cpu_sec"),
"the user-facing pooled rate must still emit a finding; got {names:?}",
);
}
#[test]
fn compare_rows_higher_is_worse_inversion() {
let rows_a = vec![cmp_row("t", "tiny-1llc", true, 0.0, 1000)];
let rows_b = vec![cmp_row("t", "tiny-1llc", true, 0.0, 500)];
let res = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
let iters_delta = res
.findings
.iter()
.find(|d| d.metric.name == "total_iterations")
.expect("total_iterations should produce a delta");
assert!(
iters_delta.is_regression,
"iterations decrease is a regression"
);
assert_eq!(iters_delta.delta, -500.0);
assert_eq!(res.regressions, 1);
assert_eq!(res.improvements, 0);
let rows_a2 = vec![cmp_row("t", "tiny-1llc", true, 10.0, 0)];
let rows_b2 = vec![cmp_row("t", "tiny-1llc", true, 30.0, 0)];
let res_up = compare_rows_by(
&rows_a2,
&rows_b2,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
let spread_up = res_up
.findings
.iter()
.find(|d| d.metric.name == "worst_spread")
.expect("worst_spread should produce a delta");
assert!(spread_up.is_regression, "spread increase is a regression");
assert_eq!(spread_up.delta, 20.0);
let res_down = compare_rows_by(
&rows_b2,
&rows_a2,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
let spread_down = res_down
.findings
.iter()
.find(|d| d.metric.name == "worst_spread")
.expect("worst_spread should produce a delta");
assert!(
!spread_down.is_regression,
"spread decrease is an improvement"
);
assert_eq!(spread_down.delta, -20.0);
}
#[test]
fn compare_rows_skipped_side_drops_pair_into_excluded_pairs() {
let mut row_a = cmp_row("t", "tiny-1llc", true, 10.0, 100);
let mut row_b = cmp_row("t", "tiny-1llc", true, 10.0, 100);
row_a.skipped = true; let res = compare_rows_by(
&[row_a.clone()],
&[row_b.clone()],
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(res.regressions, 0);
assert_eq!(res.improvements, 0);
assert_eq!(
res.excluded_pairs, 1,
"skipped side must count as excluded_pairs, not produce deltas"
);
row_a.skipped = false;
row_b.skipped = true;
let res = compare_rows_by(
&[row_a],
&[row_b],
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(res.regressions, 0);
assert_eq!(res.improvements, 0);
assert_eq!(res.excluded_pairs, 1);
}
#[test]
fn compare_rows_skips_failed_scenarios() {
let rows_a = vec![
cmp_row("test_ok", "tiny-1llc", true, 10.0, 1000),
cmp_row("test_failed_b", "tiny-1llc", true, 10.0, 1000),
cmp_row("test_failed_a", "tiny-1llc", false, 10.0, 1000),
];
let rows_b = vec![
cmp_row("test_ok", "tiny-1llc", true, 30.0, 500),
cmp_row("test_failed_b", "tiny-1llc", false, 30.0, 500),
cmp_row("test_failed_a", "tiny-1llc", true, 30.0, 500),
];
let res = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::uniform(10.0),
);
assert_eq!(
res.excluded_pairs, 2,
"test_failed_a and test_failed_b skip"
);
assert_eq!(res.regressions, 2);
assert_eq!(res.improvements, 0);
for d in &res.findings {
assert_eq!(d.scenario, "test_ok");
}
}
#[test]
fn compare_rows_filter_substring() {
let rows_a = vec![
cmp_row("alpha", "tiny-1llc", true, 10.0, 0),
cmp_row("beta", "tiny-1llc", true, 10.0, 0),
];
let rows_b = vec![
cmp_row("alpha", "tiny-1llc", true, 30.0, 0),
cmp_row("beta", "tiny-1llc", true, 30.0, 0),
];
let res = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
Some("alpha"),
&ComparisonPolicy::default(),
);
assert_eq!(res.regressions, 1, "only alpha row should compare");
assert_eq!(res.findings.len(), 1);
assert_eq!(res.findings[0].scenario, "alpha");
assert_eq!(res.findings[0].work_type, "SpinWait");
let res_topo = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
Some("tiny"),
&ComparisonPolicy::default(),
);
assert_eq!(res_topo.regressions, 2, "both rows match 'tiny' topology");
assert_eq!(res_topo.findings.len(), 2);
let res_none = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
Some("nomatch"),
&ComparisonPolicy::default(),
);
assert_eq!(res_none.regressions, 0);
assert_eq!(res_none.improvements, 0);
assert_eq!(res_none.unchanged, 0);
assert_eq!(res_none.excluded_pairs, 0);
}
#[test]
fn compare_rows_threshold_override() {
let rows_a = vec![cmp_row("t", "tiny-1llc", true, 100.0, 0)];
let rows_b = vec![cmp_row("t", "tiny-1llc", true, 106.0, 0)];
let res_default = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
let spread_default = res_default
.findings
.iter()
.find(|d| d.metric.name == "worst_spread");
assert!(
spread_default.is_none(),
"default rel 0.25 must classify 6% change as unchanged"
);
let res_override = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::uniform(5.0),
);
let spread_override = res_override
.findings
.iter()
.find(|d| d.metric.name == "worst_spread")
.expect("override 5% must surface 6% spread change");
assert!(spread_override.is_regression);
assert_eq!(spread_override.delta, 6.0);
let rows_a_small = vec![cmp_row("t", "tiny-1llc", true, 1.0, 0)];
let rows_b_small = vec![cmp_row("t", "tiny-1llc", true, 1.5, 0)];
let res_small = compare_rows_by(
&rows_a_small,
&rows_b_small,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::uniform(1.0),
);
assert!(
!res_small
.findings
.iter()
.any(|d| d.metric.name == "worst_spread"),
"abs gate must still block tiny absolute moves"
);
}
#[test]
fn comparison_policy_rel_threshold_resolution_priority() {
let empty = ComparisonPolicy::default();
assert_eq!(
empty.rel_threshold("worst_spread", 0.25),
0.25,
"empty policy must fall through to the registry default_rel",
);
let uniform = ComparisonPolicy::uniform(10.0);
assert_eq!(
uniform.rel_threshold("worst_spread", 0.25),
0.10,
"uniform(10.0) must override the registry default_rel \
with 10.0 / 100.0 = 0.10",
);
let mut per_metric = ComparisonPolicy::uniform(10.0);
per_metric
.per_metric_percent
.insert("worst_spread".to_string(), 5.0);
assert_eq!(
per_metric.rel_threshold("worst_spread", 0.25),
0.05,
"per-metric override (5.0) must win over default_percent \
(10.0) and the registry default (0.25)",
);
assert_eq!(
per_metric.rel_threshold("worst_gap_ms", 0.25),
0.10,
"metrics not in the per-metric map must still see the \
default_percent (10.0 → 0.10), not the registry default",
);
}
#[test]
fn wake_latency_tail_ratio_compares_via_ext_metrics() {
let metric = metric_def("worst_wake_latency_tail_ratio")
.expect("worst_wake_latency_tail_ratio must be registered in METRICS");
let key = "worst_wake_latency_tail_ratio";
let low_a = make_row("tail_low", "tiny-1llc", true, 0.0);
let low_b = make_row("tail_low", "tiny-1llc", true, 0.0);
assert!(
metric.read(&low_a).is_none(),
"absent ext key must read as None (accessor is |_| None, no ext entry)",
);
let below = compare_rows_by(
std::slice::from_ref(&low_a),
std::slice::from_ref(&low_b),
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(
below.regressions, 0,
"absent tail-ratio key (identical rows) must surface no regression",
);
assert!(
below.findings.is_empty(),
"absent tail-ratio key (identical rows) must emit no findings",
);
let mut hi_a = make_row("tail_hi", "tiny-1llc", true, 0.0);
let mut hi_b = make_row("tail_hi", "tiny-1llc", true, 0.0);
hi_a.ext_metrics.insert(key.to_string(), 2.0);
hi_b.ext_metrics.insert(key.to_string(), 20.0);
assert_eq!(
metric.read(&hi_a),
Some(2.0),
"present ext key must read via the ext fallback",
);
let above = compare_rows_by(
std::slice::from_ref(&hi_a),
std::slice::from_ref(&hi_b),
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(
above.regressions, 1,
"a present-key 10x tail blow-up must surface as a regression; \
threshold wiring has a gap otherwise",
);
}
#[test]
fn compare_rows_handles_none_from_absent_ext_key_as_zero() {
let metric =
metric_def("worst_wake_latency_tail_ratio").expect("tail ratio metric must be registered");
let row_a = make_row("none_branch", "tiny-1llc", true, 0.0);
let row_b = make_row("none_branch", "tiny-1llc", true, 0.0);
assert!(
metric.read(&row_a).is_none(),
"absent ext key must read None on A — otherwise this test is not \
exercising the None branch of compare_rows",
);
assert!(
metric.read(&row_b).is_none(),
"absent ext key must read None on B",
);
let report = compare_rows_by(
std::slice::from_ref(&row_a),
std::slice::from_ref(&row_b),
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(
report.regressions, 0,
"None accessor result must land as unchanged, not a regression",
);
assert_eq!(
report.improvements, 0,
"None accessor result must land as unchanged, not an improvement",
);
assert!(
report.findings.is_empty(),
"no findings must be emitted when the accessor returns None; \
got: {:?}",
report.findings,
);
}
#[test]
fn comparison_policy_load_json_round_trip() {
let mut original = ComparisonPolicy::uniform(10.0);
original
.per_metric_percent
.insert("worst_spread".to_string(), 5.0);
original
.per_metric_percent
.insert("worst_p99_wake_latency_us".to_string(), 20.0);
let json = serde_json::to_string(&original).expect("serialize policy");
let tmp = tempfile::NamedTempFile::new().expect("create tempfile");
std::fs::write(tmp.path(), json).expect("write policy file");
let loaded = ComparisonPolicy::load_json(tmp.path()).expect("load policy");
assert_eq!(
loaded.default_percent,
Some(10.0),
"default_percent must round-trip",
);
assert_eq!(
loaded.per_metric_percent.get("worst_spread"),
Some(&5.0),
"per-metric worst_spread override must round-trip",
);
assert_eq!(
loaded.per_metric_percent.get("worst_p99_wake_latency_us"),
Some(&20.0),
"per-metric worst_p99 override must round-trip",
);
for metric_name in ["worst_spread", "worst_p99_wake_latency_us", "worst_gap_ms"] {
assert_eq!(
loaded.rel_threshold(metric_name, 0.25),
original.rel_threshold(metric_name, 0.25),
"load_json round-trip must preserve threshold \
resolution for {metric_name}",
);
}
}
#[test]
fn comparison_policy_load_json_nonexistent_path_surfaces_path() {
let path = std::path::Path::new("/nonexistent/ktstr/policy-DOES-NOT-EXIST.json");
let err = ComparisonPolicy::load_json(path).expect_err("nonexistent path must fail");
let rendered = format!("{err:#}");
assert!(
rendered.contains(&path.display().to_string()),
"error must name the missing path so a user can see \
which file was expected; got: {rendered}",
);
assert!(
rendered.to_ascii_lowercase().contains("read")
|| rendered.to_ascii_lowercase().contains("no such"),
"error must describe the read failure (either the \
`with_context` \"read comparison policy from ...\" \
prefix or std's underlying \"No such file...\" \
reason); got: {rendered}",
);
}
#[test]
fn comparison_policy_load_json_malformed_json_surfaces_path_and_parse_context() {
let tmp = tempfile::NamedTempFile::new().expect("tempfile");
std::fs::write(tmp.path(), "this is not json at all {{{").expect("write");
let err = ComparisonPolicy::load_json(tmp.path()).expect_err("malformed JSON must fail");
let rendered = format!("{err:#}");
assert!(
rendered.contains(&tmp.path().display().to_string()),
"malformed-JSON error must name the path; got: {rendered}",
);
assert!(
rendered.to_ascii_lowercase().contains("parse")
|| rendered.to_ascii_lowercase().contains("expected"),
"malformed-JSON error must include a parse-context \
hint (either the `with_context` \"parse comparison \
policy from ...\" prefix, or serde_json's \"expected \
...\" reason); got: {rendered}",
);
}
#[test]
fn comparison_policy_load_json_rejects_unknown_fields() {
let tmp = tempfile::NamedTempFile::new().expect("tempfile");
std::fs::write(tmp.path(), r#"{"default_percentage": 10.0}"#).expect("write");
let err = ComparisonPolicy::load_json(tmp.path()).expect_err("unknown field must fail");
let rendered = format!("{err:#}");
assert!(
rendered.contains("default_percentage")
|| rendered.to_ascii_lowercase().contains("unknown"),
"unknown-field error must name the typo so a user \
can fix the policy file; got: {rendered}",
);
}
#[test]
fn comparison_policy_validate_rejects_negative_default_percent() {
let policy = ComparisonPolicy::uniform(-10.0);
let err = policy
.validate()
.expect_err("negative default_percent must fail validation");
let rendered = format!("{err:#}");
assert!(
rendered.contains("default_percent"),
"validation error must name the field; got: {rendered}",
);
assert!(
rendered.contains("-10"),
"validation error must echo the rejected value; got: {rendered}",
);
}
#[test]
fn comparison_policy_validate_rejects_unknown_per_metric_keys() {
let mut policy = ComparisonPolicy::default();
policy
.per_metric_percent
.insert("wrost_spread".to_string(), 5.0); let err = policy
.validate()
.expect_err("unknown per-metric key must fail validation");
let rendered = format!("{err:#}");
assert!(
rendered.contains("wrost_spread"),
"validation error must echo the unknown key so a user \
can see the typo; got: {rendered}",
);
assert!(
rendered.contains("worst_spread"),
"validation error should include the registered \
metric list so users can find the right spelling; \
got: {rendered}",
);
}
#[test]
fn comparison_policy_validate_rejects_negative_per_metric_value() {
let mut policy = ComparisonPolicy::default();
policy
.per_metric_percent
.insert("worst_spread".to_string(), -5.0);
let err = policy
.validate()
.expect_err("negative per-metric percent must fail");
let rendered = format!("{err:#}");
assert!(
rendered.contains("worst_spread") && rendered.contains("-5"),
"validation error must name both the key and the \
rejected value; got: {rendered}",
);
}
#[test]
fn comparison_policy_load_json_accepts_partial_fields() {
let tmp = tempfile::NamedTempFile::new().expect("create tempfile");
std::fs::write(tmp.path(), "{}").expect("write empty policy");
let loaded = ComparisonPolicy::load_json(tmp.path()).expect("load empty policy");
assert_eq!(loaded.default_percent, None);
assert!(loaded.per_metric_percent.is_empty());
std::fs::write(tmp.path(), r#"{"default_percent": 7.5}"#).expect("write partial policy");
let loaded = ComparisonPolicy::load_json(tmp.path()).expect("load partial policy");
assert_eq!(loaded.default_percent, Some(7.5));
assert!(loaded.per_metric_percent.is_empty());
std::fs::write(
tmp.path(),
r#"{"per_metric_percent": {"worst_spread": 3.0}}"#,
)
.expect("write per-metric-only policy");
let loaded = ComparisonPolicy::load_json(tmp.path()).expect("load per-metric-only policy");
assert_eq!(loaded.default_percent, None);
assert_eq!(loaded.per_metric_percent.get("worst_spread"), Some(&3.0),);
}
#[test]
fn comparison_policy_from_cli_flags_resolves_each_branch() {
let p = ComparisonPolicy::from_cli_flags(Some(15.0), None).expect("threshold resolves");
assert_eq!(p.default_percent, Some(15.0));
assert!(p.per_metric_percent.is_empty());
assert!(
ComparisonPolicy::from_cli_flags(Some(-1.0), None).is_err(),
"negative --threshold must be rejected before the dual-gate math",
);
let tmp = tempfile::NamedTempFile::new().expect("tempfile");
std::fs::write(tmp.path(), r#"{"default_percent": 8.0}"#).expect("write policy");
let p = ComparisonPolicy::from_cli_flags(None, Some(tmp.path())).expect("policy file resolves");
assert_eq!(p.default_percent, Some(8.0));
let p = ComparisonPolicy::from_cli_flags(None, None).expect("default resolves");
assert_eq!(p.default_percent, None);
assert!(
ComparisonPolicy::from_cli_flags(Some(10.0), Some(tmp.path())).is_err(),
"--threshold + --policy together must error",
);
}
#[test]
fn compare_rows_per_metric_policy_resolves_each_metric_independently() {
let mut row_a = cmp_row("t", "tiny-1llc", true, 100.0, 0);
row_a
.ext_metrics
.insert("worst_median_wake_latency_us".to_string(), 100.0);
let mut row_b = cmp_row("t", "tiny-1llc", true, 106.0, 0);
row_b
.ext_metrics
.insert("worst_median_wake_latency_us".to_string(), 110.0);
let mut policy = ComparisonPolicy::uniform(20.0);
policy
.per_metric_percent
.insert("worst_spread".to_string(), 5.0);
let res = compare_rows_by(&[row_a], &[row_b], LEGACY_PAIRING_DIMS, None, &policy);
let spread_finding = res
.findings
.iter()
.find(|f| f.metric.name == "worst_spread");
assert!(
spread_finding.is_some(),
"worst_spread per-metric override (5%) must fire on 6% \
delta; got findings: {:?}",
res.findings
.iter()
.map(|f| f.metric.name)
.collect::<Vec<_>>(),
);
let spread_finding = spread_finding.unwrap();
assert!(spread_finding.is_regression, "6% > 5% → regression");
let wake_finding = res
.findings
.iter()
.find(|f| f.metric.name == "worst_median_wake_latency_us");
assert!(
wake_finding.is_none(),
"worst_median_wake_latency_us 10% delta must fall \
under default_percent 20% and be unchanged. The \
regression would indicate `compare_rows` ignored \
default_percent for non-per-metric entries; got \
finding: {wake_finding:?}",
);
assert_eq!(
res.regressions, 1,
"exactly one regression expected — the per-metric \
spread override should win on spread, and the \
default_percent should suppress wake latency. Got: \
regressions={}, improvements={}, unchanged={}",
res.regressions, res.improvements, res.unchanged,
);
}
#[test]
fn compare_rows_duplicate_key_first_match_wins() {
let rows_a = vec![
cmp_row("t", "tiny-1llc", true, 10.0, 0),
cmp_row("t", "tiny-1llc", true, 29.0, 0),
];
let rows_b = vec![cmp_row("t", "tiny-1llc", true, 30.0, 0)];
let res = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(res.regressions, 1, "first match (spread=10) must win");
let spread = res
.findings
.iter()
.find(|d| d.metric.name == "worst_spread")
.expect("worst_spread regression should fire");
assert_eq!(
spread.val_a, 10.0,
"val_a must come from the first matching row"
);
assert_eq!(spread.delta, 20.0);
}
#[test]
fn compare_rows_filter_excludes_failed_from_skip_count() {
let rows_a = vec![
cmp_row("alpha", "tiny-1llc", true, 10.0, 0),
cmp_row("beta", "tiny-1llc", false, 10.0, 0),
];
let rows_b = vec![
cmp_row("alpha", "tiny-1llc", true, 30.0, 0),
cmp_row("beta", "tiny-1llc", true, 30.0, 0),
];
let unfiltered = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(unfiltered.excluded_pairs, 1);
assert_eq!(unfiltered.regressions, 1, "alpha still regresses");
let filtered = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
Some("alpha"),
&ComparisonPolicy::default(),
);
assert_eq!(filtered.excluded_pairs, 0);
assert_eq!(filtered.regressions, 1);
assert_eq!(filtered.findings.len(), 1);
assert_eq!(filtered.findings[0].scenario, "alpha");
}
#[test]
fn compare_rows_filter_substring_matches_scheduler() {
let mut a1 = cmp_row("test1", "tiny-1llc", true, 10.0, 0);
a1.scheduler = "scx_alpha".into();
let mut a2 = cmp_row("test2", "tiny-1llc", true, 10.0, 0);
a2.scheduler = "scx_beta".into();
let mut b1 = cmp_row("test1", "tiny-1llc", true, 30.0, 0);
b1.scheduler = "scx_alpha".into();
let mut b2 = cmp_row("test2", "tiny-1llc", true, 30.0, 0);
b2.scheduler = "scx_beta".into();
let res = compare_rows_by(
&[a1, a2],
&[b1, b2],
LEGACY_PAIRING_DIMS,
Some("scx_alpha"),
&ComparisonPolicy::default(),
);
assert_eq!(res.regressions, 1, "only the scx_alpha row compares");
assert_eq!(res.findings.len(), 1);
assert_eq!(res.findings[0].scenario, "test1");
assert_eq!(res.new_in_b, 0);
assert_eq!(res.removed_from_a, 0);
}
#[test]
fn compare_rows_tracks_new_and_removed_rows() {
let rows_a = vec![
cmp_row("alpha", "tiny-1llc", true, 10.0, 0),
cmp_row("gamma", "tiny-1llc", true, 10.0, 0),
];
let rows_b = vec![
cmp_row("alpha", "tiny-1llc", true, 30.0, 0),
cmp_row("beta", "tiny-1llc", true, 30.0, 0),
];
let res = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
None,
&ComparisonPolicy::default(),
);
assert_eq!(res.regressions, 1, "alpha regresses on worst_spread");
assert_eq!(res.new_in_b, 1, "beta is new on B side");
assert_eq!(res.removed_from_a, 1, "gamma is removed on B side");
assert_eq!(res.excluded_pairs, 0);
}
#[test]
fn compare_rows_filter_applies_to_new_and_removed_counters() {
let rows_a = vec![
cmp_row("alpha", "tiny-1llc", true, 10.0, 0),
cmp_row("gamma", "tiny-1llc", true, 10.0, 0),
];
let rows_b = vec![
cmp_row("alpha", "tiny-1llc", true, 30.0, 0),
cmp_row("beta", "tiny-1llc", true, 30.0, 0),
];
let res = compare_rows_by(
&rows_a,
&rows_b,
LEGACY_PAIRING_DIMS,
Some("alpha"),
&ComparisonPolicy::default(),
);
assert_eq!(res.regressions, 1);
assert_eq!(res.new_in_b, 0, "beta is filtered out, not new");
assert_eq!(res.removed_from_a, 0, "gamma is filtered out, not removed");
}
fn host_ctx(release: &str, kernel_cmdline: Option<&str>) -> crate::host_context::HostContext {
crate::host_context::HostContext {
kernel_name: Some("Linux".to_string()),
kernel_release: Some(release.to_string()),
kernel_cmdline: kernel_cmdline.map(str::to_string),
..Default::default()
}
}
#[test]
fn format_host_delta_both_present_identical() {
let ctx = host_ctx("6.14.0", Some("preempt=lazy"));
let out = format_host_delta(Some(&ctx), Some(&ctx), "a-run", "b-run");
assert_eq!(out, "\nhost: identical between 'a-run' and 'b-run'\n");
}
#[test]
fn format_host_delta_both_present_differ() {
let ha = host_ctx("6.14.0", Some("preempt=lazy"));
let hb = host_ctx("6.15.1", Some("preempt=lazy"));
let out = format_host_delta(Some(&ha), Some(&hb), "a", "b");
assert!(
out.starts_with("\nhost delta ('a' → 'b'):\n"),
"got: {out:?}"
);
let body = &out["\nhost delta ('a' → 'b'):\n".len()..];
assert!(
!body.is_empty(),
"differing contexts must produce a diff body"
);
assert!(
out.ends_with('\n'),
"differ arm must end with a newline for contiguous-section output: {out:?}",
);
}
#[test]
fn format_host_delta_left_only() {
let ctx = host_ctx("6.14.0", Some("preempt=lazy"));
let out = format_host_delta(Some(&ctx), None, "a-run", "b-run");
assert_eq!(out, "\nhost: captured in 'a-run' only, delta unavailable\n");
}
#[test]
fn format_host_delta_right_only() {
let ctx = host_ctx("6.14.0", Some("preempt=lazy"));
let out = format_host_delta(None, Some(&ctx), "a-run", "b-run");
assert_eq!(out, "\nhost: captured in 'b-run' only, delta unavailable\n");
}
#[test]
fn format_host_delta_both_absent_emits_nothing() {
assert_eq!(format_host_delta(None, None, "a", "b"), "");
}
#[test]
fn format_host_delta_identical_with_arch_surfaces_arch() {
let ctx = crate::host_context::HostContext {
kernel_name: Some("Linux".to_string()),
arch: Some("x86_64".to_string()),
..Default::default()
};
let out = format_host_delta(Some(&ctx), Some(&ctx), "a", "b");
assert_eq!(
out,
"\nhost: identical between 'a' and 'b' (arch: x86_64)\n",
);
}
#[test]
fn format_host_delta_identical_partial_arch_falls_back() {
let ha = crate::host_context::HostContext {
kernel_name: Some("Linux".to_string()),
arch: Some("x86_64".to_string()),
..Default::default()
};
let hb = crate::host_context::HostContext {
kernel_name: Some("Linux".to_string()),
arch: None,
..Default::default()
};
let out = format_host_delta(Some(&ha), Some(&hb), "a", "b");
assert!(
out.starts_with("\nhost delta ('a' → 'b'):\n"),
"asymmetric arch must route through differ arm, not \
identical arm: {out:?}",
);
assert!(
out.contains("arch:"),
"differ arm must surface the arch row: {out:?}",
);
}
#[test]
fn format_host_delta_identical_both_arch_none_falls_back() {
let ctx = crate::host_context::HostContext {
kernel_name: Some("Linux".to_string()),
arch: None,
..Default::default()
};
let out = format_host_delta(Some(&ctx), Some(&ctx), "a", "b");
assert_eq!(out, "\nhost: identical between 'a' and 'b'\n");
}
#[test]
fn gauntlet_row_empty_ext_metrics_omits_key() {
let row = make_row("scn", "topo", true, 0.0);
assert!(row.ext_metrics.is_empty());
let json = serde_json::to_string(&row).unwrap();
assert!(
!json.contains("\"ext_metrics\""),
"empty ext_metrics must be omitted from JSON: {json}"
);
}
#[test]
fn gauntlet_row_non_empty_ext_metrics_emits_payload() {
let mut row = make_row("scn", "topo", true, 0.0);
row.ext_metrics.insert("custom_metric".into(), 42.5);
let json = serde_json::to_string(&row).unwrap();
assert!(
json.contains("\"custom_metric\":42.5"),
"ext_metrics payload missing: {json}"
);
}
#[test]
fn gauntlet_row_round_trip_empty_ext_metrics() {
let row = make_row("scn", "topo", true, 1.5);
let json = serde_json::to_string(&row).unwrap();
let back: GauntletRow = serde_json::from_str(&json).unwrap();
assert_eq!(back, row);
assert!(back.ext_metrics.is_empty());
}
#[test]
fn gauntlet_row_round_trip_non_empty_ext_metrics() {
let mut row = make_row("scn", "topo", false, std::f64::consts::PI);
row.ext_metrics.insert("m1".into(), 1.0);
row.ext_metrics.insert("m2".into(), 2.5);
let json = serde_json::to_string(&row).unwrap();
let back: GauntletRow = serde_json::from_str(&json).unwrap();
assert_eq!(back, row);
}
#[test]
fn gauntlet_row_round_trip_populated_cpu_budget() {
let mut row = make_row("scn", "topo", true, 1.0);
row.cpu_budget = Some(4);
row.vcpus = Some(16);
let json = serde_json::to_string(&row).unwrap();
assert!(
json.contains("\"cpu_budget\":4") && json.contains("\"vcpus\":16"),
"populated budget/vcpus must emit numeric JSON keys: {json}"
);
let back: GauntletRow = serde_json::from_str(&json).unwrap();
assert_eq!(back, row);
assert_eq!(back.cpu_budget, Some(4));
assert_eq!(back.vcpus, Some(16));
}
#[test]
fn gauntlet_row_none_cpu_budget_omits_keys() {
let row = make_row("scn", "topo", true, 1.0);
assert!(row.cpu_budget.is_none() && row.vcpus.is_none());
let json = serde_json::to_string(&row).unwrap();
assert!(
!json.contains("\"cpu_budget\"") && !json.contains("\"vcpus\""),
"None budget/vcpus must be omitted from JSON: {json}"
);
let back: GauntletRow = serde_json::from_str(&json).unwrap();
assert_eq!(back, row);
}
#[test]
fn compare_partitions_threads_dir_through_to_pool_collection() {
use crate::test_support::SidecarResult;
let alt_root = tempfile::TempDir::new().expect("create alt-root tempdir");
for (run_key, sched) in [
("__dir_thread_a__", "scx_alpha"),
("__dir_thread_b__", "scx_beta"),
] {
let run_dir = alt_root.path().join(run_key);
std::fs::create_dir_all(&run_dir).expect("create run dir");
let sidecar = SidecarResult {
test_name: "dir_thread_fixture".to_string(),
scheduler: sched.to_string(),
..SidecarResult::test_fixture()
};
let json = serde_json::to_string(&sidecar).expect("serialize fixture sidecar");
let sidecar_path = run_dir.join(format!("{run_key}.ktstr.json"));
std::fs::write(&sidecar_path, json).expect("write fixture sidecar");
}
let filter_a = RowFilter {
schedulers: vec!["scx_alpha".to_string()],
..RowFilter::default()
};
let filter_b = RowFilter {
schedulers: vec!["scx_beta".to_string()],
..RowFilter::default()
};
let exit = compare_partitions(
&filter_a,
&filter_b,
None,
&ComparisonPolicy::default(),
Some(alt_root.path()),
false,
&PhaseDisplayOptions::default(),
)
.expect("compare_partitions must pool sidecars under --dir override");
assert_eq!(
exit, 0,
"byte-identical metrics across the two scheduler \
partitions must yield zero regressions (exit 0). \
A non-zero exit means either the partitions loaded \
different data than written above or compare_rows \
regressed on identical inputs.",
);
}
#[test]
fn render_dirty_warning_silent_when_no_dirty_commits() {
let mut row = make_row("scn", "topo", true, 1.0);
row.commit = Some("abcdef1".into());
row.kernel_commit = Some("0123456".into());
let other = row.clone();
assert!(
super::render_dirty_warning(&[row], &[other]).is_none(),
"clean rows on both sides must yield no warning"
);
}
#[test]
fn render_dirty_warning_silent_on_empty_inputs() {
assert!(
super::render_dirty_warning(&[], &[]).is_none(),
"empty inputs must yield no warning"
);
}
#[test]
fn render_dirty_warning_kernel_only_dedupes_values_across_sides() {
let mut a = make_row("scn", "topo", true, 1.0);
a.kernel_commit = Some("aaaaaaa-dirty".into());
a.commit = Some("clean01".into());
let mut a2 = make_row("scn2", "topo", true, 1.0);
a2.kernel_commit = Some("aaaaaaa-dirty".into()); let mut b = make_row("scn", "topo", true, 1.0);
b.kernel_commit = Some("bbbbbbb-dirty".into());
let text = super::render_dirty_warning(&[a, a2], &[b])
.expect("dirty kernel_commit must yield warning");
assert!(
text.contains("warning: comparison includes dirty builds:"),
"missing header in {text:?}"
);
assert_eq!(
text.matches("kernel source: aaaaaaa-dirty").count(),
1,
"duplicate kernel_commit must be deduped, got {text:?}"
);
assert!(
text.contains("kernel source: bbbbbbb-dirty"),
"second distinct dirty kernel_commit must be listed, got {text:?}"
);
assert!(
!text.contains("project:"),
"no -dirty project commit; the project line must not appear: {text:?}"
);
assert!(
text.contains("Dirty runs overwrite previous results with the same HEAD."),
"missing trailer line 1 in {text:?}"
);
assert!(
text.contains("Commit changes for reproducible-ish comparisons."),
"missing trailer line 2 in {text:?}"
);
}
#[test]
fn render_dirty_warning_project_only_omits_kernel_section() {
let mut a = make_row("scn", "topo", true, 1.0);
a.commit = Some("ccccccc-dirty".into());
let text = super::render_dirty_warning(&[a], &[]).expect("dirty commit must yield warning");
assert!(
text.contains("project: ccccccc-dirty"),
"expected project line in {text:?}"
);
assert!(
!text.contains("kernel source:"),
"kernel section must not appear when only project is dirty: {text:?}"
);
}
#[test]
fn render_dirty_warning_both_dimensions_in_stable_order() {
let mut a = make_row("scn", "topo", true, 1.0);
a.kernel_commit = Some("kkkkk22-dirty".into());
a.commit = Some("pppp222-dirty".into());
let mut b = make_row("scn", "topo", true, 1.0);
b.kernel_commit = Some("kkkkk11-dirty".into());
b.commit = Some("pppp111-dirty".into());
let text =
super::render_dirty_warning(&[a], &[b]).expect("both dimensions dirty must yield warning");
let kernel11 = text
.find("kernel source: kkkkk11-dirty")
.expect("kernel11 line absent");
let kernel22 = text
.find("kernel source: kkkkk22-dirty")
.expect("kernel22 line absent");
let project11 = text
.find("project: pppp111-dirty")
.expect("project11 line absent");
let project22 = text
.find("project: pppp222-dirty")
.expect("project22 line absent");
assert!(
kernel11 < kernel22,
"kernel section must list values in lex order: {text:?}"
);
assert!(
project11 < project22,
"project section must list values in lex order: {text:?}"
);
assert!(
kernel22 < project11,
"kernel section must precede project section: {text:?}"
);
}
#[test]
fn render_dirty_warning_skips_none_and_clean_values() {
let mut clean_a = make_row("a", "topo", true, 1.0);
clean_a.commit = Some("clean01".into());
clean_a.kernel_commit = None;
let mut dirty_b = make_row("b", "topo", true, 1.0);
dirty_b.commit = None;
dirty_b.kernel_commit = Some("dddddd1-dirty".into());
let text = super::render_dirty_warning(&[clean_a], &[dirty_b])
.expect("at least one dirty value must yield warning");
assert!(
text.contains("kernel source: dddddd1-dirty"),
"dirty kernel_commit must surface in {text:?}"
);
assert!(
!text.contains("project:"),
"no dirty project commit; project section must be absent in {text:?}"
);
assert!(
!text.contains("clean01"),
"clean commit values must not appear in {text:?}"
);
}
fn budget_row(scenario: &str, budget: Option<u32>, vcpus: Option<u32>) -> GauntletRow {
let mut r = make_row(scenario, "topo", true, 1.0);
r.cpu_budget = budget;
r.vcpus = vcpus;
r
}
#[test]
fn render_overcommit_warning_none_when_clean() {
let pairing: &[Dimension] = &[Dimension::CpuBudget];
let sliced: &[Dimension] = &[];
let a = budget_row("a", Some(16), Some(16));
let b = budget_row("b", Some(32), Some(16)); assert!(
super::render_overcommit_warning(
std::slice::from_ref(&a),
std::slice::from_ref(&b),
pairing,
)
.is_none()
);
assert!(super::render_overcommit_warning(&[a], &[b], sliced).is_none());
}
#[test]
fn render_overcommit_warning_ignores_skip_rows() {
let sliced: &[Dimension] = &[];
let a = budget_row("a", None, None);
let b = budget_row("b", None, None);
assert!(super::render_overcommit_warning(&[a], &[b], sliced).is_none());
}
#[test]
fn render_overcommit_warning_flags_overcommitted_side() {
let pairing: &[Dimension] = &[Dimension::CpuBudget];
let a = budget_row("a", Some(4), Some(16));
let b = budget_row("b", Some(16), Some(16));
let text = super::render_overcommit_warning(&[a], &[b], pairing)
.expect("an overcommitted A row must warn");
assert!(text.contains("side A"), "must name side A: {text}");
assert!(text.contains("4/16"), "must list budget/vcpus: {text}");
assert!(
text.contains("run-delay"),
"warning must list run-delay as confounded: {text}",
);
assert!(
!text.contains("side B"),
"the clean B side must not be flagged: {text}",
);
}
#[test]
fn render_overcommit_warning_mixed_budget_per_group() {
let pairing: &[Dimension] = &[Dimension::CpuBudget];
let sliced = Dimension::pairing_dims(&[Dimension::CpuBudget]);
let a = budget_row("a", Some(16), Some(16));
let b1 = budget_row("b", Some(8), Some(16)); let b2 = budget_row("b", Some(16), Some(16));
let paired = super::render_overcommit_warning(
std::slice::from_ref(&a),
&[b1.clone(), b2.clone()],
pairing,
)
.expect("overcommitted B row still warns");
assert!(
paired.contains("8/16") && !paired.contains("share a pairing group"),
"pairing dim: overcommit flagged, no mixed-fold warning: {paired}",
);
let sliced_same = super::render_overcommit_warning(&[a], &[b1, b2], &sliced)
.expect("mixed budgets in one group on a sliced side must warn");
assert!(
sliced_same.contains("share a pairing group") && sliced_same.contains("side B"),
"sliced same-key: must warn B's budgets share a pairing group: {sliced_same}",
);
let mut s1 = budget_row("c", Some(16), Some(16));
s1.scheduler = "sched_a".to_string();
let mut s2 = budget_row("c", Some(32), Some(32));
s2.scheduler = "sched_b".to_string();
let clean_a = budget_row("d", Some(16), Some(16));
assert!(
super::render_overcommit_warning(&[s1, s2], std::slice::from_ref(&clean_a), &sliced)
.is_none(),
"two budgets differing on a non-budget pairing dim (scheduler) key \
separate groups -> no fold -> no warning",
);
let xa = budget_row("x", Some(16), Some(16));
let ya = budget_row("y", Some(32), Some(32));
let clean_b = budget_row("z", Some(16), Some(16));
assert!(
super::render_overcommit_warning(&[xa, ya], std::slice::from_ref(&clean_b), &sliced)
.is_none(),
"one side spanning budgets across distinct scenarios -> no fold -> no warning",
);
}
#[test]
fn render_overcommit_warning_mixed_no_overcommit_uses_else_banner() {
let sliced = Dimension::pairing_dims(&[Dimension::CpuBudget]);
let b1 = budget_row("m", Some(16), Some(16));
let b2 = budget_row("m", Some(32), Some(32));
let clean = budget_row("n", Some(16), Some(16));
let text = super::render_overcommit_warning(&[b1, b2], std::slice::from_ref(&clean), &sliced)
.expect("two non-overcommit budgets folding into one group must warn");
assert!(
text.contains("mixing two measurement conditions"),
"no-overcommit mixed-budget case must use the else-branch banner, \
not the host-overcommit banner; got: {text}",
);
assert!(
!text.contains("host-overcommitted run"),
"the else branch must NOT claim a host-overcommitted run; got: {text}",
);
assert!(
text.contains("side A") && text.contains("share a pairing group"),
"the folded side must be named with its mixed budgets; got: {text}",
);
}
#[test]
fn check_no_duplicate_pairing_keys_ok_when_all_keys_distinct() {
let rows = vec![
cmp_row("alpha", "tiny-1llc", true, 10.0, 0),
cmp_row("beta", "tiny-1llc", true, 10.0, 0),
];
assert!(
super::check_no_duplicate_pairing_keys(&rows, LEGACY_PAIRING_DIMS, "A").is_ok(),
"distinct pairing keys must pass the --no-average duplicate gate",
);
assert!(
super::check_no_duplicate_pairing_keys(&[], LEGACY_PAIRING_DIMS, "A").is_ok(),
"empty side must pass the duplicate gate",
);
}
#[test]
fn check_no_duplicate_pairing_keys_bails_on_collision_and_names_side() {
let rows = vec![
cmp_row("dup", "tiny-1llc", true, 10.0, 0),
cmp_row("dup", "tiny-1llc", true, 20.0, 0),
];
let err = super::check_no_duplicate_pairing_keys(&rows, LEGACY_PAIRING_DIMS, "B")
.expect_err("two rows sharing a pairing key must bail under --no-average");
let rendered = format!("{err:#}");
assert!(
rendered.contains("side B"),
"bail must name the offending side; got: {rendered}",
);
assert!(
rendered.contains("same pairing key"),
"bail must describe the duplicate-key condition; got: {rendered}",
);
assert!(
rendered.contains("--no-average"),
"bail must point at the --no-average flag the operator can drop; got: {rendered}",
);
assert!(
rendered.contains("--b-"),
"bail must suggest a per-side filter to disambiguate; got: {rendered}",
);
}