use super::*;
#[test]
fn aggregate_samples_counter_sums_finite_values() {
assert_eq!(
aggregate_samples(&[1.0, 2.0, 3.0], MetricKind::Counter),
Some(6.0),
);
assert_eq!(
aggregate_samples(&[1.0, f64::NAN, 3.0], MetricKind::Counter),
Some(4.0),
"NaN samples drop from the sum",
);
assert_eq!(
aggregate_samples(&[], MetricKind::Counter),
None,
"empty input → None",
);
assert_eq!(
aggregate_samples(&[f64::NAN, f64::INFINITY], MetricKind::Counter),
None,
"all-non-finite → None",
);
}
#[test]
fn aggregate_samples_gauge_avg_means_finite() {
let r = aggregate_samples(&[1.0, 2.0, 3.0], MetricKind::Gauge(GaugeAgg::Avg));
assert_eq!(r, Some(2.0));
}
#[test]
fn aggregate_samples_gauge_last_returns_last() {
let r = aggregate_samples(&[1.0, 2.0, 3.0], MetricKind::Gauge(GaugeAgg::Last));
assert_eq!(r, Some(3.0));
let r = aggregate_samples(&[1.0, 2.0, f64::NAN], MetricKind::Gauge(GaugeAgg::Last));
assert_eq!(r, Some(2.0));
}
#[test]
fn aggregate_samples_max_and_peak_pick_largest() {
let r = aggregate_samples(&[1.0, 5.0, 3.0], MetricKind::Gauge(GaugeAgg::Max));
assert_eq!(r, Some(5.0));
let r = aggregate_samples(&[1.0, 5.0, 3.0], MetricKind::Peak);
assert_eq!(r, Some(5.0));
}
#[test]
fn aggregate_samples_timestamp_returns_last() {
let r = aggregate_samples(&[100.0, 200.0, 300.0], MetricKind::Timestamp);
assert_eq!(r, Some(300.0));
}
#[test]
fn aggregate_samples_weighted_gauge_avg_pulls_toward_heavier_sample() {
let r = aggregate_samples_weighted(&[(10.0, 5), (20.0, 15)], MetricKind::Gauge(GaugeAgg::Avg));
assert_eq!(r, Some(17.5));
}
#[test]
fn aggregate_samples_gauge_avg_unweighted_is_arithmetic_mean() {
let r = aggregate_samples(&[10.0, 20.0], MetricKind::Gauge(GaugeAgg::Avg));
assert_eq!(r, Some(15.0));
}
#[test]
fn aggregate_samples_weighted_gauge_avg_zero_total_weight_falls_back_to_mean() {
let r = aggregate_samples_weighted(&[(10.0, 0), (30.0, 0)], MetricKind::Gauge(GaugeAgg::Avg));
assert_eq!(r, Some(20.0));
}
#[test]
fn aggregate_samples_weighted_counter_ignores_weights() {
let r = aggregate_samples_weighted(&[(10.0, 5), (20.0, 15)], MetricKind::Counter);
assert_eq!(r, Some(30.0));
}
#[test]
fn aggregate_samples_weighted_peak_ignores_weights() {
let r = aggregate_samples_weighted(&[(10.0, 5), (20.0, 15)], MetricKind::Peak);
assert_eq!(r, Some(20.0));
}
#[test]
fn aggregate_samples_weighted_gauge_max_ignores_weights() {
let r = aggregate_samples_weighted(&[(10.0, 5), (20.0, 15)], MetricKind::Gauge(GaugeAgg::Max));
assert_eq!(r, Some(20.0));
}
#[test]
fn aggregate_samples_weighted_distribution_worstlowest_arms() {
assert_eq!(
aggregate_samples_weighted(
&[(10.0, 5), (20.0, 15)],
MetricKind::Distribution {
source: SampleSource::RunDelayNs,
reduction: SampleReduction::Worst,
},
),
Some(20.0),
);
assert_eq!(
aggregate_samples_weighted(
&[(10.0, 5), (30.0, 15)],
MetricKind::Distribution {
source: SampleSource::WakeLatencyNs,
reduction: SampleReduction::P99,
},
),
Some(20.0),
);
assert_eq!(
aggregate_samples_weighted(
&[(10.0, 5), (30.0, 15)],
MetricKind::WorstLowest {
numerator: WorstLowestNumerator::Iterations,
denominator: WorstLowestDenominator::CpuTimeNs,
},
),
Some(20.0),
);
}
#[test]
fn aggregate_samples_weighted_distribution_worstlowest_counts_zero_weight_contributor() {
assert_eq!(
aggregate_samples_weighted(
&[(10.0, 0), (30.0, 15)],
MetricKind::Distribution {
source: SampleSource::WakeLatencyNs,
reduction: SampleReduction::P99,
},
),
Some(20.0),
);
assert_eq!(
aggregate_samples_weighted(
&[(10.0, 0), (30.0, 15)],
MetricKind::WorstLowest {
numerator: WorstLowestNumerator::Iterations,
denominator: WorstLowestDenominator::CpuTimeNs,
},
),
Some(20.0),
);
}
#[test]
fn aggregate_samples_weighted_gauge_last_and_timestamp_ignore_weights() {
let last =
aggregate_samples_weighted(&[(10.0, 5), (20.0, 15)], MetricKind::Gauge(GaugeAgg::Last));
assert_eq!(last, Some(20.0));
let ts = aggregate_samples_weighted(&[(10.0, 5), (20.0, 15)], MetricKind::Timestamp);
assert_eq!(ts, Some(20.0));
}
#[test]
fn aggregate_samples_weighted_gauge_avg_drops_nan_pairs_in_lockstep() {
let r = aggregate_samples_weighted(
&[(10.0, 5), (f64::NAN, 10), (30.0, 20)],
MetricKind::Gauge(GaugeAgg::Avg),
);
assert_eq!(r, Some(26.0));
}
#[test]
fn phase_counter_delta_returns_last_minus_first() {
assert_eq!(
phase_counter_delta(&[100.0, 150.0, 175.0, 200.0]),
Some(100.0),
);
assert_eq!(
phase_counter_delta(&[f64::NAN, 150.0, 175.0, f64::NAN]),
Some(25.0),
);
}
#[test]
fn phase_counter_delta_under_two_finite_samples_is_unmeasurable() {
assert_eq!(phase_counter_delta(&[42.0]), None);
assert_eq!(phase_counter_delta(&[f64::NAN, 42.0, f64::NAN]), None);
assert_eq!(phase_counter_delta(&[]), None);
assert_eq!(phase_counter_delta(&[f64::NAN, f64::INFINITY]), None);
}
#[test]
fn phase_counter_delta_clamps_negative_to_zero_on_counter_reset() {
assert_eq!(
phase_counter_delta(&[500.0, 600.0, 100.0]),
Some(0.0),
"last < first clamps to 0 (counter reset detected)",
);
}
#[test]
fn irq_read_sample_arms_sum_across_cpus() {
use crate::monitor::btf_offsets::{NR_SOFTIRQS, SOFTIRQ_NET_RX};
use crate::monitor::dump::{FailureDumpReport, PerCpuTimeStats};
use crate::scenario::sample::SampleSeries;
use crate::scenario::snapshot::{DrainedSnapshotEntry, MissingStatsReason};
let net_rx = |n: u64| {
let mut s = [0u64; NR_SOFTIRQS];
s[SOFTIRQ_NET_RX] = n;
s
};
let entry = DrainedSnapshotEntry {
tag: "periodic_000".to_string(),
report: FailureDumpReport {
per_cpu_time: vec![
PerCpuTimeStats {
cpu: 0,
irqs_sum: 100,
softirqs: net_rx(10),
cpustat_irq_ns: 5_000,
..Default::default()
},
PerCpuTimeStats {
cpu: 1,
irqs_sum: 200,
softirqs: net_rx(20),
cpustat_irq_ns: 7_000,
..Default::default()
},
],
..Default::default()
},
stats: Err(MissingStatsReason::NoSchedulerBinary),
elapsed_ms: Some(100),
boundary_offset_ms: None,
step_index: Some(1),
};
let series = SampleSeries::from_drained_typed(vec![entry], None);
let sample = series.iter_samples().next().expect("one sample");
let read = |name: &str| {
crate::stats::metric_def(name)
.expect("registered metric")
.read_sample(&sample)
};
assert_eq!(
read("total_hardirqs"),
Some(300.0),
"cross-CPU sum of irqs_sum",
);
assert_eq!(
read("total_softirq_net_rx"),
Some(30.0),
"cross-CPU sum of softirqs[SOFTIRQ_NET_RX]",
);
assert_eq!(
read("total_irq_time_ns"),
Some(12_000.0),
"cross-CPU sum of cpustat_irq_ns",
);
let empty = DrainedSnapshotEntry {
tag: "periodic_001".to_string(),
report: FailureDumpReport::default(),
stats: Err(MissingStatsReason::NoSchedulerBinary),
elapsed_ms: Some(200),
boundary_offset_ms: None,
step_index: Some(1),
};
let empty_series = SampleSeries::from_drained_typed(vec![empty], None);
let empty_sample = empty_series.iter_samples().next().expect("one sample");
assert_eq!(
crate::stats::metric_def("total_hardirqs")
.expect("registered metric")
.read_sample(&empty_sample),
None,
"empty per_cpu_time -> None (loud-absent)",
);
}
#[test]
fn irq_read_sample_arms_saturate_on_per_cpu_overflow() {
use crate::monitor::btf_offsets::{NR_SOFTIRQS, SOFTIRQ_NET_RX};
use crate::monitor::dump::{FailureDumpReport, PerCpuTimeStats};
use crate::scenario::sample::SampleSeries;
use crate::scenario::snapshot::{DrainedSnapshotEntry, MissingStatsReason};
let net_rx = |n: u64| {
let mut s = [0u64; NR_SOFTIRQS];
s[SOFTIRQ_NET_RX] = n;
s
};
let entry = DrainedSnapshotEntry {
tag: "periodic_000".to_string(),
report: FailureDumpReport {
per_cpu_time: vec![
PerCpuTimeStats {
cpu: 0,
irqs_sum: u64::MAX,
softirqs: net_rx(u64::MAX),
cpustat_irq_ns: u64::MAX,
..Default::default()
},
PerCpuTimeStats {
cpu: 1,
irqs_sum: 5,
softirqs: net_rx(5),
cpustat_irq_ns: 5,
..Default::default()
},
],
..Default::default()
},
stats: Err(MissingStatsReason::NoSchedulerBinary),
elapsed_ms: Some(100),
boundary_offset_ms: None,
step_index: Some(1),
};
let series = SampleSeries::from_drained_typed(vec![entry], None);
let sample = series.iter_samples().next().expect("one sample");
let read = |name: &str| {
crate::stats::metric_def(name)
.expect("registered metric")
.read_sample(&sample)
};
assert_eq!(
read("total_hardirqs"),
Some(u64::MAX as f64),
"irqs_sum spatial sum saturates, not wraps",
);
assert_eq!(
read("total_softirq_net_rx"),
Some(u64::MAX as f64),
"softirq spatial sum saturates",
);
assert_eq!(
read("total_irq_time_ns"),
Some(u64::MAX as f64),
"cpustat_irq_ns spatial sum saturates",
);
}
#[test]
fn irq_counter_folds_per_phase_delta_and_rate() {
use crate::assert::build_phase_buckets;
use crate::monitor::btf_offsets::{NR_SOFTIRQS, SOFTIRQ_NET_RX};
use crate::monitor::dump::{FailureDumpReport, PerCpuTimeStats};
use crate::scenario::sample::SampleSeries;
use crate::scenario::snapshot::{DrainedSnapshotEntry, MissingStatsReason};
let cpu = |cpu: u32, irqs_sum: u64, net_rx: u64| {
let mut softirqs = [0u64; NR_SOFTIRQS];
softirqs[SOFTIRQ_NET_RX] = net_rx;
PerCpuTimeStats {
cpu,
irqs_sum,
softirqs,
..Default::default()
}
};
let freeze = |tag: &str, elapsed_ms: u64, c0: PerCpuTimeStats, c1: PerCpuTimeStats| {
DrainedSnapshotEntry {
tag: tag.to_string(),
report: FailureDumpReport {
per_cpu_time: vec![c0, c1],
..Default::default()
},
stats: Err(MissingStatsReason::NoSchedulerBinary),
elapsed_ms: Some(elapsed_ms),
boundary_offset_ms: None,
step_index: Some(1),
}
};
let two = SampleSeries::from_drained_typed(
vec![
freeze("periodic_000", 1_000, cpu(0, 100, 10), cpu(1, 200, 20)),
freeze("periodic_001", 4_000, cpu(0, 400, 40), cpu(1, 500, 60)),
],
None,
);
let buckets = build_phase_buckets(&two);
let bucket = buckets
.iter()
.find(|b| b.step_index == 1)
.expect("step-1 bucket present");
assert_eq!(
bucket.get("total_hardirqs"),
Some(600.0),
"cross-CPU last-minus-first: (400+500)-(100+200)",
);
assert_eq!(
bucket.get("total_softirq_net_rx"),
Some(70.0),
"cross-CPU NET_RX delta: (40+60)-(10+20)",
);
assert_eq!(
bucket.get("hardirq_rate"),
Some(200.0),
"600 hardirqs / 3.0 s capture window",
);
let one = SampleSeries::from_drained_typed(
vec![freeze(
"periodic_000",
1_000,
cpu(0, 100, 10),
cpu(1, 200, 20),
)],
None,
);
let buckets = build_phase_buckets(&one);
let bucket = buckets
.iter()
.find(|b| b.step_index == 1)
.expect("step-1 bucket present");
assert_eq!(
bucket.get("total_hardirqs"),
None,
"single-freeze phase: counter delta unmeasurable -> None",
);
assert_eq!(
bucket.get("hardirq_rate"),
None,
"no counter component -> no rate co-insertion",
);
}
#[test]
fn run_level_irq_rate_uses_whole_run_span_not_phase_sum() {
use crate::assert::{
build_phase_buckets, populate_run_ext_metrics, populate_run_ext_metrics_from_phases,
};
use crate::monitor::dump::{FailureDumpReport, PerCpuTimeStats};
use crate::scenario::sample::SampleSeries;
use crate::scenario::snapshot::{DrainedSnapshotEntry, MissingStatsReason};
let freeze = |tag: &str, elapsed_ms: u64, step: u16, irqs_sum: u64| DrainedSnapshotEntry {
tag: tag.to_string(),
report: FailureDumpReport {
per_cpu_time: vec![PerCpuTimeStats {
cpu: 0,
irqs_sum,
..Default::default()
}],
..Default::default()
},
stats: Err(MissingStatsReason::NoSchedulerBinary),
elapsed_ms: Some(elapsed_ms),
boundary_offset_ms: None,
step_index: Some(step),
};
let series = SampleSeries::from_drained_typed(
vec![
freeze("periodic_000", 1_000, 1, 100),
freeze("periodic_001", 4_000, 1, 400),
freeze("periodic_002", 6_000, 2, 600),
freeze("periodic_003", 9_000, 2, 900),
],
None,
);
let buckets = build_phase_buckets(&series);
let mut ext = std::collections::BTreeMap::new();
populate_run_ext_metrics(&series, &mut ext);
populate_run_ext_metrics_from_phases(&buckets, &mut ext);
assert_eq!(
ext.get("total_hardirqs").copied(),
Some(800.0),
"whole-run delta 900-100, not the Σ-per-phase 300+300=600",
);
assert_eq!(
ext.get("total_phase_wall_sec").copied(),
Some(8.0),
"whole-run span 8 s, not the Σ-per-phase 6 s",
);
let rate = ext
.get("hardirq_rate")
.copied()
.expect("hardirq_rate derived at run level");
assert!(
(rate - 100.0).abs() < 1e-9,
"whole-run rate 800/8s=100, not the mismatched 800/6s=133.3; got {rate}",
);
}
#[test]
fn run_level_irq_rate_survives_sparse_multi_phase() {
use crate::assert::{
build_phase_buckets, populate_run_ext_metrics, populate_run_ext_metrics_from_phases,
};
use crate::monitor::dump::{FailureDumpReport, PerCpuTimeStats};
use crate::scenario::sample::SampleSeries;
use crate::scenario::snapshot::{DrainedSnapshotEntry, MissingStatsReason};
let freeze = |tag: &str, elapsed_ms: u64, step: u16, irqs_sum: u64| DrainedSnapshotEntry {
tag: tag.to_string(),
report: FailureDumpReport {
per_cpu_time: vec![PerCpuTimeStats {
cpu: 0,
irqs_sum,
..Default::default()
}],
..Default::default()
},
stats: Err(MissingStatsReason::NoSchedulerBinary),
elapsed_ms: Some(elapsed_ms),
boundary_offset_ms: None,
step_index: Some(step),
};
let series = SampleSeries::from_drained_typed(
vec![
freeze("periodic_000", 1_000, 1, 100),
freeze("periodic_001", 9_000, 2, 500),
],
None,
);
let buckets = build_phase_buckets(&series);
for b in &buckets {
assert_eq!(
b.get("total_hardirqs"),
None,
"single-freeze phase has no measurable per-phase counter delta",
);
}
let mut ext = std::collections::BTreeMap::new();
populate_run_ext_metrics(&series, &mut ext);
populate_run_ext_metrics_from_phases(&buckets, &mut ext);
assert_eq!(
ext.get("total_hardirqs").copied(),
Some(400.0),
"whole-run delta survives where per-phase deltas vanish",
);
assert_eq!(
ext.get("total_phase_wall_sec").copied(),
Some(8.0),
"whole-run span 9-1=8 s",
);
let rate = ext
.get("hardirq_rate")
.copied()
.expect("rate derives where a phase-summed numerator would vanish");
assert!((rate - 50.0).abs() < 1e-9, "400/8s=50; got {rate}");
}
fn irq_spatial_freeze(
tag: &str,
elapsed_ms: u64,
step: u16,
cpus: &[(u32, u64)],
) -> crate::scenario::snapshot::DrainedSnapshotEntry {
use crate::monitor::dump::{FailureDumpReport, PerCpuTimeStats};
use crate::scenario::snapshot::MissingStatsReason;
crate::scenario::snapshot::DrainedSnapshotEntry {
tag: tag.to_string(),
report: FailureDumpReport {
per_cpu_time: cpus
.iter()
.map(|&(cpu, irqs_sum)| PerCpuTimeStats {
cpu,
irqs_sum,
..Default::default()
})
.collect(),
..Default::default()
},
stats: Err(MissingStatsReason::NoSchedulerBinary),
elapsed_ms: Some(elapsed_ms),
boundary_offset_ms: None,
step_index: Some(step),
}
}
fn irq_spatial_bucket(
freezes: Vec<crate::scenario::snapshot::DrainedSnapshotEntry>,
) -> (Option<f64>, Option<f64>) {
let series = crate::scenario::sample::SampleSeries::from_drained_typed(freezes, None);
let buckets = crate::assert::build_phase_buckets(&series);
let b = buckets
.into_iter()
.find(|b| b.step_index == 1)
.expect("step-1 bucket present");
(
b.get("max_cpu_hardirqs"),
b.get("max_cpu_hardirq_concentration"),
)
}
#[test]
fn max_cpu_hardirqs_happy_path() {
let (max, conc) = irq_spatial_bucket(vec![
irq_spatial_freeze("periodic_000", 1_000, 1, &[(0, 100), (1, 200)]),
irq_spatial_freeze("periodic_001", 4_000, 1, &[(0, 200), (1, 500)]),
]);
assert_eq!(max, Some(300.0), "busiest CPU's delta (cpu1: 500-200)");
let conc = conc.expect("concentration present with 2 reporting CPUs");
assert!(
(conc - 1.5).abs() < 1e-9,
"300 / mean(100,300)=200 = 1.5; got {conc}",
);
}
#[test]
fn max_cpu_hardirqs_uses_per_cpu_delta_not_totals_when_busiest_cpu_shifts() {
let (max, conc) = irq_spatial_bucket(vec![
irq_spatial_freeze("periodic_000", 1_000, 1, &[(0, 100), (1, 50)]),
irq_spatial_freeze("periodic_001", 4_000, 1, &[(0, 150), (1, 400)]),
]);
assert_eq!(
max,
Some(350.0),
"max of per-CPU deltas (d1=350), NOT max(totals)=400-100=300",
);
let conc = conc.expect("concentration present");
assert!(
(conc - 1.75).abs() < 1e-9,
"350 / mean(50,350)=200 = 1.75; got {conc}",
);
}
#[test]
fn max_cpu_hardirqs_uses_win_ordered_endpoints_not_positional() {
let (max, _) = irq_spatial_bucket(vec![
irq_spatial_freeze("periodic_000", 4_000, 1, &[(0, 500), (1, 300)]),
irq_spatial_freeze("periodic_001", 1_000, 1, &[(0, 100), (1, 50)]),
]);
assert_eq!(
max,
Some(400.0),
"win-ordered cpu0 500-100=400 (max); positional .first()/.last() would clamp to 0",
);
}
#[test]
fn max_cpu_hardirqs_skips_cpu_absent_from_an_endpoint() {
let (max, conc) = irq_spatial_bucket(vec![
irq_spatial_freeze("periodic_000", 1_000, 1, &[(0, 100), (1, 9999)]),
irq_spatial_freeze("periodic_001", 4_000, 1, &[(0, 300)]),
]);
assert_eq!(
max,
Some(200.0),
"only cpu0 (present in both freezes) contributes"
);
assert_eq!(conc, None, "1-CPU intersection → concentration omitted");
}
#[test]
fn max_cpu_hardirqs_none_for_single_freeze() {
let (max, conc) = irq_spatial_bucket(vec![irq_spatial_freeze(
"periodic_000",
1_000,
1,
&[(0, 100), (1, 200)],
)]);
assert_eq!(max, None, "single freeze → no per-CPU delta measurable");
assert_eq!(conc, None, "single freeze → no concentration");
}
#[test]
fn max_cpu_hardirq_concentration_omitted_for_single_cpu() {
let (max, conc) = irq_spatial_bucket(vec![
irq_spatial_freeze("periodic_000", 1_000, 1, &[(0, 100)]),
irq_spatial_freeze("periodic_001", 4_000, 1, &[(0, 700)]),
]);
assert_eq!(max, Some(600.0), "cpu0 delta 700-100");
assert_eq!(conc, None, "1 CPU → concentration omitted (a trivial 1.0)");
}
#[test]
fn max_cpu_hardirq_concentration_absent_when_no_irqs() {
let (max, conc) = irq_spatial_bucket(vec![
irq_spatial_freeze("periodic_000", 1_000, 1, &[(0, 500), (1, 500)]),
irq_spatial_freeze("periodic_001", 4_000, 1, &[(0, 500), (1, 500)]),
]);
assert_eq!(
max,
Some(0.0),
"both CPUs' counters didn't advance → measured zero"
);
assert_eq!(
conc, None,
"mean==0 → concentration absent (no div-by-zero/NaN)"
);
}
#[test]
fn max_cpu_hardirqs_run_level_auto_folds_max_across_phases() {
use crate::assert::{
build_phase_buckets, populate_run_ext_metrics, populate_run_ext_metrics_from_phases,
};
use crate::scenario::sample::SampleSeries;
let series = SampleSeries::from_drained_typed(
vec![
irq_spatial_freeze("periodic_000", 1_000, 1, &[(0, 100), (1, 200)]),
irq_spatial_freeze("periodic_001", 4_000, 1, &[(0, 200), (1, 500)]),
irq_spatial_freeze("periodic_002", 6_000, 2, &[(0, 1000), (1, 1000)]),
irq_spatial_freeze("periodic_003", 9_000, 2, &[(0, 1100), (1, 1900)]),
],
None,
);
let buckets = build_phase_buckets(&series);
let mut ext = std::collections::BTreeMap::new();
populate_run_ext_metrics(&series, &mut ext);
populate_run_ext_metrics_from_phases(&buckets, &mut ext);
assert_eq!(
ext.get("max_cpu_hardirqs").copied(),
Some(900.0),
"Peak auto-folds max-across-phases: max(phase1 Δ=300, phase2 Δ=900)",
);
let conc = ext
.get("max_cpu_hardirq_concentration")
.copied()
.expect("concentration auto-folds to run-level too (also Peak)");
assert!(
(conc - 1.8).abs() < 1e-9,
"concentration auto-folds max-across-phases: max(1.5, 1.8) = 1.8; got {conc}",
);
}
fn softirq_net_rx_freeze(
tag: &str,
elapsed_ms: u64,
step: u16,
cpus: &[(u32, u64)],
) -> crate::scenario::snapshot::DrainedSnapshotEntry {
use crate::monitor::btf_offsets::SOFTIRQ_NET_RX;
use crate::monitor::dump::{FailureDumpReport, PerCpuTimeStats};
use crate::scenario::snapshot::MissingStatsReason;
crate::scenario::snapshot::DrainedSnapshotEntry {
tag: tag.to_string(),
report: FailureDumpReport {
per_cpu_time: cpus
.iter()
.map(|&(cpu, net_rx)| {
let mut c = PerCpuTimeStats {
cpu,
..Default::default()
};
c.softirqs[SOFTIRQ_NET_RX] = net_rx;
c
})
.collect(),
..Default::default()
},
stats: Err(MissingStatsReason::NoSchedulerBinary),
elapsed_ms: Some(elapsed_ms),
boundary_offset_ms: None,
step_index: Some(step),
}
}
fn softirq_net_rx_bucket(
freezes: Vec<crate::scenario::snapshot::DrainedSnapshotEntry>,
) -> (Option<f64>, Option<f64>) {
let series = crate::scenario::sample::SampleSeries::from_drained_typed(freezes, None);
let buckets = crate::assert::build_phase_buckets(&series);
let b = buckets
.into_iter()
.find(|b| b.step_index == 1)
.expect("step-1 bucket present");
(
b.get("max_cpu_softirq_net_rx"),
b.get("max_cpu_softirq_net_rx_concentration"),
)
}
#[test]
fn max_cpu_softirq_net_rx_happy_path() {
let (max, conc) = softirq_net_rx_bucket(vec![
softirq_net_rx_freeze("periodic_000", 1_000, 1, &[(0, 100), (1, 200)]),
softirq_net_rx_freeze("periodic_001", 4_000, 1, &[(0, 200), (1, 500)]),
]);
assert_eq!(
max,
Some(300.0),
"busiest CPU's NET_RX softirq delta (cpu1: 500-200)",
);
let conc = conc.expect("concentration present with 2 reporting CPUs");
assert!(
(conc - 1.5).abs() < 1e-9,
"300 / mean(100,300)=200 = 1.5; got {conc}",
);
}
#[test]
fn max_cpu_softirq_net_rx_uses_per_cpu_delta_when_busiest_cpu_shifts() {
let (max, conc) = softirq_net_rx_bucket(vec![
softirq_net_rx_freeze("periodic_000", 1_000, 1, &[(0, 100), (1, 50)]),
softirq_net_rx_freeze("periodic_001", 4_000, 1, &[(0, 150), (1, 400)]),
]);
assert_eq!(
max,
Some(350.0),
"max of per-CPU NET_RX deltas (d1=350), NOT max(totals)=400-100=300",
);
let conc = conc.expect("concentration present");
assert!(
(conc - 1.75).abs() < 1e-9,
"350 / mean(50,350)=200 = 1.75; got {conc}",
);
}
#[test]
fn max_cpu_softirq_net_rx_run_level_auto_folds_max_across_phases() {
use crate::assert::{
build_phase_buckets, populate_run_ext_metrics, populate_run_ext_metrics_from_phases,
};
use crate::scenario::sample::SampleSeries;
let series = SampleSeries::from_drained_typed(
vec![
softirq_net_rx_freeze("periodic_000", 1_000, 1, &[(0, 100), (1, 200)]),
softirq_net_rx_freeze("periodic_001", 4_000, 1, &[(0, 200), (1, 500)]),
softirq_net_rx_freeze("periodic_002", 6_000, 2, &[(0, 1000), (1, 1000)]),
softirq_net_rx_freeze("periodic_003", 9_000, 2, &[(0, 1100), (1, 1900)]),
],
None,
);
let buckets = build_phase_buckets(&series);
let mut ext = std::collections::BTreeMap::new();
populate_run_ext_metrics(&series, &mut ext);
populate_run_ext_metrics_from_phases(&buckets, &mut ext);
assert_eq!(
ext.get("max_cpu_softirq_net_rx").copied(),
Some(900.0),
"Peak auto-folds max-across-phases: max(phase1 Δ=300, phase2 Δ=900)",
);
let conc = ext
.get("max_cpu_softirq_net_rx_concentration")
.copied()
.expect("concentration auto-folds to run-level too (also Peak)");
assert!(
(conc - 1.8).abs() < 1e-9,
"concentration auto-folds max-across-phases: max(1.5, 1.8) = 1.8; got {conc}",
);
}
fn cgroup_psi_freeze(
tag: &str,
elapsed_ms: u64,
step: u16,
leaves: &[(u64, u64, u64)],
) -> crate::scenario::snapshot::DrainedSnapshotEntry {
use crate::monitor::cgroup_walk::CgroupPsiStat;
use crate::monitor::dump::FailureDumpReport;
use crate::scenario::snapshot::MissingStatsReason;
crate::scenario::snapshot::DrainedSnapshotEntry {
tag: tag.to_string(),
report: FailureDumpReport {
cgroup_psi: leaves
.iter()
.map(|&(cgroup_kva, total_ns, avg10_raw)| CgroupPsiStat {
cgroup_kva,
total_ns,
avg10_raw,
serial_nr: cgroup_kva,
})
.collect(),
..Default::default()
},
stats: Err(MissingStatsReason::NoSchedulerBinary),
elapsed_ms: Some(elapsed_ms),
boundary_offset_ms: None,
step_index: Some(step),
}
}
fn cgroup_psi_freeze_with_serial(
tag: &str,
elapsed_ms: u64,
step: u16,
leaves: &[(u64, u64, u64, u64)],
) -> crate::scenario::snapshot::DrainedSnapshotEntry {
use crate::monitor::cgroup_walk::CgroupPsiStat;
use crate::monitor::dump::FailureDumpReport;
use crate::scenario::snapshot::MissingStatsReason;
crate::scenario::snapshot::DrainedSnapshotEntry {
tag: tag.to_string(),
report: FailureDumpReport {
cgroup_psi: leaves
.iter()
.map(
|&(cgroup_kva, serial_nr, total_ns, avg10_raw)| CgroupPsiStat {
cgroup_kva,
total_ns,
avg10_raw,
serial_nr,
},
)
.collect(),
..Default::default()
},
stats: Err(MissingStatsReason::NoSchedulerBinary),
elapsed_ms: Some(elapsed_ms),
boundary_offset_ms: None,
step_index: Some(step),
}
}
fn cgroup_psi_bucket(
freezes: Vec<crate::scenario::snapshot::DrainedSnapshotEntry>,
) -> (Option<f64>, Option<f64>, Option<f64>) {
let series = crate::scenario::sample::SampleSeries::from_drained_typed(freezes, None);
let buckets = crate::assert::build_phase_buckets(&series);
let b = buckets
.into_iter()
.find(|b| b.step_index == 1)
.expect("step-1 bucket present");
(
b.get("max_cgroup_irq_pressure"),
b.get("max_cgroup_irq_pressure_concentration"),
b.get("max_cgroup_psi_irq_avg10"),
)
}
#[test]
fn max_cgroup_irq_pressure_happy_path() {
let (max, conc, avg10) = cgroup_psi_bucket(vec![
cgroup_psi_freeze(
"periodic_000",
1_000,
1,
&[(0xA00, 100_000, 2048), (0xB00, 200_000, 4096)],
),
cgroup_psi_freeze(
"periodic_001",
4_000,
1,
&[(0xA00, 200_000, 6144), (0xB00, 500_000, 8192)],
),
]);
assert_eq!(
max,
Some(300.0),
"busiest leaf's delta decoded to µs (B: (500_000-200_000)/1000)"
);
let conc = conc.expect("concentration present with 2 reporting leaves");
assert!(
(conc - 1.5).abs() < 1e-9,
"300 / mean(100,300)=200 = 1.5; got {conc}"
);
let avg10 = avg10.expect("avg10 gauge present");
assert!(
(avg10 - 4.0).abs() < 1e-9,
"max-across-freezes of max-across-leaves: max(2%,4%)=4; got {avg10}"
);
}
#[test]
fn max_cgroup_irq_pressure_uses_per_leaf_delta_when_busiest_shifts() {
let (max, conc, _) = cgroup_psi_bucket(vec![
cgroup_psi_freeze(
"periodic_000",
1_000,
1,
&[(0xA00, 100_000, 0), (0xB00, 50_000, 0)],
),
cgroup_psi_freeze(
"periodic_001",
4_000,
1,
&[(0xA00, 150_000, 0), (0xB00, 400_000, 0)],
),
]);
assert_eq!(
max,
Some(350.0),
"max of per-leaf deltas (dB=350µs), NOT max(totals)=400-100=300µs"
);
let conc = conc.expect("concentration present");
assert!(
(conc - 1.75).abs() < 1e-9,
"350 / mean(50,350)=200 = 1.75; got {conc}"
);
}
#[test]
fn max_cgroup_irq_pressure_uses_win_ordered_endpoints_not_positional() {
let (max, _, _) = cgroup_psi_bucket(vec![
cgroup_psi_freeze(
"periodic_000",
4_000,
1,
&[(0xA00, 500_000, 0), (0xB00, 300_000, 0)],
),
cgroup_psi_freeze(
"periodic_001",
1_000,
1,
&[(0xA00, 100_000, 0), (0xB00, 50_000, 0)],
),
]);
assert_eq!(
max,
Some(400.0),
"win-ordered A (500_000-100_000)/1000=400µs (max); positional would clamp to 0",
);
}
#[test]
fn max_cgroup_irq_pressure_skips_leaf_absent_from_an_endpoint() {
let (max, conc, _) = cgroup_psi_bucket(vec![
cgroup_psi_freeze(
"periodic_000",
1_000,
1,
&[(0xA00, 100_000, 0), (0xB00, 9_999_000, 0)],
),
cgroup_psi_freeze("periodic_001", 4_000, 1, &[(0xA00, 300_000, 0)]),
]);
assert_eq!(
max,
Some(200.0),
"only A (present in both freezes) contributes"
);
assert_eq!(conc, None, "1-leaf intersection → concentration omitted");
}
#[test]
fn max_cgroup_psi_irq_avg10_present_on_single_freeze_while_delta_absent() {
let (max, conc, avg10) = cgroup_psi_bucket(vec![cgroup_psi_freeze(
"periodic_000",
1_000,
1,
&[(0xA00, 100_000, 2048), (0xB00, 200_000, 4096)],
)]);
assert_eq!(max, None, "single freeze → no per-leaf delta measurable");
assert_eq!(conc, None, "single freeze → no concentration");
let avg10 = avg10.expect("avg10 is an instantaneous gauge, present on one freeze");
assert!(
(avg10 - 2.0).abs() < 1e-9,
"max-across-leaves of the lone freeze: max(1%,2%)=2; got {avg10}"
);
}
#[test]
fn max_cgroup_irq_pressure_concentration_omitted_for_single_leaf() {
let (max, conc, _) = cgroup_psi_bucket(vec![
cgroup_psi_freeze("periodic_000", 1_000, 1, &[(0xA00, 100_000, 0)]),
cgroup_psi_freeze("periodic_001", 4_000, 1, &[(0xA00, 700_000, 0)]),
]);
assert_eq!(max, Some(600.0), "A delta (700_000-100_000)/1000");
assert_eq!(conc, None, "1 leaf → concentration omitted (a trivial 1.0)");
}
#[test]
fn max_cgroup_irq_pressure_concentration_absent_when_no_pressure() {
let (max, conc, _) = cgroup_psi_bucket(vec![
cgroup_psi_freeze(
"periodic_000",
1_000,
1,
&[(0xA00, 500_000, 0), (0xB00, 500_000, 0)],
),
cgroup_psi_freeze(
"periodic_001",
4_000,
1,
&[(0xA00, 500_000, 0), (0xB00, 500_000, 0)],
),
]);
assert_eq!(
max,
Some(0.0),
"both leaves' counters didn't advance → measured zero"
);
assert_eq!(
conc, None,
"mean==0 → concentration absent (no div-by-zero/NaN)"
);
}
#[test]
fn cgroup_psi_metrics_all_absent_when_no_leaf_reported() {
let (max, conc, avg10) = cgroup_psi_bucket(vec![
cgroup_psi_freeze("periodic_000", 1_000, 1, &[]),
cgroup_psi_freeze("periodic_001", 4_000, 1, &[]),
]);
assert_eq!(max, None, "no leaf → no pressure delta");
assert_eq!(conc, None, "no leaf → no concentration");
assert_eq!(avg10, None, "no leaf → no avg10 gauge");
}
#[test]
fn cgroup_psi_metrics_run_level_auto_fold_max_across_phases() {
use crate::assert::{
build_phase_buckets, populate_run_ext_metrics, populate_run_ext_metrics_from_phases,
};
use crate::scenario::sample::SampleSeries;
let series = SampleSeries::from_drained_typed(
vec![
cgroup_psi_freeze(
"periodic_000",
1_000,
1,
&[(0xA00, 100_000, 2048), (0xB00, 200_000, 4096)],
),
cgroup_psi_freeze(
"periodic_001",
4_000,
1,
&[(0xA00, 200_000, 2048), (0xB00, 500_000, 4096)],
),
cgroup_psi_freeze(
"periodic_002",
6_000,
2,
&[(0xA00, 1_000_000, 10240), (0xB00, 1_000_000, 12288)],
),
cgroup_psi_freeze(
"periodic_003",
9_000,
2,
&[(0xA00, 1_100_000, 10240), (0xB00, 1_900_000, 12288)],
),
],
None,
);
let buckets = build_phase_buckets(&series);
let mut ext = std::collections::BTreeMap::new();
populate_run_ext_metrics(&series, &mut ext);
populate_run_ext_metrics_from_phases(&buckets, &mut ext);
assert_eq!(
ext.get("max_cgroup_irq_pressure").copied(),
Some(900.0),
"Peak auto-folds max-across-phases: max(phase1 Δ=300µs, phase2 Δ=900µs)",
);
let conc = ext
.get("max_cgroup_irq_pressure_concentration")
.copied()
.expect("concentration auto-folds run-level too (Peak)");
assert!(
(conc - 1.8).abs() < 1e-9,
"max(phase1 1.5, phase2 900/mean(100,900)=1.8) = 1.8; got {conc}",
);
let avg10 = ext
.get("max_cgroup_psi_irq_avg10")
.copied()
.expect("avg10 gauge auto-folds run-level too (Peak)");
assert!(
(avg10 - 6.0).abs() < 1e-9,
"max(phase1 2%, phase2 6%) = 6; got {avg10}",
);
}
#[test]
fn max_cgroup_irq_pressure_disambiguates_reused_kva_by_serial() {
let (max, conc, _) = cgroup_psi_bucket(vec![
cgroup_psi_freeze_with_serial(
"periodic_000",
1_000,
1,
&[(0xA00, 1, 100_000, 0), (0xB00, 2, 500_000, 0)],
),
cgroup_psi_freeze_with_serial(
"periodic_001",
4_000,
1,
&[(0xA00, 1, 200_000, 0), (0xB00, 3, 9_000_000, 0)],
),
]);
assert_eq!(
max,
Some(100.0),
"only A (kva+serial match in both freezes) contributes 100µs; the reused-KVA \
0xB00 (serial 2→3) is dropped, NOT differenced into a bogus 8500µs",
);
assert_eq!(
conc, None,
"1-leaf serial-matched intersection → concentration omitted",
);
}
#[test]
fn aggregate_samples_for_phase_returns_none_for_derived_kinds() {
let mk = |kind: MetricKind| MetricDef {
name: "x",
accessor: |_| None,
display_unit: "",
polarity: crate::test_support::Polarity::LowerBetter,
default_abs: 0.0,
default_rel: 0.0,
kind,
};
for kind in [
MetricKind::Rate {
numerator: "a",
denominator: "b",
},
MetricKind::Distribution {
source: SampleSource::WakeLatencyNs,
reduction: SampleReduction::P99,
},
MetricKind::WorstLowest {
numerator: WorstLowestNumerator::Iterations,
denominator: WorstLowestDenominator::NumWorkers,
},
MetricKind::WakeLatencyTailRatio,
MetricKind::WorstCrossNodeRatio,
MetricKind::PerPhase,
] {
assert!(kind.is_derived(), "{kind:?} must be is_derived");
assert_eq!(kind.merge_kind(), MergeKind::Recompute);
assert_eq!(
aggregate_samples_for_phase(&mk(kind), &[1.0, 2.0, 3.0]),
None,
"derived kind {kind:?} must have no per-phase reduction",
);
}
}
#[test]
fn aggregate_samples_for_phase_dispatches_on_kind() {
let counter = MetricDef {
name: "total_test_counter",
accessor: |_| None,
display_unit: "",
polarity: crate::test_support::Polarity::HigherBetter,
default_abs: 0.0,
default_rel: 0.0,
kind: MetricKind::Counter,
};
assert_eq!(
aggregate_samples_for_phase(&counter, &[100.0, 150.0, 175.0]),
Some(75.0),
"Counter kind must reduce by last - first, not by sum",
);
assert_ne!(
aggregate_samples_for_phase(&counter, &[100.0, 150.0, 175.0]),
Some(425.0),
"Counter kind MUST NOT collapse to flat-run sum across a phase",
);
let peak = MetricDef {
name: "max_test_peak",
accessor: |_| None,
display_unit: "",
polarity: crate::test_support::Polarity::LowerBetter,
default_abs: 0.0,
default_rel: 0.0,
kind: MetricKind::Peak,
};
assert_eq!(
aggregate_samples_for_phase(&peak, &[1.0, 5.0, 3.0]),
Some(5.0),
"Peak kind must reduce by max",
);
let gauge_avg = MetricDef {
name: "worst_test_gauge",
accessor: |_| None,
display_unit: "",
polarity: crate::test_support::Polarity::LowerBetter,
default_abs: 0.0,
default_rel: 0.0,
kind: MetricKind::Gauge(GaugeAgg::Avg),
};
assert_eq!(
aggregate_samples_for_phase(&gauge_avg, &[2.0, 4.0, 6.0]),
Some(4.0),
"Gauge(Avg) kind must reduce by arithmetic mean",
);
let delta_sum = MetricDef {
name: "total_test_delta",
accessor: |_| None,
display_unit: "",
polarity: crate::test_support::Polarity::LowerBetter,
default_abs: 0.0,
default_rel: 0.0,
kind: MetricKind::DeltaSum,
};
assert_eq!(
aggregate_samples_for_phase(&delta_sum, &[10.0, 20.0, 5.0]),
Some(35.0),
"DeltaSum kind must reduce by sum of per-read deltas",
);
assert_eq!(
aggregate_samples(&[10.0, 20.0, 5.0], MetricKind::DeltaSum),
Some(35.0),
"DeltaSum flat-run reduction is also a sum",
);
}
#[test]
fn rate_derives_per_phase_and_repools_across_merge() {
use std::collections::BTreeMap;
let mut phase = BTreeMap::new();
phase.insert("iters".to_string(), 1000.0);
phase.insert("secs".to_string(), 4.0);
derive_rate_metrics_from(&mut phase, std::iter::once(("rate", "iters", "secs")));
assert_eq!(
phase.get("rate").copied(),
Some(250.0),
"per-phase rate = num/denom",
);
let mut merged = BTreeMap::new();
merged.insert("iters".to_string(), 1000.0 + 10.0); merged.insert("secs".to_string(), 1.0 + 9.0); derive_rate_metrics_from(&mut merged, std::iter::once(("rate", "iters", "secs")));
assert_eq!(
merged.get("rate").copied(),
Some(101.0),
"merged rate must re-pool Σnum/Σdenom",
);
let mean_of_ratios = (1000.0 + (10.0 / 9.0)) / 2.0;
assert!(
(merged.get("rate").copied().unwrap() - mean_of_ratios).abs() > 100.0,
"re-pool must differ from mean-of-ratios (got {:?}, mean-of-ratios {mean_of_ratios})",
merged.get("rate"),
);
}
#[test]
fn rate_absent_on_missing_component_zero_or_nonfinite() {
use std::collections::BTreeMap;
let mut m = BTreeMap::new();
m.insert("iters".to_string(), 5.0);
derive_rate_metrics_from(&mut m, std::iter::once(("rate", "iters", "secs")));
assert!(!m.contains_key("rate"), "absent denom -> no rate key");
m.insert("secs".to_string(), 0.0);
derive_rate_metrics_from(&mut m, std::iter::once(("rate", "iters", "secs")));
assert!(!m.contains_key("rate"), "zero denom -> no rate key");
let mut n = BTreeMap::new();
n.insert("iters".to_string(), f64::NAN);
n.insert("secs".to_string(), 2.0);
derive_rate_metrics_from(&mut n, std::iter::once(("rate", "iters", "secs")));
assert!(!n.contains_key("rate"), "NaN numerator -> no rate key");
let mut o = BTreeMap::new();
o.insert("iters".to_string(), f64::MAX);
o.insert("secs".to_string(), f64::MIN_POSITIVE);
derive_rate_metrics_from(&mut o, std::iter::once(("rate", "iters", "secs")));
assert!(!o.contains_key("rate"), "inf quotient -> no rate key");
}
#[test]
fn rate_kind_returns_none_from_per_phase_reducer() {
let rate = MetricDef {
name: "test_rate",
accessor: |_| None,
display_unit: "",
polarity: crate::test_support::Polarity::HigherBetter,
default_abs: 0.0,
default_rel: 0.0,
kind: MetricKind::Rate {
numerator: "n",
denominator: "d",
},
};
assert_eq!(
aggregate_samples_for_phase(&rate, &[1.0, 2.0, 3.0]),
None,
"Rate reduces to None per-phase; derive_rate_metrics owns it",
);
}
#[test]
#[should_panic(expected = "must be derived via derive_rate_metrics")]
fn rate_kind_panics_in_single_slice_reducer() {
let _ = aggregate_samples(
&[1.0, 2.0],
MetricKind::Rate {
numerator: "n",
denominator: "d",
},
);
}
#[test]
fn aggregate_samples_for_phase_returns_none_on_empty_or_all_nan() {
let counter = MetricDef {
name: "total_x",
accessor: |_| None,
display_unit: "",
polarity: crate::test_support::Polarity::HigherBetter,
default_abs: 0.0,
default_rel: 0.0,
kind: MetricKind::Counter,
};
assert_eq!(aggregate_samples_for_phase(&counter, &[]), None);
assert_eq!(
aggregate_samples_for_phase(&counter, &[f64::NAN, f64::NAN]),
None,
);
let peak = MetricDef {
name: "max_x",
accessor: |_| None,
display_unit: "",
polarity: crate::test_support::Polarity::LowerBetter,
default_abs: 0.0,
default_rel: 0.0,
kind: MetricKind::Peak,
};
assert_eq!(aggregate_samples_for_phase(&peak, &[]), None);
assert_eq!(
aggregate_samples_for_phase(&peak, &[f64::NAN, f64::INFINITY]),
None,
);
}
#[test]
fn outlier_metrics_are_lower_better_in_registry() {
for (display, registry_key, _) in OUTLIER_METRICS {
let def = metric_def(registry_key).unwrap_or_else(|| {
panic!(
"OUTLIER_METRICS entry {display:?} names registry_key {registry_key:?}, \
which is not in the METRICS registry — every outlier metric must map to a \
registry entry whose polarity the guard can verify."
)
});
assert!(
matches!(def.polarity, crate::test_support::Polarity::LowerBetter),
"OUTLIER_METRICS entry {display:?} (registry_key {registry_key:?}) resolves to \
registry polarity {:?}, but outlier detection flags HIGH values and reads \
absent ext values as a 0.0 best-case sentinel — both require LowerBetter. A \
HigherBetter metric here silently corrupts outlier detection; make \
find_outliers polarity-aware before adding one.",
def.polarity,
);
}
}
#[test]
fn every_metric_has_kind_consistent_with_naming() {
for m in METRICS {
if matches!(m.kind, MetricKind::Counter | MetricKind::DeltaSum) {
assert!(
m.name.starts_with("total_") || m.name.ends_with("_count"),
"Counter/DeltaSum-kind metric must follow total_*/*_count naming, got {:?}",
m.name,
);
}
if matches!(m.kind, MetricKind::Peak) {
assert!(
m.name.starts_with("max_") || m.name == "worst_gap_ms",
"Peak-kind metric must use max_* naming OR be a documented worst-* peak, got {:?}",
m.name,
);
}
if matches!(m.kind, MetricKind::Distribution { .. }) {
assert_eq!(
m.polarity,
crate::test_support::Polarity::LowerBetter,
"Distribution-kind metric {:?} must be LowerBetter \
(the carrier-less fold maxes); got {:?}",
m.name,
m.polarity,
);
}
if matches!(m.kind, MetricKind::WorstLowest { .. }) {
assert_eq!(
m.polarity,
crate::test_support::Polarity::HigherBetter,
"WorstLowest-kind metric {:?} must be HigherBetter \
(the lowest-wins fold treats lowest as worst); got {:?}",
m.name,
m.polarity,
);
}
if matches!(m.kind, MetricKind::WorstCrossNodeRatio) {
assert_eq!(
m.polarity,
crate::test_support::Polarity::LowerBetter,
"WorstCrossNodeRatio-kind metric {:?} must be LowerBetter \
(the highest-wins fold treats highest as worst); got {:?}",
m.name,
m.polarity,
);
}
if let MetricKind::Rate {
numerator,
denominator,
} = m.kind
{
assert!(
m.name.ends_with("_rate")
|| m.name.contains("_per_")
|| m.name.ends_with("_fraction")
|| m.name.ends_with("_ratio"),
"Rate-kind metric must use *_rate / *_per_* / *_fraction / *_ratio naming, got {:?}",
m.name,
);
for comp in [numerator, denominator] {
let cd = metric_def(comp).unwrap_or_else(|| {
panic!(
"Rate metric {:?} component {comp:?} is not registered",
m.name
)
});
assert!(
!matches!(cd.kind, MetricKind::Rate { .. }),
"Rate metric {:?} component {comp:?} must not itself be Rate \
(a rate-of-a-rate breaks the associative re-derive)",
m.name,
);
}
}
if matches!(m.kind, MetricKind::PerRunDistribution) {
assert!(
m.name.ends_with("_whole"),
"PerRunDistribution-kind metric must use *_whole naming, got {:?}",
m.name,
);
}
let looks_like_rate = m.name.ends_with("_rate")
|| m.name.contains("_per_sec")
|| m.name.contains("_per_cpu_sec");
if looks_like_rate
&& m.name != "worst_iterations_per_cpu_sec"
&& m.name != "taobench_hit_rate"
{
assert!(
matches!(m.kind, MetricKind::Rate { .. }),
"metric {:?} is named like a per-second rate but is not \
MetricKind::Rate (register it as a Rate, or allowlist it \
here if it is intentionally a non-re-pooled gauge)",
m.name,
);
}
}
}
#[test]
fn mean_std_basic() {
let xs = [1.0_f64, 2.0, 3.0, 4.0, 5.0];
let m = mean(xs.iter().copied());
let s = std_dev(xs.iter().copied());
assert!((m - 3.0).abs() < 0.01);
assert!(s > 1.0);
}
#[test]
fn mean_std_empty_returns_zero() {
let empty: [f64; 0] = [];
assert_eq!(mean(empty.iter().copied()), 0.0);
assert_eq!(std_dev(empty.iter().copied()), 0.0);
let single = [7.5_f64];
assert!((mean(single.iter().copied()) - 7.5).abs() < f64::EPSILON);
assert_eq!(std_dev(single.iter().copied()), 0.0);
}
#[test]
fn mean_std_skips_non_finite() {
let xs = [1.0_f64, f64::NAN, 3.0, f64::INFINITY, 5.0];
assert!((mean(xs.iter().copied()) - 3.0).abs() < 1e-9);
assert!((std_dev(xs.iter().copied()) - 2.0).abs() < 1e-9);
}
#[test]
fn mean_std_handles_negative_values() {
let xs = [-2.0_f64, -1.0, 0.0, 1.0, 2.0];
let m = mean(xs.iter().copied());
let s = std_dev(xs.iter().copied());
assert!(
(m - 0.0).abs() < 1e-9,
"mean of symmetric values should be 0, got {m}"
);
assert!((s - 1.58113883).abs() < 1e-6, "std dev mismatch, got {s}");
}
#[test]
fn mean_std_handles_large_values() {
let large = 1e150_f64;
let xs = [large, large * 2.0, large * 3.0];
let m = mean(xs.iter().copied());
let s = std_dev(xs.iter().copied());
assert!(
(m - large * 2.0).abs() / large < 1e-12,
"mean of large values"
);
assert!((s - large).abs() / large < 1e-12, "std dev of large values");
}
#[test]
fn mean_std_handles_subnormal_values() {
let tiny = f64::MIN_POSITIVE / 2.0; let xs = [tiny, tiny * 2.0, tiny * 3.0];
let m = mean(xs.iter().copied());
assert_eq!(m, 2.0 * tiny, "subnormals must be summed/averaged exactly");
}
#[test]
fn std_dev_two_values_bessel_corrected() {
let xs = [3.0_f64, 7.0];
let s = std_dev(xs.iter().copied());
assert!(
(s - 2.8284271247461903).abs() < 1e-9,
"Bessel-corrected std dev for two values"
);
}
#[test]
fn find_outliers_empty_input() {
let rows: Vec<GauntletRow> = vec![];
let outliers = find_outliers(&rows);
assert!(outliers.is_empty(), "empty input should yield no outliers");
}
#[test]
fn find_outliers_no_pass_rows() {
let r1 = make_row("s1", "t1", false, 10.0); let mut r2 = make_row("s2", "t2", true, 20.0);
r2.skipped = true; let rows = vec![r1, r2];
let outliers = find_outliers(&rows);
assert!(outliers.is_empty(), "no pass rows should yield no outliers");
}
#[test]
fn find_outliers_single_scenario_no_outlier() {
let r1 = make_row("only", "t1", true, 10.0);
let r2 = make_row("only", "t2", true, 12.0);
let r3 = make_row("only", "t3", true, 11.0);
let rows = vec![r1, r2, r3];
let outliers = find_outliers(&rows);
assert!(
outliers.is_empty(),
"single scenario cannot produce outliers"
);
}
#[test]
fn find_outliers_detects_clear_outlier() {
let mut rows = Vec::new();
for i in 0..5 {
rows.push(make_row("normal1", &format!("t{i}"), true, 5.0));
}
for i in 5..10 {
rows.push(make_row("normal2", &format!("t{i}"), true, 5.0));
}
rows.push(make_row("outlier", "t10", true, 100.0));
rows.push(make_row("outlier", "t11", true, 110.0));
let outliers = find_outliers(&rows);
assert!(!outliers.is_empty(), "should detect outlier scenario");
let spread_outlier = outliers.iter().find(|o| o.metric == "spread");
assert!(
spread_outlier.is_some(),
"should have spread metric outlier"
);
let outlier = spread_outlier.unwrap();
assert_eq!(outlier.scenario, "outlier");
assert!(
outlier.sigma > 2.0,
"sigma should exceed 2.0 threshold, got {}",
outlier.sigma
);
assert!(outlier.worst_topos.contains(&"t10".to_string()));
assert!(outlier.worst_topos.contains(&"t11".to_string()));
}
#[test]
fn find_outliers_threshold_is_strictly_greater() {
let rows = vec![
make_row("A", "t1", true, 0.0),
make_row("A", "t2", true, 0.0),
make_row("B", "t3", true, 10.0),
make_row("B", "t4", true, 10.0),
];
let outliers = find_outliers(&rows);
let spread_outliers: Vec<_> = outliers.iter().filter(|o| o.metric == "spread").collect();
assert!(
spread_outliers.is_empty(),
"no outlier when below threshold"
);
}
#[test]
fn find_outliers_flags_scenario_above_threshold() {
let mut rows: Vec<GauntletRow> = (0..10)
.map(|i| make_row(&format!("normal{i}"), "t", true, 10.0))
.collect();
rows.push(make_row("hot", "t", true, 100.0));
let outliers = find_outliers(&rows);
let spread: Vec<_> = outliers.iter().filter(|o| o.metric == "spread").collect();
assert!(
spread.iter().any(|o| o.scenario == "hot"),
"a scenario far above the 2-sigma threshold must be flagged as a spread outlier",
);
}
#[test]
fn find_outliers_skips_zero_std_metrics() {
let r1 = make_row("s1", "t1", true, 10.0);
let r2 = make_row("s2", "t2", true, 10.0);
let r3 = make_row("s3", "t3", true, 10.0);
let rows = vec![r1, r2, r3];
let outliers = find_outliers(&rows);
let spread_outliers: Vec<_> = outliers.iter().filter(|o| o.metric == "spread").collect();
assert!(
spread_outliers.is_empty(),
"zero std dev should skip metric"
);
}
#[test]
fn find_outliers_sorts_by_sigma_descending() {
let mut rows = Vec::new();
for i in 0..15 {
rows.push(make_row("normal1", &format!("t{i}"), true, 5.0));
}
for i in 15..30 {
rows.push(make_row("normal2", &format!("t{i}"), true, 5.0));
}
rows.push(make_row("outlier1", "t30", true, 100.0));
rows.push(make_row("outlier1", "t31", true, 110.0));
rows.push(make_row("outlier2", "t40", true, 140.0));
rows.push(make_row("outlier2", "t41", true, 150.0));
let outliers = find_outliers(&rows);
let spread_outliers: Vec<_> = outliers.iter().filter(|o| o.metric == "spread").collect();
assert!(
spread_outliers.len() >= 2,
"should have at least 2 spread outliers, got {}",
spread_outliers.len()
);
let first = &spread_outliers[0];
let second = &spread_outliers[1];
assert!(
first.sigma >= second.sigma,
"outliers should be sorted by sigma descending"
);
assert_eq!(
first.scenario, "outlier2",
"extreme outlier should be first"
);
}
#[test]
fn find_worst_topos_empty_when_no_matching_scenario() {
let r1 = make_row("s1", "t1", true, 10.0);
let rows = vec![r1];
let accessor: MetricAccessor = |r| r.spread;
let worst = find_worst_topos(&rows, "nonexistent", accessor, 5.0);
assert!(
worst.is_empty(),
"no matching scenario should yield empty vec"
);
}
#[test]
fn find_worst_topos_filters_by_threshold() {
let mut r1 = make_row("s1", "t1", true, 10.0);
r1.spread = 5.0;
let mut r2 = make_row("s1", "t2", true, 10.0);
r2.spread = 15.0;
let mut r3 = make_row("s1", "t3", true, 10.0);
r3.spread = 25.0;
let rows = vec![r1, r2, r3];
let accessor: MetricAccessor = |r| r.spread;
let worst = find_worst_topos(&rows, "s1", accessor, 15.0);
assert_eq!(worst.len(), 1, "only t3 should exceed threshold");
assert!(worst.contains(&"t3".to_string()));
assert!(
!worst.contains(&"t2".to_string()),
"t2 at threshold should not be included"
);
}
#[test]
fn find_worst_topos_includes_failed_rows() {
let mut r1 = make_row("s1", "t1", true, 10.0); r1.spread = 30.0;
let mut r2 = make_row("s1", "t2", false, 10.0); r2.spread = 40.0;
let rows = vec![r1, r2];
let accessor: MetricAccessor = |r| r.spread;
let worst = find_worst_topos(&rows, "s1", accessor, 20.0);
assert_eq!(worst.len(), 2);
assert!(worst.contains(&"t1".to_string()));
assert!(worst.contains(&"t2".to_string()));
}
#[test]
fn group_field_unknown_column_returns_none() {
let row = make_row("s1", "t1", true, 10.0);
assert!(group_field(&row, "scenario").is_some());
assert!(group_field(&row, "topology").is_some());
assert!(group_field(&row, "work_type").is_some());
assert!(group_field(&row, "invalid").is_none());
assert!(group_field(&row, "").is_none());
assert!(group_field(&row, "Spread").is_none()); }
#[test]
fn group_field_extracts_correct_dimension() {
let row = make_row("my_scenario", "my_topo", true, 10.0);
assert_eq!(group_field(&row, "scenario"), Some("my_scenario"));
assert_eq!(group_field(&row, "topology"), Some("my_topo"));
assert_eq!(group_field(&row, "work_type"), Some("SpinWait"));
}
#[test]
fn format_dimension_summary_computed_values() {
let mut r1 = make_row("slow", "tiny-1llc", false, 20.0);
r1.gap_ms = 200;
r1.imbalance_ratio = 2.5; r1.max_dsq_depth = 8; r1.stuck_count = 2.0; r1.fallback_count = 15; let r2 = make_row("fast", "tiny-1llc", true, 4.0);
let rows = vec![r1, r2];
let out = format_dimension_summary(&rows, "scenario");
let slow_pos = out.find("slow").unwrap();
let fast_pos = out.find("fast").unwrap();
assert!(
slow_pos < fast_pos,
"slow should sort before fast, got:\n{out}"
);
assert!(out.contains("0/1 passed"), "slow: 0/1 passed, got:\n{out}");
assert!(
out.contains("avg_spread=20.0%"),
"slow: avg_spread=20.0%, got:\n{out}"
);
assert!(
out.contains("avg_gap=200ms"),
"slow: avg_gap=200ms, got:\n{out}"
);
assert!(out.contains("imbal=2.5"), "slow: imbal=2.5, got:\n{out}");
assert!(out.contains("dsq=8"), "slow: dsq=8, got:\n{out}");
assert!(out.contains("stuck=2"), "slow: stuck=2, got:\n{out}");
assert!(
out.contains("fallback=15"),
"slow: fallback=15, got:\n{out}"
);
assert!(out.contains("1/1 passed"), "fast: 1/1 passed, got:\n{out}");
}
#[test]
fn format_dimension_summary_renders_inconclusive_bucket_distinctly() {
let mut r_pass = make_row("group_a", "t1", true, 5.0);
r_pass.skipped = false;
r_pass.inconclusive = false;
let mut r_inc = make_row("group_a", "t1", false, 5.0);
r_inc.skipped = false;
r_inc.inconclusive = true;
let mut r_fail = make_row("group_a", "t1", false, 5.0);
r_fail.skipped = false;
r_fail.inconclusive = false;
let rows = vec![r_pass, r_inc, r_fail];
let out = format_dimension_summary(&rows, "scenario");
assert!(
out.contains("1/3 passed"),
"expected '1/3 passed' for 1-pass-of-3: got:\n{out}"
);
assert!(
out.contains("1 inconclusive"),
"inconclusive row must NOT silently fold into the failed \
bucket; got:\n{out}"
);
assert!(
out.contains("1 failed"),
"real Fail row must render as 1 failed (not be hidden by \
the inconclusive subtraction); got:\n{out}"
);
assert!(
out.contains("0 skipped"),
"no Skip contributor; skipped bucket must be 0: got:\n{out}"
);
}
#[test]
fn analyze_rows_empty() {
assert!(analyze_rows(&[]).is_empty());
}
#[test]
fn analyze_rows_with_work_type_diversity() {
let mut rows = vec![
make_row("a", "t1", true, 5.0),
make_row("a", "t1", true, 6.0),
];
rows[0].work_type = "SpinWait".into();
rows[1].work_type = "Bursty".into();
let report = analyze_rows(&rows);
assert!(
report.contains("By work_type"),
"should show work_type section when diverse"
);
assert!(report.contains("SpinWait"), "should list SpinWait");
assert!(report.contains("Bursty"), "should list Bursty");
}
#[test]
fn analyze_rows_no_work_type_section_when_uniform() {
let rows = vec![
make_row("a", "t1", true, 5.0),
make_row("b", "t2", true, 8.0),
];
let report = analyze_rows(&rows);
assert!(
!report.contains("By work_type"),
"should not show work_type when uniform"
);
}