use std::collections::BTreeMap;
use std::path::Path;
use crate::gate_stamp::{Note, Run, RunOutcome, NOTES_REF};
pub fn now() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct History {
pub runs: Vec<(String, Run)>,
}
impl History {
pub fn is_empty(&self) -> bool {
self.runs.is_empty()
}
fn within(&self, now: u64, window_days: u64) -> Vec<(&str, &Run)> {
let floor = if window_days == 0 {
0
} else {
now.saturating_sub(window_days.saturating_mul(86_400))
};
let mut runs: Vec<(&str, &Run)> = self
.runs
.iter()
.filter(|(_, r)| r.at >= floor)
.map(|(fp, r)| (fp.as_str(), r))
.collect();
runs.sort_by(|a, b| a.1.at.cmp(&b.1.at).then_with(|| a.1.gate.cmp(&b.1.gate)));
runs
}
}
pub fn history_in(repo: &Path) -> History {
let Some(list) = crate::git::stdout_in(repo, &["notes", "--ref", NOTES_REF, "list"]) else {
return History::default();
};
let mut keyed: BTreeMap<String, String> = BTreeMap::new();
let mut stdin = String::new();
for line in list.lines() {
let mut t = line.split_whitespace();
if let (Some(blob), Some(object)) = (t.next(), t.next()) {
keyed.insert(blob.to_string(), object.to_string());
stdin.push_str(blob);
stdin.push('\n');
}
}
if keyed.is_empty() {
return History::default();
}
let Some(batch) = crate::git::stdout_piped_in(repo, &["cat-file", "--batch"], stdin.as_bytes())
else {
return History::default();
};
let mut history = History::default();
for (blob, body) in parse_batch(&batch) {
let Some(fingerprint) = keyed.get(&blob) else {
continue;
};
for run in Note::parse(&body).runs {
history.runs.push((fingerprint.clone(), run));
}
}
history
}
fn parse_batch(out: &str) -> Vec<(String, String)> {
let mut found = Vec::new();
let mut current: Option<(String, Vec<&str>)> = None;
for line in out.lines() {
let mut t = line.split_whitespace();
let (oid, kind, size) = (t.next(), t.next(), t.next());
let is_header = matches!((oid, kind, size), (Some(o), Some("blob"), Some(s))
if o.len() >= 40
&& o.chars().all(|c| c.is_ascii_hexdigit())
&& s.parse::<u64>().is_ok()
&& t.next().is_none());
if is_header {
if let Some((oid, body)) = current.take() {
found.push((oid, body.join("\n")));
}
current = Some((oid.unwrap_or_default().to_string(), Vec::new()));
continue;
}
if let Some((_, body)) = current.as_mut() {
body.push(line);
}
}
if let Some((oid, body)) = current.take() {
found.push((oid, body.join("\n")));
}
found
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Thresholds {
pub window_days: u64,
pub min_runs: usize,
pub noop_ratio_percent: u64,
pub noop_median_secs: u64,
pub fast_pass_ms: u64,
pub stale_runs: usize,
pub stale_days: u64,
}
impl Default for Thresholds {
fn default() -> Self {
Thresholds {
window_days: 90,
min_runs: 5,
noop_ratio_percent: 10,
noop_median_secs: 30,
fast_pass_ms: 1_000,
stale_runs: 10,
stale_days: 30,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FlagKind {
NoOpSuspect,
Flaky,
Stale,
}
impl FlagKind {
pub fn as_str(self) -> &'static str {
match self {
FlagKind::NoOpSuspect => "no-op suspect",
FlagKind::Flaky => "flaky",
FlagKind::Stale => "stale",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Flag {
pub kind: FlagKind,
pub why: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GateReport {
pub gate: String,
pub runs: usize,
pub passed: usize,
pub failed: usize,
pub unavailable: usize,
pub median_ms: u64,
pub last_ms: u64,
pub last_at: u64,
pub last_outcome: Option<RunOutcome>,
pub fingerprints: usize,
pub missed: usize,
pub flags: Vec<Flag>,
pub abstained: Option<String>,
}
impl GateReport {
pub fn last_age_days(&self, now: u64) -> u64 {
now.saturating_sub(self.last_at) / 86_400
}
}
pub fn summarise(history: &History, now: u64, t: &Thresholds) -> Vec<GateReport> {
let runs = history.within(now, t.window_days);
let mut by_gate: BTreeMap<&str, Vec<(&str, &Run)>> = BTreeMap::new();
for &(fp, run) in &runs {
by_gate
.entry(run.gate.as_str())
.or_default()
.push((fp, run));
}
by_gate
.into_iter()
.map(|(gate, mine)| one_gate(gate, &mine, &runs, now, t))
.collect()
}
fn one_gate(
gate: &str,
mine: &[(&str, &Run)],
all: &[(&str, &Run)],
now: u64,
t: &Thresholds,
) -> GateReport {
let verdicts: Vec<(&str, &Run)> = mine
.iter()
.filter(|(_, r)| r.outcome.is_verdict())
.copied()
.collect();
let passed = verdicts
.iter()
.filter(|(_, r)| r.outcome == RunOutcome::Passed)
.count();
let failed = verdicts.len() - passed;
let unavailable = mine
.iter()
.filter(|(_, r)| r.outcome == RunOutcome::Unavailable)
.count();
let median_ms = median(&verdicts.iter().map(|(_, r)| r.ms).collect::<Vec<_>>());
let last = mine.last().copied();
let mut fingerprints: Vec<&str> = mine.iter().map(|(fp, _)| *fp).collect();
fingerprints.sort_unstable();
fingerprints.dedup();
let last_at = last.map(|(_, r)| r.at).unwrap_or(0);
let mut later: Vec<&str> = all
.iter()
.filter(|(_, r)| r.at > last_at && r.gate != gate)
.map(|(fp, _)| *fp)
.collect();
later.sort_unstable();
later.dedup();
let missed = later
.iter()
.filter(|fp| !fingerprints.contains(*fp))
.count();
let mut report = GateReport {
gate: gate.to_string(),
runs: mine.len(),
passed,
failed,
unavailable,
median_ms,
last_ms: last.map(|(_, r)| r.ms).unwrap_or(0),
last_at,
last_outcome: last.map(|(_, r)| r.outcome),
fingerprints: fingerprints.len(),
missed,
flags: Vec::new(),
abstained: None,
};
if let Some(flag) = stale_flag(&report, all, now, t) {
report.flags.push(flag);
}
if verdicts.len() < t.min_runs {
report.abstained = Some(format!(
"insufficient history ({} verdict{})",
verdicts.len(),
if verdicts.len() == 1 { "" } else { "s" }
));
return report;
}
if let Some(flag) = noop_flag(&verdicts, median_ms, t) {
report.flags.push(flag);
}
if let Some(flag) = flaky_flag(gate, &verdicts) {
report.flags.push(flag);
}
report
}
fn noop_flag(verdicts: &[(&str, &Run)], median_ms: u64, t: &Thresholds) -> Option<Flag> {
let (_, last) = *verdicts.last()?;
if median_ms >= t.noop_median_secs.saturating_mul(1_000)
&& last.ms.saturating_mul(100) < median_ms.saturating_mul(t.noop_ratio_percent)
{
return Some(Flag {
kind: FlagKind::NoOpSuspect,
why: format!(
"last run {} against a median of {} ({}% of it, threshold {}%)",
duration(last.ms),
duration(median_ms),
last.ms.saturating_mul(100) / median_ms.max(1),
t.noop_ratio_percent
),
});
}
if last.outcome == RunOutcome::Passed
&& last.ms < t.fast_pass_ms
&& verdicts[..verdicts.len() - 1]
.iter()
.all(|(_, r)| r.ms >= t.fast_pass_ms)
{
return Some(Flag {
kind: FlagKind::NoOpSuspect,
why: format!(
"passed in {}, and none of the {} earlier runs ever finished under {}",
duration(last.ms),
verdicts.len() - 1,
duration(t.fast_pass_ms)
),
});
}
None
}
fn flaky_flag(gate: &str, verdicts: &[(&str, &Run)]) -> Option<Flag> {
let mut by_fp: BTreeMap<&str, (usize, usize)> = BTreeMap::new();
for &(fp, run) in verdicts {
let e = by_fp.entry(fp).or_insert((0, 0));
if run.outcome == RunOutcome::Passed {
e.0 += 1;
} else {
e.1 += 1;
}
}
let split: Vec<(&str, usize, usize)> = by_fp
.into_iter()
.filter(|(_, (pass, fail))| *pass > 0 && *fail > 0)
.map(|(fp, (pass, fail))| (fp, pass, fail))
.collect();
let (fp, pass, fail) = *split.first()?;
Some(Flag {
kind: FlagKind::Flaky,
why: format!(
"{gate} both passed ({pass}) and failed ({fail}) on tree {}{}",
short(fp),
if split.len() > 1 {
format!(", and on {} other tree(s)", split.len() - 1)
} else {
String::new()
}
),
})
}
fn stale_flag(report: &GateReport, all: &[(&str, &Run)], now: u64, t: &Thresholds) -> Option<Flag> {
if report.runs == 0 {
return None;
}
let newest_other = all
.iter()
.filter(|(_, r)| r.gate != report.gate)
.map(|(_, r)| r.at)
.max()?;
if newest_other <= report.last_at {
return None;
}
if report.missed >= t.stale_runs {
return Some(Flag {
kind: FlagKind::Stale,
why: format!(
"{} later tree(s) were judged by other gates and not by this one (threshold {})",
report.missed, t.stale_runs
),
});
}
let age = report.last_age_days(now);
if age >= t.stale_days {
return Some(Flag {
kind: FlagKind::Stale,
why: format!(
"last ran {age} days ago (threshold {}), while another gate ran {} days ago",
t.stale_days,
now.saturating_sub(newest_other) / 86_400
),
});
}
None
}
pub fn order_by_evidence(
gates: &[String],
history: &History,
now: u64,
window_days: u64,
) -> Vec<usize> {
let runs = history.within(now, window_days);
let mut stats: BTreeMap<&str, (u64, u64, Vec<u64>)> = BTreeMap::new();
for (_, run) in &runs {
if !run.outcome.is_verdict() {
continue;
}
let e = stats.entry(run.gate.as_str()).or_insert((0, 0, Vec::new()));
e.0 += 1;
if run.outcome == RunOutcome::Failed {
e.1 += 1;
}
e.2.push(run.ms);
}
let weight = |name: &String| -> Option<(u64, u64, u64)> {
let (runs, fails, ms) = stats.get(name.as_str())?;
(*fails > 0).then(|| (*fails, *runs, median(ms).max(1)))
};
let mut order: Vec<usize> = (0..gates.len()).collect();
order.sort_by(|&a, &b| {
match (weight(&gates[a]), weight(&gates[b])) {
(Some((fa, ra, ma)), Some((fb, rb, mb))) => {
let left = u128::from(fa) * u128::from(rb) * u128::from(mb);
let right = u128::from(fb) * u128::from(ra) * u128::from(ma);
right.cmp(&left).then(a.cmp(&b))
}
(Some(_), None) => std::cmp::Ordering::Less,
(None, Some(_)) => std::cmp::Ordering::Greater,
(None, None) => a.cmp(&b),
}
});
order
}
fn median(values: &[u64]) -> u64 {
if values.is_empty() {
return 0;
}
let mut v = values.to_vec();
v.sort_unstable();
let mid = v.len() / 2;
if v.len() % 2 == 1 {
v[mid]
} else {
v[mid - 1]
}
}
pub fn duration(ms: u64) -> String {
if ms < 1_000 {
return format!("{ms} ms");
}
if ms < 60_000 {
return format!("{}.{} s", ms / 1000, (ms % 1000) / 100);
}
format!("{} m {:02} s", ms / 60_000, (ms % 60_000) / 1000)
}
fn short(oid: &str) -> &str {
oid.get(..8).unwrap_or(oid)
}
#[cfg(test)]
mod tests {
use super::*;
fn run(at: u64, gate: &str, outcome: RunOutcome, ms: u64) -> Run {
Run {
at,
gate: gate.to_string(),
outcome,
ms,
}
}
fn history(rows: &[(&str, Run)]) -> History {
History {
runs: rows
.iter()
.map(|(fp, r)| ((*fp).to_string(), r.clone()))
.collect(),
}
}
fn owned_history(rows: Vec<(String, Run)>) -> History {
History { runs: rows }
}
const NOW: u64 = 1_800_000_000;
const DAY: u64 = 86_400;
fn of<'a>(reports: &'a [GateReport], gate: &str) -> &'a GateReport {
reports
.iter()
.find(|r| r.gate == gate)
.unwrap_or_else(|| panic!("no report for {gate}: {reports:?}"))
}
fn kinds(r: &GateReport) -> Vec<FlagKind> {
r.flags.iter().map(|f| f.kind).collect()
}
#[test]
fn a_suite_that_collapsed_to_milliseconds_is_a_no_op_suspect() {
let mut rows: Vec<(&str, Run)> = (0..8)
.map(|i| {
(
"tree0",
run(
NOW - (10 - i) * DAY,
"pre-push-cargo-test",
RunOutcome::Passed,
600_000,
),
)
})
.collect();
rows.push((
"tree9",
run(NOW - DAY, "pre-push-cargo-test", RunOutcome::Passed, 400),
));
let reports = summarise(&history(&rows), NOW, &Thresholds::default());
let r = of(&reports, "pre-push-cargo-test");
assert_eq!(kinds(r), vec![FlagKind::NoOpSuspect]);
assert!(
r.flags[0].why.contains("400 ms") && r.flags[0].why.contains("threshold 10%"),
"the flag must carry the comparison it made: {:?}",
r.flags[0].why
);
}
#[test]
fn a_gate_that_never_once_took_real_time_is_also_suspect() {
let rows: Vec<(&str, Run)> = (0..6)
.map(|i| {
(
"tree0",
run(
NOW - (7 - i) * DAY,
"pre-push-audit-python",
RunOutcome::Passed,
if i == 5 { 30 } else { 4_000 },
),
)
})
.collect();
let reports = summarise(&history(&rows), NOW, &Thresholds::default());
assert_eq!(
kinds(of(&reports, "pre-push-audit-python")),
vec![FlagKind::NoOpSuspect]
);
}
#[test]
fn one_fingerprint_with_two_answers_is_flaky() {
let rows = [
(
"treeA",
run(NOW - 6 * DAY, "pre-push-pytest", RunOutcome::Passed, 90_000),
),
(
"treeA",
run(NOW - 5 * DAY, "pre-push-pytest", RunOutcome::Failed, 88_000),
),
(
"treeA",
run(NOW - 4 * DAY, "pre-push-pytest", RunOutcome::Passed, 91_000),
),
(
"treeB",
run(NOW - 3 * DAY, "pre-push-pytest", RunOutcome::Passed, 92_000),
),
(
"treeB",
run(NOW - 2 * DAY, "pre-push-pytest", RunOutcome::Passed, 90_500),
),
];
let r = summarise(&history(&rows), NOW, &Thresholds::default());
let r = of(&r, "pre-push-pytest");
assert_eq!(kinds(r), vec![FlagKind::Flaky]);
assert!(r.flags[0].why.contains("treeA"), "{:?}", r.flags[0].why);
}
#[test]
fn a_gate_left_behind_by_its_neighbours_is_stale() {
let mut rows = vec![(
"tree0".to_string(),
run(
NOW - 40 * DAY,
"pre-push-go-test",
RunOutcome::Passed,
5_000,
),
)];
for i in 0..12u64 {
rows.push((
format!("tree{}", i + 1),
run(
NOW - (12 - i) * DAY,
"pre-push-cargo-test",
RunOutcome::Passed,
5_000,
),
));
}
let reports = summarise(&owned_history(rows), NOW, &Thresholds::default());
let stale = of(&reports, "pre-push-go-test");
assert!(kinds(stale).contains(&FlagKind::Stale), "{:?}", stale.flags);
assert!(
of(&reports, "pre-push-cargo-test").flags.is_empty(),
"the gate that kept running is not stale"
);
}
#[test]
fn too_little_history_abstains_out_loud() {
let rows = [
(
"tree0",
run(
NOW - 2 * DAY,
"pre-push-cargo-test",
RunOutcome::Passed,
600_000,
),
),
(
"tree1",
run(NOW - DAY, "pre-push-cargo-test", RunOutcome::Passed, 300),
),
];
let reports = summarise(&history(&rows), NOW, &Thresholds::default());
let r = of(&reports, "pre-push-cargo-test");
assert_eq!(
r.abstained.as_deref(),
Some("insufficient history (2 verdicts)")
);
assert!(
r.flags.is_empty(),
"a two-run history must not raise a flag: {:?}",
r.flags
);
assert_eq!((r.runs, r.passed), (2, 2), "the counts are still reported");
}
#[test]
fn an_unavailable_run_is_never_counted_as_a_pass() {
let rows = [
(
"tree0",
run(
NOW - 2 * DAY,
"pre-push-audit-go",
RunOutcome::Unavailable,
200,
),
),
(
"tree1",
run(NOW - DAY, "pre-push-audit-go", RunOutcome::Unavailable, 210),
),
];
let reports = summarise(&history(&rows), NOW, &Thresholds::default());
let r = of(&reports, "pre-push-audit-go");
assert_eq!((r.passed, r.failed, r.unavailable), (0, 0, 2));
}
#[test]
fn the_window_excludes_what_is_older_than_it() {
let rows = [
(
"tree0",
run(
NOW - 200 * DAY,
"pre-push-cargo-test",
RunOutcome::Failed,
500,
),
),
(
"tree1",
run(NOW - DAY, "pre-push-cargo-test", RunOutcome::Passed, 500),
),
];
let reports = summarise(&history(&rows), NOW, &Thresholds::default());
assert_eq!(of(&reports, "pre-push-cargo-test").runs, 1);
}
#[test]
fn ordering_prefers_failures_per_second_not_failure_rate() {
let gates: Vec<String> = ["suite", "audit"].iter().map(|s| s.to_string()).collect();
let mut rows: Vec<(&str, Run)> = Vec::new();
for i in 0..6u64 {
rows.push((
"tree0",
run(
NOW - (10 - i) * DAY,
"suite",
if i < 2 {
RunOutcome::Failed
} else {
RunOutcome::Passed
},
1_200_000,
),
));
rows.push((
"tree0",
run(
NOW - (10 - i) * DAY,
"audit",
if i < 1 {
RunOutcome::Failed
} else {
RunOutcome::Passed
},
5_000,
),
));
}
let order = order_by_evidence(&gates, &history(&rows), NOW, 90);
assert_eq!(order, vec![1, 0], "the cheap audit runs first");
}
#[test]
fn with_no_history_the_declared_order_is_kept() {
let gates: Vec<String> = ["a", "b", "c"].iter().map(|s| s.to_string()).collect();
assert_eq!(
order_by_evidence(&gates, &History::default(), NOW, 90),
vec![0, 1, 2]
);
}
#[test]
fn only_gates_that_actually_failed_are_promoted() {
let gates: Vec<String> = ["a", "b", "c"].iter().map(|s| s.to_string()).collect();
let rows = [
("t", run(NOW - 3 * DAY, "a", RunOutcome::Passed, 1_000)),
("t", run(NOW - 3 * DAY, "b", RunOutcome::Passed, 1_000)),
("t", run(NOW - 2 * DAY, "c", RunOutcome::Failed, 1_000)),
("t", run(NOW - DAY, "c", RunOutcome::Passed, 1_000)),
];
assert_eq!(
order_by_evidence(&gates, &history(&rows), NOW, 90),
vec![2, 0, 1]
);
}
#[test]
fn ordering_is_a_permutation_and_never_drops_a_gate() {
let gates: Vec<String> = ["a", "b", "c", "d"].iter().map(|s| s.to_string()).collect();
let rows = [
("t", run(NOW - DAY, "b", RunOutcome::Failed, 10)),
("t", run(NOW - DAY, "d", RunOutcome::Failed, 10_000)),
];
let mut order = order_by_evidence(&gates, &history(&rows), NOW, 90);
assert_eq!(order.len(), gates.len());
order.sort_unstable();
assert_eq!(order, vec![0, 1, 2, 3]);
}
#[test]
fn cat_file_batch_output_splits_into_bodies() {
let oid = "0123456789abcdef0123456789abcdef01234567";
let other = "89abcdef0123456789abcdef0123456789abcdef";
let out = format!(
"{oid} blob 42\namont-gate-v1 pre-push-cargo-test\nrun 10 pre-push-cargo-test pass 5\n\
{other} blob 14\namont-gate-v1\n"
);
let got = parse_batch(&out);
assert_eq!(got.len(), 2);
assert_eq!(got[0].0, oid);
assert!(got[0].1.contains("run 10"));
assert_eq!(got[1].0, other);
}
#[test]
fn durations_read_like_durations() {
assert_eq!(duration(400), "400 ms");
assert_eq!(duration(5_400), "5.4 s");
assert_eq!(duration(662_000), "11 m 02 s");
}
}