use std::collections::{BTreeMap, BTreeSet};
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use super::snapshot::{CheckRun, ClippySnapshot, ClippyWarning, RunSnapshot, TestId, TestSnapshot};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum GateKind {
ChecksPass,
NoRegression,
NoNewClippy,
NoTestGaming,
EnumerationIntegrity,
FileScope,
}
impl GateKind {
#[must_use]
pub fn label(self) -> &'static str {
match self {
GateKind::ChecksPass => "checks-pass",
GateKind::NoRegression => "no-regression",
GateKind::NoNewClippy => "no-new-clippy",
GateKind::NoTestGaming => "no-test-gaming",
GateKind::EnumerationIntegrity => "enumeration-integrity",
GateKind::FileScope => "file-scope",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
#[non_exhaustive]
pub enum Violation {
CheckFailed {
desc: String,
run: String,
exit_code: Option<i32>,
},
TestRegressed {
test: TestId,
},
NewClippyWarning {
warning: ClippyWarning,
},
TestCountDropped {
baseline: usize,
current: usize,
},
NewlyIgnoredTest {
test: TestId,
},
MissingBaselineTest {
test: TestId,
},
AssertionDensityRegressed {
file: PathBuf,
baseline: usize,
current: usize,
},
OutOfScopeFile {
file: PathBuf,
},
EnumerationShrank {
target: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct GateOutcome {
pub gate: GateKind,
pub passed: bool,
pub summary: String,
pub violations: Vec<Violation>,
}
impl GateOutcome {
fn pass(gate: GateKind, summary: impl Into<String>) -> Self {
Self {
gate,
passed: true,
summary: summary.into(),
violations: Vec::new(),
}
}
fn fail(gate: GateKind, summary: impl Into<String>, violations: Vec<Violation>) -> Self {
Self {
gate,
passed: false,
summary: summary.into(),
violations,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FloorVerdict {
pub gates: Vec<GateOutcome>,
}
impl FloorVerdict {
#[must_use]
pub fn passed(&self) -> bool {
self.gates.iter().all(|g| g.passed)
}
pub fn violations(&self) -> impl Iterator<Item = &Violation> {
self.gates.iter().flat_map(|g| g.violations.iter())
}
pub fn failed_gates(&self) -> impl Iterator<Item = &GateOutcome> {
self.gates.iter().filter(|g| !g.passed)
}
}
#[derive(Debug, Clone, Copy)]
pub struct FloorInputs<'a> {
pub baseline: &'a RunSnapshot,
pub current: &'a RunSnapshot,
pub check_results: &'a [CheckRun],
pub declared_files: &'a [PathBuf],
pub changed_files: &'a [PathBuf],
pub baseline_assertions: &'a BTreeMap<PathBuf, usize>,
pub current_assertions: &'a BTreeMap<PathBuf, usize>,
pub file_scope_slack: usize,
}
#[must_use]
pub fn evaluate_floor(inputs: &FloorInputs) -> FloorVerdict {
FloorVerdict {
gates: vec![
gate_checks_pass(inputs.check_results),
gate_no_regression(&inputs.baseline.tests, &inputs.current.tests),
gate_no_new_clippy(&inputs.baseline.clippy, &inputs.current.clippy),
gate_no_test_gaming(
&inputs.baseline.tests,
&inputs.current.tests,
inputs.baseline_assertions,
inputs.current_assertions,
),
gate_enumeration_superset(&inputs.baseline.tests, &inputs.current.tests),
gate_file_scope(
inputs.declared_files,
inputs.changed_files,
inputs.file_scope_slack,
),
],
}
}
#[must_use]
pub fn gate_checks_pass(results: &[CheckRun]) -> GateOutcome {
let failed: Vec<Violation> = results
.iter()
.filter(|r| !r.passed)
.map(|r| Violation::CheckFailed {
desc: r.desc.clone(),
run: r.run.clone(),
exit_code: r.exit_code,
})
.collect();
if failed.is_empty() {
GateOutcome::pass(
GateKind::ChecksPass,
format!("all {} check(s) passed", results.len()),
)
} else {
GateOutcome::fail(
GateKind::ChecksPass,
format!("{} of {} check(s) failed", failed.len(), results.len()),
failed,
)
}
}
#[must_use]
pub fn gate_no_regression(baseline: &TestSnapshot, current: &TestSnapshot) -> GateOutcome {
let regressed: Vec<Violation> = baseline
.passed
.intersection(¤t.failed)
.map(|t| Violation::TestRegressed { test: t.clone() })
.collect();
if regressed.is_empty() {
GateOutcome::pass(GateKind::NoRegression, "no baseline-passing test now fails")
} else {
GateOutcome::fail(
GateKind::NoRegression,
format!("{} baseline-passing test(s) now fail", regressed.len()),
regressed,
)
}
}
#[must_use]
pub fn gate_no_new_clippy(baseline: &ClippySnapshot, current: &ClippySnapshot) -> GateOutcome {
let new: Vec<Violation> = current
.warnings
.difference(&baseline.warnings)
.map(|w| Violation::NewClippyWarning { warning: w.clone() })
.collect();
if new.is_empty() {
GateOutcome::pass(GateKind::NoNewClippy, "no new clippy warnings vs baseline")
} else {
GateOutcome::fail(
GateKind::NoNewClippy,
format!("{} new clippy warning(s) vs baseline", new.len()),
new,
)
}
}
#[must_use]
pub fn gate_no_test_gaming(
baseline: &TestSnapshot,
current: &TestSnapshot,
baseline_assertions: &BTreeMap<PathBuf, usize>,
current_assertions: &BTreeMap<PathBuf, usize>,
) -> GateOutcome {
let mut violations = Vec::new();
let (base_total, cur_total) = (baseline.total(), current.total());
if cur_total < base_total {
violations.push(Violation::TestCountDropped {
baseline: base_total,
current: cur_total,
});
}
let baseline_run: BTreeSet<&TestId> = baseline.passed.union(&baseline.failed).collect();
for test in current.ignored.difference(&baseline.ignored) {
if baseline_run.contains(test) {
violations.push(Violation::NewlyIgnoredTest { test: test.clone() });
}
}
let current_ids = current.all_ids();
for test in &baseline.all_ids() {
if !current_ids.contains(test) {
violations.push(Violation::MissingBaselineTest { test: test.clone() });
}
}
for (file, &base_count) in baseline_assertions {
let cur_count = current_assertions.get(file).copied().unwrap_or(0);
if cur_count < base_count {
violations.push(Violation::AssertionDensityRegressed {
file: file.clone(),
baseline: base_count,
current: cur_count,
});
}
}
if violations.is_empty() {
GateOutcome::pass(
GateKind::NoTestGaming,
format!("no gaming signal ({cur_total} tests, assertion density held)"),
)
} else {
GateOutcome::fail(
GateKind::NoTestGaming,
format!("{} test-gaming signal(s)", violations.len()),
violations,
)
}
}
#[must_use]
pub fn gate_enumeration_superset(baseline: &TestSnapshot, current: &TestSnapshot) -> GateOutcome {
let missing: Vec<Violation> = baseline
.targets
.difference(¤t.targets)
.map(|t| Violation::EnumerationShrank { target: t.clone() })
.collect();
if missing.is_empty() {
GateOutcome::pass(
GateKind::EnumerationIntegrity,
format!(
"enumerated target set held ({} target(s), tip ⊇ baseline)",
current.targets.len()
),
)
} else {
GateOutcome::fail(
GateKind::EnumerationIntegrity,
format!(
"{} enumerated test target(s) vanished vs baseline",
missing.len()
),
missing,
)
}
}
#[must_use]
pub fn gate_file_scope(declared: &[PathBuf], changed: &[PathBuf], slack: usize) -> GateOutcome {
let declared_set: BTreeSet<&PathBuf> = declared.iter().collect();
let changed_set: BTreeSet<&PathBuf> = changed.iter().collect();
let out_of_scope: Vec<PathBuf> = changed_set
.into_iter()
.filter(|f| !declared_set.contains(*f))
.cloned()
.collect();
if out_of_scope.len() <= slack {
GateOutcome::pass(
GateKind::FileScope,
format!(
"{} changed file(s), {} out-of-scope within slack {}",
changed.len(),
out_of_scope.len(),
slack
),
)
} else {
let n = out_of_scope.len();
GateOutcome::fail(
GateKind::FileScope,
format!("{n} out-of-scope file(s) exceed slack {slack}"),
out_of_scope
.into_iter()
.map(|file| Violation::OutOfScopeFile { file })
.collect(),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn tset(items: &[&str]) -> BTreeSet<TestId> {
items.iter().map(|n| tid(n)).collect()
}
fn tid(name: &str) -> TestId {
TestId::new("pkg", "lib", "pkg", name)
}
fn cset(items: &[&str]) -> BTreeSet<ClippyWarning> {
items.iter().map(|s| cw(s)).collect()
}
fn cw(lint: &str) -> ClippyWarning {
ClippyWarning {
lint: lint.to_string(),
package: "pkg".into(),
file: "src/a.rs".into(),
message: "m".into(),
}
}
fn check(desc: &str, run: &str, passed: bool, exit: Option<i32>) -> CheckRun {
CheckRun {
desc: desc.to_string(),
run: run.to_string(),
cwd: None,
passed,
exit_code: exit,
stdout: String::new(),
stderr: String::new(),
}
}
fn paths(items: &[&str]) -> Vec<PathBuf> {
items.iter().map(PathBuf::from).collect()
}
#[test]
fn checks_pass_when_all_green() {
let g = gate_checks_pass(&[
check("a", "cargo test a", true, Some(0)),
check("b", "cargo test b", true, Some(0)),
]);
assert!(g.passed);
assert!(g.violations.is_empty());
}
#[test]
fn checks_fail_reports_each_failure() {
let g = gate_checks_pass(&[
check("a", "cargo test a", true, Some(0)),
check("b", "cargo test b", false, Some(101)),
]);
assert!(!g.passed);
assert_eq!(g.violations.len(), 1);
assert_eq!(
g.violations[0],
Violation::CheckFailed {
desc: "b".into(),
run: "cargo test b".into(),
exit_code: Some(101),
}
);
}
#[test]
fn empty_checks_pass_vacuously() {
assert!(gate_checks_pass(&[]).passed);
}
#[test]
fn no_regression_on_clean_pass() {
let base = TestSnapshot {
passed: tset(&["a", "b"]),
..Default::default()
};
let cur = TestSnapshot {
passed: tset(&["a", "b"]),
..Default::default()
};
assert!(gate_no_regression(&base, &cur).passed);
}
#[test]
fn regression_when_baseline_pass_now_fails() {
let base = TestSnapshot {
passed: tset(&["a", "b"]),
..Default::default()
};
let cur = TestSnapshot {
passed: tset(&["a"]),
failed: tset(&["b"]),
..Default::default()
};
let g = gate_no_regression(&base, &cur);
assert!(!g.passed);
assert_eq!(
g.violations,
vec![Violation::TestRegressed { test: tid("b") }]
);
}
#[test]
fn new_test_failing_is_not_a_regression() {
let base = TestSnapshot {
passed: tset(&["a"]),
..Default::default()
};
let cur = TestSnapshot {
passed: tset(&["a"]),
failed: tset(&["c"]),
..Default::default()
};
assert!(gate_no_regression(&base, &cur).passed);
}
#[test]
fn no_new_clippy_when_subset() {
let base = ClippySnapshot {
warnings: cset(&["w1", "w2"]),
};
let cur = ClippySnapshot {
warnings: cset(&["w1"]), };
assert!(gate_no_new_clippy(&base, &cur).passed);
}
#[test]
fn new_clippy_warning_fails() {
let base = ClippySnapshot {
warnings: cset(&["w1"]),
};
let cur = ClippySnapshot {
warnings: cset(&["w1", "w2"]),
};
let g = gate_no_new_clippy(&base, &cur);
assert!(!g.passed);
assert_eq!(
g.violations,
vec![Violation::NewClippyWarning { warning: cw("w2") }]
);
}
#[test]
fn no_gaming_on_clean_or_expanded_suite() {
let base = TestSnapshot {
passed: tset(&["a", "b"]),
..Default::default()
};
let cur = TestSnapshot {
passed: tset(&["a", "b", "c"]), ..Default::default()
};
let mut assertions = BTreeMap::new();
assertions.insert(PathBuf::from("src/a.rs"), 3);
let g = gate_no_test_gaming(&base, &cur, &assertions, &assertions);
assert!(g.passed, "{:?}", g.violations);
}
#[test]
fn detects_count_drop_and_vanished_test() {
let base = TestSnapshot {
passed: tset(&["a", "b"]),
..Default::default()
};
let cur = TestSnapshot {
passed: tset(&["a"]), ..Default::default()
};
let g = gate_no_test_gaming(&base, &cur, &BTreeMap::new(), &BTreeMap::new());
assert!(!g.passed);
assert!(g.violations.contains(&Violation::TestCountDropped {
baseline: 2,
current: 1
}));
assert!(g
.violations
.contains(&Violation::MissingBaselineTest { test: tid("b") }));
}
#[test]
fn detects_newly_ignored_baseline_pass() {
let base = TestSnapshot {
passed: tset(&["a", "b"]),
..Default::default()
};
let cur = TestSnapshot {
passed: tset(&["a"]),
ignored: tset(&["b"]), ..Default::default()
};
let g = gate_no_test_gaming(&base, &cur, &BTreeMap::new(), &BTreeMap::new());
assert!(!g.passed);
assert!(g
.violations
.contains(&Violation::NewlyIgnoredTest { test: tid("b") }));
assert!(!g
.violations
.iter()
.any(|v| matches!(v, Violation::TestCountDropped { .. })));
assert!(!g
.violations
.iter()
.any(|v| matches!(v, Violation::MissingBaselineTest { .. })));
}
#[test]
fn detects_was_failing_now_ignored() {
let base = TestSnapshot {
failed: tset(&["flaky"]),
..Default::default()
};
let cur = TestSnapshot {
ignored: tset(&["flaky"]),
..Default::default()
};
let g = gate_no_test_gaming(&base, &cur, &BTreeMap::new(), &BTreeMap::new());
assert!(g
.violations
.contains(&Violation::NewlyIgnoredTest { test: tid("flaky") }));
}
#[test]
fn brand_new_ignored_test_is_not_gaming() {
let base = TestSnapshot {
passed: tset(&["a"]),
..Default::default()
};
let cur = TestSnapshot {
passed: tset(&["a"]),
ignored: tset(&["new_wip"]),
..Default::default()
};
let g = gate_no_test_gaming(&base, &cur, &BTreeMap::new(), &BTreeMap::new());
assert!(!g
.violations
.iter()
.any(|v| matches!(v, Violation::NewlyIgnoredTest { .. })));
}
#[test]
fn detects_assertion_density_regression() {
let ts = TestSnapshot {
passed: tset(&["a"]),
..Default::default()
};
let mut base = BTreeMap::new();
base.insert(PathBuf::from("src/a.rs"), 5);
let mut cur = BTreeMap::new();
cur.insert(PathBuf::from("src/a.rs"), 2); let g = gate_no_test_gaming(&ts, &ts, &base, &cur);
assert!(!g.passed);
assert!(g
.violations
.contains(&Violation::AssertionDensityRegressed {
file: PathBuf::from("src/a.rs"),
baseline: 5,
current: 2,
}));
}
#[test]
fn added_assertions_and_new_files_do_not_regress() {
let ts = TestSnapshot {
passed: tset(&["a"]),
..Default::default()
};
let mut base = BTreeMap::new();
base.insert(PathBuf::from("src/a.rs"), 2);
let mut cur = BTreeMap::new();
cur.insert(PathBuf::from("src/a.rs"), 4); cur.insert(PathBuf::from("src/new.rs"), 1); assert!(gate_no_test_gaming(&ts, &ts, &base, &cur).passed);
}
#[test]
fn file_scope_passes_within_declared() {
let declared = paths(&["src/a.rs", "src/a_test.rs"]);
let changed = paths(&["src/a.rs"]);
assert!(gate_file_scope(&declared, &changed, 0).passed);
}
#[test]
fn file_scope_fails_out_of_scope_beyond_slack() {
let declared = paths(&["src/a.rs"]);
let changed = paths(&["src/a.rs", "src/secret.rs", "Cargo.toml"]);
let g = gate_file_scope(&declared, &changed, 0);
assert!(!g.passed);
assert_eq!(g.violations.len(), 2);
assert!(g.violations.contains(&Violation::OutOfScopeFile {
file: PathBuf::from("src/secret.rs")
}));
assert!(g.violations.contains(&Violation::OutOfScopeFile {
file: PathBuf::from("Cargo.toml")
}));
}
#[test]
fn file_scope_dedups_repeated_paths() {
let declared = paths(&["src/a.rs"]);
let changed = paths(&["src/dup.rs", "src/dup.rs"]);
let g = gate_file_scope(&declared, &changed, 1);
assert!(g.passed, "one distinct out-of-scope file fits slack 1");
assert!(g.violations.is_empty());
}
#[test]
fn file_scope_tolerates_within_slack() {
let declared = paths(&["src/a.rs"]);
let changed = paths(&["src/a.rs", "src/extra.rs"]);
let g = gate_file_scope(&declared, &changed, 1);
assert!(g.passed);
assert!(g.violations.is_empty());
}
fn tsnap_targets(items: &[&str]) -> TestSnapshot {
TestSnapshot {
targets: items.iter().map(ToString::to_string).collect(),
..Default::default()
}
}
#[test]
fn enumeration_superset_passes_when_targets_held_or_grew() {
let base = tsnap_targets(&["p/lib/p", "p/test/e2e"]);
assert!(gate_enumeration_superset(&base, &base).passed);
let grew = tsnap_targets(&["p/lib/p", "p/test/e2e", "p/test/new"]);
let g = gate_enumeration_superset(&base, &grew);
assert!(g.passed, "{g:#?}");
assert!(g.violations.is_empty());
}
#[test]
fn enumeration_shrink_fails_closed() {
let base = tsnap_targets(&["p/lib/p", "p/test/e2e"]);
let tip = tsnap_targets(&["p/lib/p"]);
let g = gate_enumeration_superset(&base, &tip);
assert!(!g.passed);
assert_eq!(
g.violations,
vec![Violation::EnumerationShrank {
target: "p/test/e2e".to_string()
}]
);
let base_run = RunSnapshot {
tests: base,
..Default::default()
};
let tip_run = RunSnapshot {
tests: tip,
..Default::default()
};
let declared = paths(&["src/a.rs"]);
let assertions = BTreeMap::new();
let inputs = FloorInputs {
baseline: &base_run,
current: &tip_run,
check_results: &[check("c", "cargo test", true, Some(0))],
declared_files: &declared,
changed_files: &declared,
baseline_assertions: &assertions,
current_assertions: &assertions,
file_scope_slack: 0,
};
let verdict = evaluate_floor(&inputs);
assert!(!verdict.passed());
assert!(verdict
.failed_gates()
.any(|g| g.gate == GateKind::EnumerationIntegrity));
}
#[test]
fn evaluate_floor_is_green_on_a_clean_run() {
let base = RunSnapshot {
tests: TestSnapshot {
passed: tset(&["a"]),
..Default::default()
},
clippy: ClippySnapshot {
warnings: cset(&["w1"]),
},
coverage: None,
};
let cur = base.clone();
let declared = paths(&["src/a.rs"]);
let changed = paths(&["src/a.rs"]);
let assertions = BTreeMap::new();
let inputs = FloorInputs {
baseline: &base,
current: &cur,
check_results: &[check("c", "cargo test", true, Some(0))],
declared_files: &declared,
changed_files: &changed,
baseline_assertions: &assertions,
current_assertions: &assertions,
file_scope_slack: 0,
};
let verdict = evaluate_floor(&inputs);
assert!(verdict.passed(), "{verdict:#?}");
assert_eq!(verdict.gates.len(), 6);
assert_eq!(verdict.violations().count(), 0);
}
#[test]
fn evaluate_floor_aggregates_multiple_gate_failures() {
let base = RunSnapshot {
tests: TestSnapshot {
passed: tset(&["a", "b"]),
..Default::default()
},
clippy: ClippySnapshot::default(),
coverage: None,
};
let cur = RunSnapshot {
tests: TestSnapshot {
passed: tset(&["a"]),
failed: tset(&["b"]), ..Default::default()
},
clippy: ClippySnapshot {
warnings: cset(&["new-warn"]), },
coverage: None,
};
let declared = paths(&["src/a.rs"]);
let changed = paths(&["src/a.rs", "src/out.rs"]); let assertions = BTreeMap::new();
let inputs = FloorInputs {
baseline: &base,
current: &cur,
check_results: &[check("c", "cargo test", false, Some(101))], declared_files: &declared,
changed_files: &changed,
baseline_assertions: &assertions,
current_assertions: &assertions,
file_scope_slack: 0,
};
let verdict = evaluate_floor(&inputs);
assert!(!verdict.passed());
assert_eq!(verdict.failed_gates().count(), 4);
assert!(
verdict
.gates
.iter()
.find(|g| g.gate == GateKind::NoTestGaming)
.unwrap()
.passed
);
}
}