use std::fs::{File, FileTimes};
use std::os::unix::fs::PermissionsExt;
use std::time::{Duration, SystemTime};
use super::*;
use crate::core::build_lease::slots::HolderRecord;
struct Fixture {
_tmp: tempfile::TempDir,
pool: PathBuf,
home: PathBuf,
store: SlotDir,
}
fn fixture() -> Fixture {
let tmp = tempfile::tempdir().expect("tempdir");
let base = std::fs::canonicalize(tmp.path()).expect("canonical tempdir");
let pool = base.join("pool");
std::fs::create_dir_all(&pool).expect("pool root");
let home = base.join("home");
std::fs::create_dir_all(&home).expect("home");
let store = SlotDir::at(base.join("store")).expect("lease store");
Fixture {
_tmp: tmp,
pool,
home,
store,
}
}
fn run(f: &Fixture, measure: &mut dyn FnMut(&Path) -> Option<f32>) -> SweepOutcome {
sweep(&f.pool, &f.home, &f.store, 85, measure, &|| false)
}
fn make_slot(pool: &Path, repo: &str, index: u32, age: Duration) -> PathBuf {
let slot = pool.join(repo).join(format!("slot-{index}"));
let debug = slot.join("debug");
std::fs::create_dir_all(&debug).expect("slot tree");
std::fs::write(debug.join("artifact"), b"x").expect("artifact");
let when = FileTimes::new().set_modified(SystemTime::now() - age);
for path in [debug.join("artifact"), debug.clone(), slot.clone()] {
File::open(&path)
.and_then(|f| f.set_times(when))
.expect("set mtime");
}
slot
}
const HOUR: Duration = Duration::from_secs(3600);
fn per_slot(base: f32) -> impl FnMut(&Path) -> Option<f32> {
move |root: &Path| {
let left = list_pool_slots(root).expect("list").len();
Some(base + 5.0 * left as f32)
}
}
fn swept(outcome: SweepOutcome) -> EvictReport {
match outcome {
SweepOutcome::Swept(report) => report,
other => panic!("expected a sweep, got {other:?}"),
}
}
#[test]
fn the_threshold_defaults_below_the_worktree_guard() {
assert_eq!(effective_evict_pct(None, 90), DEFAULT_EVICT_PCT);
assert_eq!(effective_evict_pct(Some(70), 90), 70);
}
#[test]
fn a_threshold_at_or_over_the_guard_is_held_below_it() {
assert_eq!(effective_evict_pct(Some(95), 90), 89);
assert_eq!(effective_evict_pct(Some(90), 90), 89);
assert_eq!(effective_evict_pct(None, 80), 79);
assert_eq!(
effective_evict_pct(None, 1),
1,
"never 0: 0 would evict always"
);
}
#[test]
fn an_out_of_range_threshold_uses_the_default() {
assert_eq!(effective_evict_pct(Some(0), 90), DEFAULT_EVICT_PCT);
assert_eq!(effective_evict_pct(Some(100), 90), DEFAULT_EVICT_PCT);
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn slots_are_listed_oldest_first() {
let f = fixture();
let newest = make_slot(&f.pool, "o/r", 0, HOUR);
let oldest = make_slot(&f.pool, "o2/r2", 3, 3 * HOUR);
let middle = make_slot(&f.pool, "o/r", 1, 2 * HOUR);
std::fs::create_dir_all(f.pool.join("o/r/.slot-0.seeding.1.2")).expect("staging");
std::fs::create_dir_all(f.pool.join("o/r/slot-x")).expect("odd name");
std::fs::write(f.pool.join("o/r/slot-9"), b"").expect("file");
let listed: Vec<(PathBuf, u32)> = list_pool_slots(&f.pool)
.expect("list")
.into_iter()
.map(|s| (s.path, s.index))
.collect();
assert_eq!(listed, vec![(oldest, 3), (middle, 1), (newest, 0)]);
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_missing_pool_is_empty() {
let f = fixture();
let missing = f.pool.join("absent");
assert!(
list_pool_slots(&missing)
.expect("absent is empty")
.is_empty()
);
let outcome = sweep(
&missing,
&f.home,
&f.store,
85,
&mut |_: &Path| Some(99.0),
&|| false,
);
assert_eq!(outcome, SweepOutcome::NoPool);
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn an_unmeasurable_volume_evicts_nothing() {
let f = fixture();
let slot = make_slot(&f.pool, "o/r", 0, HOUR);
let outcome = run(&f, &mut |_: &Path| None);
assert_eq!(outcome, SweepOutcome::Unmeasurable);
assert!(slot.is_dir());
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn an_under_threshold_volume_evicts_nothing() {
let f = fixture();
let slot = make_slot(&f.pool, "o/r", 0, HOUR);
let outcome = run(&f, &mut |_: &Path| Some(84.9));
assert_eq!(outcome, SweepOutcome::UnderThreshold(84.9));
assert!(slot.is_dir());
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn an_unlistable_pool_evicts_nothing() {
let f = fixture();
make_slot(&f.pool, "o/r", 0, HOUR);
std::fs::set_permissions(&f.pool, std::fs::Permissions::from_mode(0o000)).expect("chmod");
let outcome = run(&f, &mut |_: &Path| Some(99.0));
std::fs::set_permissions(&f.pool, std::fs::Permissions::from_mode(0o755)).expect("chmod");
assert!(
matches!(outcome, SweepOutcome::Unlistable(_)),
"{outcome:?}"
);
assert!(f.pool.join("o/r/slot-0").is_dir());
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn an_over_threshold_volume_evicts_oldest_first_until_below() {
let f = fixture();
let oldest = make_slot(&f.pool, "o/r", 0, 3 * HOUR);
let middle = make_slot(&f.pool, "p/q", 1, 2 * HOUR);
let newest = make_slot(&f.pool, "o/r", 2, HOUR);
let report = swept(run(&f, &mut per_slot(75.0)));
assert_eq!(report.evicted, vec![oldest.clone(), middle.clone()]);
assert!(report.spared.is_empty() && !report.failed());
assert_eq!(report.usage_after, Some(80.0));
assert!(!oldest.exists() && !middle.exists());
assert!(newest.is_dir(), "the newest slot keeps its warm cache");
let leftovers: Vec<_> = std::fs::read_dir(f.pool.join("o/r"))
.expect("repo dir")
.flatten()
.filter(|e| e.file_name().to_string_lossy().contains(".evicting."))
.collect();
assert!(leftovers.is_empty(), "{leftovers:?}");
assert!(f.store.try_acquire(0).expect("lock").is_some());
}
fn sweep_with_busy_slot_0(f: &Fixture) -> (Vec<(PathBuf, Spared)>, bool) {
let busy = f.pool.join("o/r/slot-0");
let idle = make_slot(&f.pool, "o/r", 1, HOUR);
let report = swept(run(f, &mut |_: &Path| Some(95.0)));
assert!(busy.is_dir(), "the busy slot was evicted");
assert!(!report.failed(), "{:?}", report.failed);
(report.spared, report.evicted == vec![idle])
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_held_lease_spares_its_slot() {
let f = fixture();
make_slot(&f.pool, "o/r", 0, 3 * HOUR);
let _held = f.store.try_acquire(0).expect("lock").expect("free");
let (spared, idle_evicted) = sweep_with_busy_slot_0(&f);
assert_eq!(spared, vec![(f.pool.join("o/r/slot-0"), Spared::LeaseHeld)]);
assert!(idle_evicted);
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn an_orphaned_build_spares_its_slot() {
let f = fixture();
make_slot(&f.pool, "o/r", 0, 3 * HOUR);
let mut child = std::process::Command::new("sleep")
.arg("30")
.spawn()
.expect("stand-in build");
let mut record = HolderRecord::new(0, "cargo test", "/repo");
record.child_pid = Some(child.id());
let mut guard = f.store.try_acquire(0).expect("lock").expect("free");
guard.write_record(&record).expect("record");
guard.release_keeping_record();
let (spared, idle_evicted) = sweep_with_busy_slot_0(&f);
let _ = child.kill();
let _ = child.wait();
assert_eq!(
spared,
vec![(f.pool.join("o/r/slot-0"), Spared::OrphanedBuild(child.id()))]
);
assert!(idle_evicted);
assert!(
!std::fs::read(f.store.path().join("slot-0.lock"))
.expect("read")
.is_empty()
);
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_corrupt_slot_record_spares_its_slot() {
let f = fixture();
make_slot(&f.pool, "o/r", 0, 3 * HOUR);
std::fs::write(f.store.path().join("slot-0.lock"), b"not a record").expect("corrupt");
let (spared, idle_evicted) = sweep_with_busy_slot_0(&f);
assert!(
matches!(spared.as_slice(), [(_, Spared::Unknown(e))] if e.contains("corrupt")),
"{spared:?}"
);
assert!(idle_evicted);
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_broken_slot_file_spares_its_slot() {
let f = fixture();
make_slot(&f.pool, "o/r", 0, 3 * HOUR);
std::fs::create_dir_all(f.store.path().join("slot-0.lock")).expect("a directory, not a file");
let (spared, idle_evicted) = sweep_with_busy_slot_0(&f);
assert!(
matches!(spared.as_slice(), [(_, Spared::Unknown(e))] if e.contains("Is a directory")),
"{spared:?}"
);
assert!(idle_evicted);
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_held_cargo_lock_spares_its_slot() {
let f = fixture();
let slot = make_slot(&f.pool, "o/r", 0, 3 * HOUR);
let lock = File::create(slot.join("debug/.cargo-lock")).expect("lock file");
lock.try_lock().expect("hold cargo's lock");
let (spared, idle_evicted) = sweep_with_busy_slot_0(&f);
drop(lock);
assert_eq!(spared, vec![(slot, Spared::CargoLockHeld)]);
assert!(idle_evicted);
assert!(f.store.try_acquire(0).expect("lock").is_some());
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_live_seed_staging_spares_its_slot() {
let f = fixture();
let slot = make_slot(&f.pool, "o/r", 0, 3 * HOUR);
let staging = f
.pool
.join(format!("o/r/.slot-0.seeding.{}.1", std::process::id()));
std::fs::create_dir_all(&staging).expect("staging tree");
let (spared, idle_evicted) = sweep_with_busy_slot_0(&f);
assert_eq!(spared, vec![(slot, Spared::Seeding)]);
assert!(idle_evicted);
assert!(
staging.is_dir(),
"a live seed's staging tree is not touched"
);
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_failed_removal_is_reported_and_left_for_the_next_sweep() {
let f = fixture();
let slot = make_slot(&f.pool, "o/r", 0, HOUR);
let foreign = f.pool.join("o/r/.slot-7.evicting.1.1");
std::fs::create_dir_all(&foreign).expect("foreign tree");
let debug = slot.join("debug");
std::fs::set_permissions(&debug, std::fs::Permissions::from_mode(0o555)).expect("chmod");
let report = swept(run(&f, &mut |_: &Path| Some(95.0)));
assert!(report.failed(), "{report:?}");
assert!(report.evicted.is_empty());
let doomed: Vec<PathBuf> = std::fs::read_dir(f.pool.join("o/r"))
.expect("repo dir")
.flatten()
.map(|e| e.path())
.filter(|p| p != &foreign && p.to_string_lossy().contains(".evicting."))
.collect();
assert_eq!(doomed.len(), 1, "{doomed:?}");
assert!(
!slot.exists(),
"the slot name is gone even though the delete failed"
);
std::fs::set_permissions(
doomed[0].join("debug"),
std::fs::Permissions::from_mode(0o755),
)
.expect("chmod back");
let again = swept(run(&f, &mut |_: &Path| Some(95.0)));
assert_eq!(again.leftovers_removed, 1);
assert!(!again.failed(), "{again:?}");
assert!(!doomed[0].exists());
assert!(foreign.is_dir(), "a live process's tree is left to it");
}
fn refusal(f: &Fixture, root: &Path, home: &Path, reading: Option<f32>) -> String {
let mut reads = 0;
let outcome = sweep(
root,
home,
&f.store,
85,
&mut |_: &Path| {
reads += 1;
reading
},
&|| false,
);
assert_eq!(reads, 0, "a refused root was measured: {outcome:?}");
match outcome {
SweepOutcome::Refused(why) => why,
other => panic!("expected a refusal, got {other:?}"),
}
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn the_filesystem_root_is_refused() {
let f = fixture();
let why = refusal(&f, Path::new("/"), &f.home, None);
assert!(why.contains("filesystem root"), "{why}");
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn the_home_directory_is_refused() {
let f = fixture();
let slot = make_slot(&f.home, "o/r", 0, HOUR);
let why = refusal(&f, &f.home, &f.home, Some(99.0));
assert!(why.contains("home directory or an ancestor"), "{why}");
assert!(slot.is_dir(), "nothing under home is evicted");
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn an_ancestor_of_home_is_refused() {
let f = fixture();
let slot = make_slot(&f.pool, "o/r", 0, HOUR);
let home = f.pool.join("users/me");
std::fs::create_dir_all(&home).expect("home under the pool");
let why = refusal(&f, &f.pool, &home, Some(99.0));
assert!(why.contains("home directory or an ancestor"), "{why}");
assert!(slot.is_dir(), "nothing above home is evicted");
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn an_unresolvable_root_is_refused() {
let f = fixture();
let base = f.pool.parent().expect("tempdir").to_path_buf();
std::os::unix::fs::symlink(base.join("loop-b"), base.join("loop-a")).expect("link a");
std::os::unix::fs::symlink(base.join("loop-a"), base.join("loop-b")).expect("link b");
let why = refusal(&f, &base.join("loop-a"), &f.home, Some(99.0));
assert!(why.contains("does not resolve"), "{why}");
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn an_unresolvable_home_is_refused() {
let f = fixture();
let slot = make_slot(&f.pool, "o/r", 0, HOUR);
let home = f.pool.parent().expect("tempdir").join("no-such-home");
let why = refusal(&f, &f.pool, &home, Some(99.0));
assert!(
why.contains("home") && why.contains("does not resolve"),
"{why}"
);
assert!(slot.is_dir());
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_symlink_to_home_is_refused() {
let f = fixture();
let slot = make_slot(&f.home, "o/r", 0, HOUR);
let link = f.pool.join("link-to-home");
std::os::unix::fs::symlink(&f.home, &link).expect("symlink");
let why = refusal(&f, &link, &f.home, Some(99.0));
assert!(why.contains("home directory or an ancestor"), "{why}");
assert!(slot.is_dir());
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_symlink_to_the_filesystem_root_is_refused() {
let f = fixture();
let link = f.pool.join("link-to-root");
std::os::unix::fs::symlink("/", &link).expect("symlink");
let why = refusal(&f, &link, &f.home, None);
assert!(why.contains("filesystem root"), "{why}");
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_dot_suffixed_home_is_refused() {
let f = fixture();
let slot = make_slot(&f.home, "o/r", 0, HOUR);
let why = refusal(&f, &f.home.join("."), &f.home, Some(99.0));
assert!(why.contains("home directory or an ancestor"), "{why}");
assert!(slot.is_dir());
}
#[test]
fn an_inode_alias_of_an_ancestor_of_home_is_refused() {
let f = fixture();
let base = f.pool.parent().expect("tempdir").to_path_buf();
let shared: FileId = (7, 42);
let id_of = |p: &Path| -> Option<FileId> {
if p == f.pool || p == base {
Some(shared)
} else {
Some((7, 1))
}
};
match check_pool_root_with(&f.pool, &f.home, &id_of) {
RootCheck::Refused(why) => assert!(why.contains("same directory"), "{why}"),
_ => panic!("an inode alias of an ancestor of home was not refused"),
}
let skip = |p: &Path| -> Option<FileId> { (p == f.pool).then_some(shared) };
assert!(matches!(
check_pool_root_with(&f.pool, &f.home, &skip),
RootCheck::Safe(_)
));
}
#[test]
fn a_root_with_unreadable_metadata_is_refused() {
let f = fixture();
let blind = |_: &Path| -> Option<FileId> { None };
match check_pool_root_with(&f.pool, &f.home, &blind) {
RootCheck::Refused(why) => assert!(why.contains("unreadable metadata"), "{why}"),
_ => panic!("a root with unreadable metadata was not refused"),
}
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_case_variant_of_home_is_refused() {
let f = fixture();
let base = f.pool.parent().expect("tempdir").to_path_buf();
let variant = base.join("HOME");
if !variant.is_dir() {
eprintln!("skipped: the temp volume is case-sensitive, so HOME is not home");
return;
}
let slot = make_slot(&f.home, "o/r", 0, HOUR);
let why = refusal(&f, &variant, &f.home, Some(99.0));
assert!(
why.contains("home") || why.contains("same directory"),
"{why}"
);
assert!(slot.is_dir());
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn the_sweep_walks_the_canonical_root() {
let f = fixture();
let slot = make_slot(&f.pool, "o/r", 0, HOUR);
let link = f.home.join("pool-link");
std::os::unix::fs::symlink(&f.pool, &link).expect("symlink");
let mut seen = Vec::new();
let mut gone = false;
let report = swept(sweep(
&link,
&f.home,
&f.store,
85,
&mut |p: &Path| {
seen.push(p.to_path_buf());
let reading = if gone { 10.0 } else { 99.0 };
gone = !slot.exists();
Some(reading)
},
&|| false,
));
assert_eq!(report.evicted, vec![slot]);
assert!(
seen.iter().all(|p| p == &f.pool),
"measured a non-canonical path: {seen:?}"
);
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_cancel_below_the_threshold_is_not_reported() {
let f = fixture();
let slot = make_slot(&f.pool, "o/r", 0, HOUR);
let mut reads = 0;
let report = swept(sweep(
&f.pool,
&f.home,
&f.store,
85,
&mut |_: &Path| {
reads += 1;
Some(if reads == 1 { 99.0 } else { 10.0 })
},
&|| true,
));
assert!(!report.cancelled, "{report:?}");
assert!(report.evicted.is_empty() && slot.is_dir());
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_cancelled_sweep_evicts_no_further_slots() {
let f = fixture();
let oldest = make_slot(&f.pool, "o/r", 0, 3 * HOUR);
let middle = make_slot(&f.pool, "o/r", 1, 2 * HOUR);
let newest = make_slot(&f.pool, "p/q", 2, HOUR);
let probe = oldest.clone();
let cancelled = move || !probe.exists();
let report = swept(sweep(
&f.pool,
&f.home,
&f.store,
85,
&mut |_: &Path| Some(99.0),
&cancelled,
));
assert_eq!(report.evicted, vec![oldest]);
assert!(report.cancelled && !report.failed(), "{report:?}");
assert!(middle.is_dir() && newest.is_dir());
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_sweep_cancelled_before_it_starts_touches_nothing() {
let f = fixture();
let slot = make_slot(&f.pool, "o/r", 0, HOUR);
let leftover = f
.pool
.join(format!("o/r/.slot-3.evicting.{}.1", std::process::id()));
std::fs::create_dir_all(&leftover).expect("leftover tree");
let report = swept(sweep(
&f.pool,
&f.home,
&f.store,
85,
&mut |_: &Path| Some(99.0),
&|| true,
));
assert!(report.cancelled, "{report:?}");
assert_eq!(report.leftovers_removed, 0);
assert!(report.evicted.is_empty());
assert!(leftover.is_dir() && slot.is_dir());
}
#[test]
fn an_unreadable_repo_dir_reads_as_live_staging() {
let tmp = tempfile::tempdir().expect("tempdir");
let repo = tmp.path().join("o/r");
std::fs::create_dir_all(&repo).expect("repo dir");
std::fs::set_permissions(&repo, std::fs::Permissions::from_mode(0o000)).expect("chmod");
let live = live_staging(&repo, 0);
std::fs::set_permissions(&repo, std::fs::Permissions::from_mode(0o755)).expect("chmod");
assert!(live, "an unlistable parent must spare the slot");
}
#[test]
#[serial_test::serial(build_slot_fds)]
fn a_failed_rename_is_reported_and_keeps_the_slot() {
let f = fixture();
let slot = make_slot(&f.pool, "o/r", 0, HOUR);
let repo = f.pool.join("o/r");
std::fs::set_permissions(&repo, std::fs::Permissions::from_mode(0o555)).expect("chmod");
let report = swept(run(&f, &mut |_: &Path| Some(95.0)));
std::fs::set_permissions(&repo, std::fs::Permissions::from_mode(0o755)).expect("chmod");
assert!(
matches!(report.failed.as_slice(), [(p, e)] if p == &slot && e.starts_with("rename to")),
"{report:?}"
);
assert!(report.evicted.is_empty());
assert!(slot.is_dir());
assert!(f.store.try_acquire(0).expect("lock").is_some());
}