1mod copies;
4pub mod reconcile;
5
6pub(crate) fn has_archived_seats(home: &Path, state: &RunState) -> Result<bool> {
7 let artifacts = home.join("runs").join(&state.id).join("artifacts");
8 if !artifacts.try_exists()? {
9 return Ok(false);
10 }
11 ensure!(
12 plain_dir(&artifacts) && artifacts.canonicalize()? == artifacts,
13 "noncanonical artifacts"
14 );
15 for (i, entry) in std::fs::read_dir(artifacts)?.enumerate() {
16 ensure!(i < 4096, "eviction record inspection limit exceeded");
17 let name = entry?.file_name();
18 let name = name.to_string_lossy();
19 if name.starts_with("eviction-") && name.ends_with(".json") {
20 return Ok(true);
21 }
22 }
23 Ok(false)
24}
25
26pub(crate) async fn restore_execution_seats(home: &Path, state: &RunState) -> Result<()> {
29 copies::restore_execution_seats(home, state).await
30}
31
32pub async fn source_diff(evidence: &Path) -> Result<Vec<u8>> {
34 crate::fold_evidence::validate_snapshot(evidence).await?;
35 Ok(std::fs::read(evidence.join("change.patch"))?)
36}
37
38pub async fn restore_source(evidence: &Path, destination: &Path) -> Result<()> {
40 crate::fold_evidence::restore_snapshot(evidence, destination).await
41}
42
43pub fn seat_inspections(home: &Path, id: &str) -> Result<Vec<String>> {
45 ensure!(crate::run::is_run_id(id), "invalid run id");
46 copies::seat_inspections(home, id)
47}
48
49pub async fn seat_evidence(home: &Path, id: &str) -> Result<Vec<String>> {
51 copies::seat_evidence(home, id).await
52}
53
54pub async fn seat_heads(
56 home: &Path,
57 id: &str,
58) -> Result<std::collections::BTreeMap<String, String>> {
59 copies::seat_heads(home, id).await
60}
61
62pub async fn seat_diff(home: &Path, id: &str, head: &str) -> Result<Vec<u8>> {
64 copies::seat_diff(home, id, head).await
65}
66
67pub async fn restore_seat(home: &Path, id: &str, head: &str, destination: &Path) -> Result<()> {
69 copies::restore_seat(home, id, head, destination).await
70}
71
72use crate::{
73 git,
74 proc::Quiet as _,
75 run::{Liveness, RunState},
76};
77use anyhow::{Context, Result, ensure};
78use jiff::Timestamp;
79use serde::{Deserialize, Serialize};
80use std::{
81 fs::{File, OpenOptions},
82 path::{Path, PathBuf},
83};
84
85#[derive(Debug)]
89pub(crate) struct OutputLease {
90 _file: std::sync::Arc<OutputFile>,
91 _activity: bool,
92}
93
94#[derive(Debug)]
95struct OutputFile(File);
96
97impl std::ops::Deref for OutputFile {
98 type Target = File;
99 fn deref(&self) -> &File {
100 &self.0
101 }
102}
103
104#[cfg(unix)]
105impl Drop for OutputFile {
106 fn drop(&mut self) {
107 use std::os::fd::AsRawFd;
108 unsafe {
110 libc::flock(self.0.as_raw_fd(), libc::LOCK_UN);
111 }
112 }
113}
114
115impl OutputLease {
116 #[cfg(unix)]
117 pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
118 use std::os::fd::AsRawFd;
119 self._file.as_raw_fd()
120 }
121
122 pub(crate) fn pre_clean(dir: &Path) -> Result<Self> {
124 let file = OpenOptions::new()
125 .create(true)
126 .truncate(false)
127 .read(true)
128 .write(true)
129 .open(dir.join("magi-pre-clean.lock"))?;
130 crate::pr_head::try_lock_file(&file)
131 .context("pre-clean already running; retry after descendants terminate")?;
132 Ok(Self {
133 _file: std::sync::Arc::new(OutputFile(file)),
134 _activity: false,
135 })
136 }
137
138 pub(crate) fn duplicate(&self) -> Self {
139 Self {
140 _file: std::sync::Arc::clone(&self._file),
141 _activity: false,
142 }
143 }
144
145 pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
146 if !worktree.join(".git").is_file() {
147 return Ok(None);
148 }
149 Self::take(worktree, false).map(Some)
150 }
151 pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
152 use std::hash::{Hash, Hasher};
153 let mut hash = std::collections::hash_map::DefaultHasher::new();
154 let mut normalized = cache.to_owned();
155 let mut missing = Vec::new();
156 while !normalized.exists() {
157 let Some(name) = normalized.file_name() else {
158 break;
159 };
160 missing.push(name.to_owned());
161 if !normalized.pop() {
162 break;
163 }
164 }
165 normalized = normalized.canonicalize().unwrap_or(normalized);
166 for part in missing.into_iter().rev() {
167 normalized.push(part);
168 }
169 normalized.hash(&mut hash);
170 let dir = home.join("cache-leases");
171 std::fs::create_dir_all(&dir)?;
172 Self::lock(
173 &dir.join(format!("outputs-{:x}.lock", hash.finish())),
174 exclusive,
175 )
176 }
177 #[cfg(unix)]
181 pub(crate) fn admit(home: &Path, paths: &[PathBuf], min_free: u64) -> Result<Vec<Self>> {
182 Self::admit_with(home, paths, min_free, crate::disk::free_bytes)
183 }
184
185 #[cfg(unix)]
186 pub(crate) fn admit_with(
187 home: &Path,
188 paths: &[PathBuf],
189 min_free: u64,
190 free_bytes: impl Fn(&Path) -> Result<u64>,
191 ) -> Result<Vec<Self>> {
192 use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
193 if min_free == 0 {
194 return Ok(Vec::new());
195 }
196 const RESERVE: u64 = 1024 * 1024 * 1024;
197 std::fs::create_dir_all(home)?;
198 let dir = home.canonicalize()?.join("disk-admission");
199 std::fs::create_dir_all(&dir)?;
200 ensure!(
201 plain_dir(&dir) && dir.canonicalize()? == dir,
202 "disk admission directory is not canonical"
203 );
204 let mut devices = std::collections::BTreeSet::new();
205 let mut reservations = Vec::new();
206 for path in paths {
207 let probe = path
208 .ancestors()
209 .find(|p| p.exists())
210 .context("disk admission path missing")?;
211 let device = std::fs::metadata(probe)?.dev();
212 if !devices.insert(device) {
213 continue;
214 }
215 let open = |path: &Path| -> Result<File> {
216 Ok(OpenOptions::new()
217 .read(true)
218 .write(true)
219 .create(true)
220 .truncate(false)
221 .custom_flags(libc::O_NOFOLLOW)
222 .open(path)?)
223 };
224 let admission = Self {
228 _file: std::sync::Arc::new(OutputFile(open(&dir.join(format!("{device}.lock")))?)),
229 _activity: false,
230 };
231 let started = std::time::Instant::now();
232 loop {
233 match crate::pr_head::try_lock_file(&admission._file) {
234 Ok(()) => break,
235 Err(error)
236 if matches!(
237 error.kind(),
238 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
239 ) && started.elapsed() < std::time::Duration::from_secs(1) =>
240 {
241 std::thread::sleep(std::time::Duration::from_millis(5));
242 }
243 Err(error) => {
244 return Err(error)
245 .context("disk admission busy or unavailable; retry without dispatch");
246 }
247 }
248 }
249 let mut active = 0u64;
250 let mut available = None;
251 for slot in 0..1024 {
252 let file = dir.join(format!("{device}-{slot}.reserve"));
253 if !file.exists() {
254 if available.is_none() {
255 available = Some(file);
256 }
257 continue;
258 }
259 let file = open(&file)?;
260 match crate::pr_head::try_lock_file(&file) {
261 Ok(()) if available.is_none() => {
262 available = Some(dir.join(format!("{device}-{slot}.reserve")))
263 }
264 Ok(()) => {}
265 Err(_) => active = active.saturating_add(RESERVE),
266 }
267 }
268 let free = free_bytes(probe)?;
269 let required = min_free.saturating_add(active).saturating_add(RESERVE);
270 ensure!(
271 free >= required,
272 "disk admission blocked: {free} bytes free; floor {min_free}, active writer reservations {active}, next writer reservation {RESERVE} bytes (estimates, not quotas)"
273 );
274 let file = open(&available.context("disk admission reservation slots exhausted")?)?;
275 crate::pr_head::try_lock_file(&file)?;
276 reservations.push(Self {
277 _file: std::sync::Arc::new(OutputFile(file)),
278 _activity: false,
279 });
280 }
281 Ok(reservations)
282 }
283
284 #[cfg(not(unix))]
285 pub(crate) fn admit(_home: &Path, _paths: &[PathBuf], _min_free: u64) -> Result<Vec<Self>> {
286 Ok(Vec::new())
288 }
289
290 pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
291 let out = std::process::Command::new("git")
292 .args(["rev-parse", "--absolute-git-dir"])
293 .current_dir(worktree)
294 .quiet()
295 .output()?;
296 ensure!(
297 out.status.success(),
298 "cannot identify output lease directory"
299 );
300 let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
301 Self::lock(&dir.join("magi-output.lock"), exclusive)
302 }
303 fn lock(path: &Path, exclusive: bool) -> Result<Self> {
304 let file = OpenOptions::new()
305 .create(true)
306 .truncate(false)
307 .read(true)
308 .write(true)
309 .open(path)
310 .with_context(|| format!("output lease: open {}", path.display()))?;
311 #[cfg(unix)]
312 {
313 use std::os::fd::AsRawFd;
314 let mode = if exclusive {
315 libc::LOCK_EX
316 } else {
317 libc::LOCK_SH
318 };
319 ensure!(
321 unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
322 "output lease: outputs are in use or being reclaimed: {}",
323 path.display()
324 );
325 }
326 #[cfg(not(unix))]
327 let _ = exclusive;
328 if !exclusive {
329 file.set_len(1)?;
331 file.set_modified(std::time::SystemTime::now())
332 .with_context(|| format!("record output activity {}", path.display()))?;
333 }
334 Ok(Self {
335 _file: std::sync::Arc::new(OutputFile(file)),
336 _activity: !exclusive,
337 })
338 }
339}
340#[cfg(unix)]
341impl Drop for OutputLease {
342 fn drop(&mut self) {
343 if self._activity
344 && let Err(error) = self._file.set_modified(std::time::SystemTime::now())
345 {
346 let _ = self._file.set_len(2);
349 tracing::warn!("could not record output lease release activity: {error}");
350 }
351 }
352}
353
354#[derive(Default, Debug, Serialize, Deserialize)]
355#[serde(default)]
356pub(crate) struct Report {
357 pub checked_at: Option<Timestamp>,
358 pub limit_bytes: u64,
359 pub reclaim_budget_bytes: u64,
360 pub measured_bytes: u64,
362 pub freed_bytes: u64,
364 pub measured_allocated_file_bytes: Option<u64>,
365 pub proven_reclaimable_bytes: u64,
367 pub proven_reclaimable_allocated_bytes: Option<u64>,
370 pub protected_or_unproven_bytes: u64,
371 pub estimated_released_allocated_bytes: Option<u64>,
374 pub observed_home_free_change_bytes: Option<i64>,
377 pub retired: usize,
378 pub evicted_copies: usize,
379 pub source_scan: Option<serde_json::Value>,
380 pub removed_empty_roots: usize,
381 pub source_unlinked_logical_bytes: u64,
382 pub source_unlinked_allocated_bytes: Option<u64>,
383 pub source_evidence_added_logical_bytes: u64,
384 pub source_evidence_added_allocated_bytes: Option<u64>,
385 pub retained: Vec<String>,
386 pub failures: Vec<String>,
387}
388
389#[cfg(unix)]
390fn allocated_inventory(
391 root: &Path,
392 seen: &mut std::collections::BTreeSet<(u64, u64)>,
393) -> Result<u64> {
394 use std::os::unix::fs::MetadataExt;
395 ensure!(
396 plain_dir(root) && root.canonicalize()? == root,
397 "allocated inventory root is not canonical"
398 );
399 let mut dirs = vec![root.to_owned()];
400 let mut bytes = 0u64;
401 while let Some(dir) = dirs.pop() {
402 for entry in std::fs::read_dir(dir)? {
403 let path = entry?.path();
404 let metadata = std::fs::symlink_metadata(&path)?;
405 if metadata.is_dir() {
406 dirs.push(path);
407 } else if metadata.is_file() && seen.insert((metadata.dev(), metadata.ino())) {
408 bytes = bytes.saturating_add(metadata.blocks().saturating_mul(512));
409 }
410 }
411 }
412 Ok(bytes)
413}
414
415#[derive(Default)]
416struct ReleasedAllocation {
417 #[cfg(unix)]
418 files: std::collections::BTreeMap<PathBuf, (u64, u64, u64, u64)>,
419 #[cfg(unix)]
420 removed: std::collections::BTreeMap<(u64, u64), u64>,
421}
422
423impl ReleasedAllocation {
424 fn of(paths: &[PathBuf]) -> Result<Self> {
425 #[cfg(unix)]
426 {
427 use std::os::unix::fs::MetadataExt;
428 let mut files = std::collections::BTreeMap::new();
429 for path in paths {
430 let m = std::fs::symlink_metadata(path)?;
431 ensure!(m.is_file(), "allocation candidate is not regular");
432 files.insert(
433 path.clone(),
434 (m.dev(), m.ino(), m.nlink(), m.blocks().saturating_mul(512)),
435 );
436 }
437 Ok(Self {
438 files,
439 removed: Default::default(),
440 })
441 }
442 #[cfg(not(unix))]
443 {
444 let _ = paths;
445 Ok(Self::default())
446 }
447 }
448 fn reclaimable_allocated_bytes(&self) -> Option<u64> {
449 #[cfg(unix)]
450 {
451 let mut inodes = std::collections::BTreeMap::new();
452 for &(device, inode, links, blocks) in self.files.values() {
453 let entry = inodes
454 .entry((device, inode))
455 .or_insert((0u64, links, blocks));
456 entry.0 += 1;
457 }
458 Some(
459 inodes
460 .values()
461 .filter(|(count, links, _)| count == links)
462 .fold(0u64, |sum, (_, _, blocks)| sum.saturating_add(*blocks)),
463 )
464 }
465 #[cfg(not(unix))]
466 {
467 None
468 }
469 }
470
471 fn unlinked(&mut self, path: &Path) -> Option<u64> {
472 #[cfg(unix)]
473 {
474 let (device, inode, links, blocks) = self.files[path];
475 let removed = self.removed.entry((device, inode)).or_default();
476 *removed += 1;
477 Some(if *removed == links { blocks } else { 0 })
478 }
479 #[cfg(not(unix))]
480 {
481 let _ = path;
482 None
483 }
484 }
485}
486
487fn plain_dir(path: &Path) -> bool {
488 std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
489}
490
491fn run_paths_missing(state: &RunState) -> Result<bool> {
494 for path in [state.worktree_root(), state.repo.clone()] {
495 match std::fs::metadata(&path) {
496 Ok(_) => {}
497 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
498 Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
499 }
500 }
501 Ok(false)
502}
503
504pub(crate) async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
506 let root = state.worktree_root();
507 if !plain_dir(&root) {
508 return Ok(Vec::new());
509 }
510 let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
511 registered_worktrees(state, &list)
512}
513
514fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
515 let root = state.worktree_root();
516 if !plain_dir(&root) {
517 return Ok(Vec::new());
518 }
519 let root = root.canonicalize()?;
520 let mut paths = Vec::new();
521 for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
522 let path = PathBuf::from(path);
523 #[cfg(windows)]
524 let path = if plain_dir(&path) {
525 git::canonical_output_path(path.to_str().context("non UTF-8 registration")?)?
526 } else {
527 continue;
528 };
529 if plain_dir(&path)
530 && path.parent() == Some(root.as_path())
531 && path.canonicalize()? == path
532 && path.join(".git").is_file()
533 {
534 paths.push(path);
535 }
536 }
537 Ok(paths)
538}
539
540async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
543 validate_cargo_target(worktree, false).await
544}
545
546async fn validate_cargo_target(worktree: &Path, allow_missing_tag: bool) -> Result<PathBuf> {
547 let target = worktree.join("target");
548 ensure!(plain_dir(&target), "target absent or not a plain directory");
549 ensure!(target.canonicalize()? == target, "target path changed");
550 ensure!(
551 git::git(worktree, &["ls-files", "--", "target"])
552 .await?
553 .is_empty(),
554 "tracked target contents"
555 );
556 ensure!(
557 git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
558 .await?
559 .ok(),
560 "target is not ignored"
561 );
562 cache_tag(&target, allow_missing_tag)?;
563 let path = target.join(".rustc_info.json");
564 plain_file(&path, "rustc identity")?;
565 let info: serde_json::Value = serde_json::from_slice(
566 &std::fs::read(&path).with_context(|| format!("read rustc identity {}", path.display()))?,
567 )
568 .with_context(|| format!("parse rustc identity {}", path.display()))?;
569 ensure!(
570 info.get("rustc_fingerprint").is_some(),
571 "unknown Cargo identity {}",
572 path.display()
573 );
574 if allow_missing_tag && !target.join("CACHEDIR.TAG").exists() {
575 ensure!(
576 info.get("rustc_fingerprint")
577 .is_some_and(serde_json::Value::is_u64)
578 && info
579 .get("outputs")
580 .and_then(serde_json::Value::as_object)
581 .is_some_and(|outputs| outputs.values().any(|output| {
582 output.get("success").and_then(serde_json::Value::as_bool) == Some(true)
583 && output
584 .get("stdout")
585 .and_then(serde_json::Value::as_str)
586 .is_some_and(|stdout| stdout.starts_with("rustc "))
587 && output
588 .get("stderr")
589 .is_some_and(serde_json::Value::is_string)
590 }))
591 && info
592 .get("successes")
593 .is_some_and(serde_json::Value::is_object),
594 "missing-tag rustc identity structure invalid: {}",
595 path.display()
596 );
597 let mut profiles = 0;
598 for profile in ["debug", "release"] {
599 let dir = target.join(profile);
600 if !dir.exists() {
601 continue;
602 }
603 profile_output_dirs(&dir, true)?;
604 plain_file(&dir.join(".cargo-lock"), "missing-tag Cargo lock")?;
605 profiles += 1;
606 }
607 ensure!(
608 profiles > 0,
609 "missing-tag proof: no Cargo profile in {}",
610 target.display()
611 );
612 }
613 for name in ["debug", "release"] {
614 let path = target.join(name);
615 match std::fs::symlink_metadata(&path) {
616 Ok(metadata) => ensure!(
617 metadata.is_dir() && path.canonicalize()? == path,
618 "profile validation: not a plain directory: {}",
619 path.display()
620 ),
621 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
622 Err(error) => {
623 return Err(error)
624 .with_context(|| format!("profile validation: inspect {}", path.display()));
625 }
626 }
627 }
628 Ok(target)
629}
630
631fn plain_file(path: &Path, stage: &str) -> Result<()> {
632 let meta = std::fs::symlink_metadata(path)
633 .with_context(|| format!("{stage}: inspect {}", path.display()))?;
634 ensure!(
635 meta.is_file() && path.canonicalize()? == path,
636 "{stage}: not a plain file: {}",
637 path.display()
638 );
639 Ok(())
640}
641
642fn cache_tag(target: &Path, allow_missing: bool) -> Result<()> {
643 let path = target.join("CACHEDIR.TAG");
644 match std::fs::symlink_metadata(&path) {
645 Err(error) if allow_missing && error.kind() == std::io::ErrorKind::NotFound => {
646 return Ok(());
647 }
648 result => {
649 result.with_context(|| format!("cache tag: inspect {}", path.display()))?;
650 }
651 }
652 plain_file(&path, "cache tag")?;
653 let tag = std::fs::read_to_string(&path)
654 .with_context(|| format!("cache tag: read {}", path.display()))?;
655 ensure!(
656 tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55"),
657 "unknown Cargo cache tag: {}",
658 path.display()
659 );
660 Ok(())
661}
662
663fn output_age(target: &Path, now: Timestamp, grace: u64) -> Result<()> {
667 let mut dirs = vec![target.to_owned()];
668 let mut entries = 0;
669 while let Some(dir) = dirs.pop() {
670 for entry in std::fs::read_dir(&dir)
671 .with_context(|| format!("output activity: list {}", dir.display()))?
672 {
673 let path = entry?.path();
674 entries += 1;
675 ensure!(
676 entries <= 100_000,
677 "output activity inventory too large: {}",
678 target.display()
679 );
680 let metadata = std::fs::symlink_metadata(&path)
681 .with_context(|| format!("output activity: inspect {}", path.display()))?;
682 if metadata.is_dir() {
683 if dir != target
685 || matches!(
686 path.file_name().and_then(|s| s.to_str()),
687 Some("debug" | "release")
688 )
689 {
690 dirs.push(path);
691 }
692 } else if metadata.is_file()
693 && (dir != target || path.file_name().is_some_and(|n| n == ".rustc_info.json"))
694 {
695 let at =
696 Timestamp::try_from(metadata.modified().with_context(|| {
697 format!("output activity: modified {}", path.display())
698 })?)?;
699 ensure!(
700 crate::clean::due(now, at, grace),
701 "output activity within grace period: {} modified {at} (now {now}, grace {grace}s)",
702 path.display()
703 );
704 }
705 }
706 }
707 Ok(())
708}
709
710pub(crate) fn artifact_signature(path: &Path) -> Result<bool> {
711 use std::io::Read as _;
712 let mut header = [0; 8];
713 let n = File::open(path)?.read(&mut header)?;
714 let header = &header[..n];
715 Ok(header.starts_with(b"\x7fELF")
716 || header.starts_with(b"!<arch>\n")
717 || header.starts_with(b"rust\0\0\0")
718 || header.starts_with(b"MZ\x90\0")
719 || [
720 [0xfe, 0xed, 0xfa, 0xce],
721 [0xce, 0xfa, 0xed, 0xfe],
722 [0xfe, 0xed, 0xfa, 0xcf],
723 [0xcf, 0xfa, 0xed, 0xfe],
724 [0xca, 0xfe, 0xba, 0xbe],
725 [0xca, 0xfe, 0xba, 0xbf],
726 ]
727 .iter()
728 .any(|magic| header.starts_with(magic)))
729}
730
731fn profile_output_dirs(profile: &Path, require_proof: bool) -> Result<Vec<PathBuf>> {
735 let mut present = std::collections::BTreeSet::new();
736 for path in [
737 profile.to_owned(),
738 profile.join("deps"),
739 profile.join(".fingerprint"),
740 profile.join("build"),
741 ] {
742 match std::fs::symlink_metadata(&path) {
743 Ok(_) => {
744 ensure!(
745 plain_dir(&path) && path.canonicalize()? == path,
746 "Cargo profile proof: {} is not a plain directory",
747 path.display()
748 );
749 present.insert(path);
750 }
751 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
752 Err(e) => return Err(e.into()),
753 }
754 }
755 if !present.contains(profile) {
756 return Ok(Vec::new());
757 }
758 let deps = present.contains(&profile.join("deps"));
759 let fingerprint = present.contains(&profile.join(".fingerprint"));
760 let mut dirs = vec![profile.to_owned()];
761 if deps || fingerprint {
762 ensure!(
763 !require_proof || (deps && fingerprint),
764 "incomplete Cargo v1 profile proof"
765 );
766 if deps {
767 dirs.push(profile.join("deps"));
768 }
769 if present.contains(&profile.join("build")) {
772 for entry in std::fs::read_dir(profile.join("build"))? {
773 let path = entry?.path();
774 if plain_dir(&path) {
775 for entry in std::fs::read_dir(&path)? {
776 ensure!(
777 !entry?.file_name().to_str().is_some_and(unit_hash),
778 "mixed Cargo profile layouts"
779 );
780 }
781 }
782 }
783 }
784 } else if present.contains(&profile.join("build")) {
785 let mut units = 0;
786 for package in std::fs::read_dir(profile.join("build"))? {
787 let package = package?.path();
788 let meta = std::fs::symlink_metadata(&package)?;
789 ensure!(
790 !meta.file_type().is_symlink(),
791 "Cargo v2 package is a symlink: {}",
792 package.display()
793 );
794 if !meta.is_dir() {
795 ensure!(!require_proof, "Cargo v2 package is not a directory");
796 continue;
797 }
798 ensure!(
799 package.canonicalize()? == package,
800 "Cargo v2 package is not a plain directory"
801 );
802 let mut package_units = 0;
803 for unit in std::fs::read_dir(&package)? {
804 let unit = unit?.path();
805 let meta = std::fs::symlink_metadata(&unit)?;
806 ensure!(
807 !meta.file_type().is_symlink(),
808 "Cargo v2 unit is a symlink: {}",
809 unit.display()
810 );
811 let structured = meta.is_dir()
815 && unit
816 .file_name()
817 .and_then(|n| n.to_str())
818 .is_some_and(unit_hash)
819 && unit.canonicalize()? == unit
820 && ["fingerprint", "out"].iter().all(|name| {
821 let path = unit.join(name);
822 plain_dir(&path) && path.canonicalize().is_ok_and(|p| p == path)
823 });
824 if structured && v2_run_script_unit(&unit) {
827 continue;
828 }
829 let complete = structured
830 && plain_file(
831 &unit.join("fingerprint/invoked.timestamp"),
832 "Cargo unit fingerprint",
833 )
834 .is_ok();
835 if !complete {
836 ensure!(!require_proof, "invalid or incomplete Cargo v2 unit");
837 continue;
838 }
839 dirs.push(unit.join("out"));
840 units += 1;
841 package_units += 1;
842 ensure!(units <= 4096, "Cargo unit inventory too large");
843 }
844 ensure!(
845 package_units > 0 || !require_proof,
846 "empty Cargo v2 package"
847 );
848 }
849 ensure!(units > 0 || !require_proof, "missing Cargo v2 units");
850 } else {
851 ensure!(!require_proof, "missing Cargo profile proof");
852 }
853 Ok(dirs)
854}
855
856fn v2_run_script_unit(unit: &Path) -> bool {
859 let fp = unit.join("fingerprint");
860 if std::fs::symlink_metadata(fp.join("invoked.timestamp")).is_ok() {
861 return false;
862 }
863 let Ok(entries) = std::fs::read_dir(&fp) else {
864 return false;
865 };
866 for entry in entries.flatten() {
867 let path = entry.path();
868 let name = path.file_name().unwrap_or_default().to_string_lossy();
869 if name.starts_with("run-build-script-")
870 && path.extension().is_none()
871 && plain_file(&path, "Cargo run fingerprint").is_ok()
872 && plain_file(&path.with_extension("json"), "Cargo run fingerprint json").is_ok()
873 {
874 return true;
875 }
876 }
877 false
878}
879
880fn v2_compiler_dep_info(dir: &Path, dep_info: &Path) -> Result<bool> {
883 let unit = dir.parent().context("Cargo unit parent")?;
884 let hash = unit.file_name().unwrap_or_default().to_string_lossy();
885 let stem = dep_info.file_stem().unwrap_or_default().to_string_lossy();
886 for entry in std::fs::read_dir(unit.join("fingerprint"))? {
887 let path = entry?.path();
888 let name = path.file_name().unwrap_or_default().to_string_lossy();
889 let library = name
892 .strip_prefix("lib-")
893 .is_some_and(|name| stem == format!("{}-{hash}", name.replace('-', "_")));
894 let binary = name
895 .strip_prefix("bin-")
896 .is_some_and(|name| stem == name.replace('-', "_"));
897 if library || binary {
898 plain_file(&path, "Cargo compiler fingerprint")?;
899 plain_file(
900 &path.with_extension("json"),
901 "Cargo compiler fingerprint json",
902 )?;
903 plain_file(
904 &path.with_file_name(format!(
905 "dep-{}",
906 path.file_name().unwrap().to_string_lossy()
907 )),
908 "Cargo compiler fingerprint dep-info",
909 )?;
910 return Ok(true);
911 }
912 }
913 Ok(false)
914}
915
916fn unit_hash(name: &str) -> bool {
917 name.len() == 16 && name.bytes().all(|b| b.is_ascii_hexdigit())
918}
919
920fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
924 let mut outputs = std::collections::BTreeSet::new();
925 for profile in ["debug", "release"] {
926 for dir in profile_output_dirs(&target.join(profile), false)? {
927 if !plain_dir(&dir) || dir.canonicalize()? != dir {
928 continue;
929 }
930 for entry in std::fs::read_dir(&dir)? {
931 let path = entry?.path();
932 if path.extension().is_none_or(|ext| ext != "d")
933 || !std::fs::symlink_metadata(&path)?.is_file()
934 || std::fs::metadata(&path)?.len() > 1024 * 1024
935 || path.canonicalize()? != path
936 {
937 continue;
938 }
939 if dir.file_name().is_some_and(|name| name == "out")
940 && !v2_compiler_dep_info(&dir, &path)?
941 {
942 continue;
943 }
944 let body = std::fs::read_to_string(&path)?;
945 for line in body.lines() {
946 let Some((names, dependencies)) = line.split_once(": ") else {
947 continue;
948 };
949 if dependencies.trim().is_empty() {
950 continue;
951 }
952 let mut words = Vec::new();
955 let mut word = String::new();
956 let mut escaped = false;
957 for c in names.chars() {
958 if escaped {
959 word.push(c);
960 escaped = false;
961 } else if c == '\\' {
962 escaped = true;
963 } else if c.is_whitespace() {
964 if !word.is_empty() {
965 words.push(std::mem::take(&mut word));
966 }
967 } else {
968 word.push(c);
969 }
970 }
971 if escaped || word.contains(['$', '%']) {
972 continue;
973 }
974 if !word.is_empty() {
975 words.push(word);
976 }
977 for name in words {
978 let output = PathBuf::from(name);
979 let stem = path.file_stem().unwrap_or_default().to_string_lossy();
980 let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
981 let artifact_name =
982 output_stem == stem || output_stem == format!("lib{stem}");
983 let artifact_kind = output.extension().is_none_or(|ext| {
984 matches!(
985 ext.to_str(),
986 Some(
987 "rlib"
988 | "rmeta"
989 | "so"
990 | "dylib"
991 | "a"
992 | "lib"
993 | "dll"
994 | "exe"
995 | "pdb"
996 )
997 )
998 });
999 if artifact_name
1000 && artifact_kind
1001 && output.parent() == Some(dir.as_path())
1002 && output.extension().is_none_or(|ext| ext != "d")
1003 && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
1004 && output.canonicalize().is_ok_and(|p| p == output)
1005 && artifact_signature(&output)?
1006 {
1007 outputs.insert(output);
1008 }
1009 }
1010 }
1011 }
1012 }
1013 }
1014 Ok(outputs.into_iter().collect())
1015}
1016
1017#[derive(Serialize, Deserialize)]
1018struct CargoResidue {
1019 target: PathBuf,
1020 files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
1021}
1022
1023fn residue_files(
1026 target: &Path,
1027 outputs: &[PathBuf],
1028) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
1029 let mut files = std::collections::BTreeMap::new();
1030 let mut dirs = vec![target.to_owned()];
1031 let mut bytes = 0usize;
1032 let mut entries = 0usize;
1033 while let Some(dir) = dirs.pop() {
1034 for entry in std::fs::read_dir(dir)? {
1035 let path = entry?.path();
1036 entries += 1;
1037 ensure!(entries <= 4096, "residue inventory too large");
1038 let metadata = std::fs::symlink_metadata(&path)?;
1039 ensure!(
1040 !metadata.file_type().is_symlink(),
1041 "symlink in Cargo residue"
1042 );
1043 if metadata.is_dir() {
1044 dirs.push(path);
1045 } else {
1046 ensure!(metadata.is_file(), "non-file in Cargo residue");
1047 if outputs.contains(&path) {
1048 continue;
1049 }
1050 ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
1051 let body = std::fs::read(&path)?;
1052 bytes = bytes.saturating_add(body.len());
1053 ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
1054 files.insert(path.strip_prefix(target)?.to_owned(), body);
1055 }
1056 }
1057 }
1058 Ok(files)
1059}
1060
1061fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
1062 Ok(home
1063 .join("runs")
1064 .join(id)
1065 .join("build-residue")
1066 .join(worktree.file_name().context("missing worktree name")?))
1067}
1068
1069fn residue_root(target: &Path) -> Result<()> {
1072 for entry in std::fs::read_dir(target)? {
1073 let name = entry?.file_name();
1074 ensure!(
1075 matches!(
1076 name.to_str(),
1077 Some("debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json")
1078 ),
1079 "unrecognized Cargo residue root entry"
1080 );
1081 }
1082 Ok(())
1083}
1084
1085fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
1086 cache_tag(target, false)?;
1087 residue_root(target)?;
1088 let files = residue_files(target, outputs)?;
1089 for (relative, body) in &files {
1090 let path = target.join(relative);
1091 let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
1092 let lock = matches!(
1093 relative.to_str(),
1094 Some("debug/.cargo-lock" | "release/.cargo-lock")
1095 ) && body.is_empty();
1096 let dep_info = path.extension().is_some_and(|ext| ext == "d")
1097 && outputs.iter().any(|output| {
1098 output.parent() == path.parent()
1099 && (output.file_stem() == path.file_stem()
1100 || output.file_stem().is_some_and(|stem| {
1101 stem.to_string_lossy()
1102 == format!(
1103 "lib{}",
1104 path.file_stem().unwrap_or_default().to_string_lossy()
1105 )
1106 }))
1107 });
1108 ensure!(identity || lock || dep_info, "unproven Cargo residue");
1109 }
1110 let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
1111 std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
1112 let tmp = proof.with_extension("tmp");
1113 std::fs::write(
1114 &tmp,
1115 serde_json::to_vec(&CargoResidue {
1116 target: target.to_owned(),
1117 files,
1118 })?,
1119 )?;
1120 std::fs::rename(tmp, proof)?;
1121 Ok(())
1122}
1123
1124pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
1125 let target = cargo_target(worktree).await?;
1126 residue_root(&target)?;
1127 let locks = cargo_locks(&target)?;
1128 let proof: CargoResidue =
1129 serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
1130 ensure!(
1131 proof.target == target && proof.files == residue_files(&target, &[])?,
1132 "Cargo residue changed or contains unproven files"
1133 );
1134 Ok(locks)
1135}
1136
1137pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
1139 let mut held = Vec::new();
1140 for (profile, name) in ["debug", "release"].into_iter().flat_map(|profile| {
1141 [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"].map(|name| (profile, name))
1142 }) {
1143 let lock = target.join(profile).join(name);
1144 match std::fs::symlink_metadata(&lock) {
1145 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
1146 result => {
1147 result.with_context(|| format!("Cargo lock: inspect {}", lock.display()))?;
1148 }
1149 }
1150 {
1151 plain_file(&lock, "Cargo lock")?;
1152 let file = OpenOptions::new()
1153 .read(true)
1154 .write(true)
1155 .open(&lock)
1156 .with_context(|| format!("Cargo lock: open {}", lock.display()))?;
1157 crate::pr_head::try_lock_file(&file)
1158 .with_context(|| format!("Cargo build is active: {}", lock.display()))?;
1159 held.push(file);
1160 }
1161 }
1162 Ok(held)
1163}
1164
1165pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
1168 let home = home.to_owned();
1169 let disk = disk.clone();
1170 let runtime = tokio::runtime::Handle::current();
1171 tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
1172 .await
1173 .unwrap_or_else(|e| Report {
1174 failures: vec![format!("retention worker failed: {e}")],
1175 ..Report::default()
1176 })
1177}
1178
1179#[derive(Default, Serialize, Deserialize)]
1180struct Measurement {
1181 at: Option<Timestamp>,
1182 bytes: u64,
1183}
1184
1185async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
1186 let mut report = Report {
1187 checked_at: Some(now),
1188 limit_bytes: disk.worktree_limit_bytes,
1189 reclaim_budget_bytes: disk.worktree_reclaim_bytes,
1190 ..Report::default()
1191 };
1192 if !cfg!(unix) {
1193 report
1194 .failures
1195 .push("Automatic output reclamation requires Unix process containment".into());
1196 return report;
1197 }
1198 let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
1199 return report;
1200 };
1201 let mut targets = Vec::new();
1202 let mut under_pressure = false;
1203 let mut registrations = std::collections::HashMap::new();
1204 let measurements_path = home.join("output-measurements.json");
1205 let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
1206 std::fs::read(&measurements_path)
1207 .ok()
1208 .and_then(|b| serde_json::from_slice(&b).ok())
1209 .unwrap_or_default();
1210 let mut observed = std::collections::BTreeSet::new();
1211 let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
1212 .into_iter()
1213 .flatten()
1214 .flatten()
1215 .map(|e| e.file_name().to_string_lossy().into_owned())
1216 .collect();
1217 ids.sort();
1218 for id in ids {
1219 if !crate::run::is_run_id(&id) {
1220 continue;
1221 }
1222 let state = match RunState::load_under(&id, home) {
1223 Ok(state) => state,
1224 Err(e) => {
1225 report
1226 .retained
1227 .push(format!("{id}: unreadable run ownership: {e:#}"));
1228 continue;
1229 }
1230 };
1231 if disk.worktree_limit_bytes == 0 {
1232 continue;
1233 }
1234 match run_paths_missing(&state) {
1235 Ok(true) => continue,
1236 Ok(false) => {}
1237 Err(e) => {
1238 report.failures.push(format!("{id}: {e:#}"));
1239 continue;
1240 }
1241 }
1242 let list = match registrations.entry(state.repo.clone()) {
1243 std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
1244 std::collections::hash_map::Entry::Vacant(entry) => {
1245 match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
1246 Ok(list) => entry.insert(list),
1247 Err(e) => {
1248 if matches!(run_paths_missing(&state), Ok(true)) {
1251 continue;
1252 }
1253 report.failures.push(format!("{id}: {e:#}"));
1254 continue;
1255 }
1256 }
1257 }
1258 };
1259 let paths = match registered_worktrees(&state, list) {
1260 Ok(paths) => paths,
1261 Err(e) => {
1262 report.failures.push(format!("{id}: {e:#}"));
1263 continue;
1264 }
1265 };
1266 for path in paths {
1267 if !path.join("target").exists() {
1268 continue;
1269 }
1270 let target = path.join("target");
1271 observed.insert(target.clone());
1272 let pressure =
1273 crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
1274 under_pressure |= pressure;
1275 let measurement = measurements.entry(target).or_default();
1276 if pressure
1277 || measurement
1278 .at
1279 .is_none_or(|at| crate::clean::due(now, at, 300))
1280 {
1281 measurement.bytes = crate::disk::dir_size(&path.join("target"));
1282 measurement.at = Some(now);
1283 }
1284 let size = measurement.bytes;
1285 report.measured_bytes = report.measured_bytes.saturating_add(size);
1286 targets.push((state.output_activity(), id.clone(), path, size));
1287 }
1288 }
1289 if under_pressure {
1290 targets.sort_by_key(|t| (std::cmp::Reverse(t.3), t.0));
1293 } else {
1294 targets.sort_by_key(|t| t.0);
1295 }
1296 let free_before = crate::disk::free_bytes(home).ok();
1297 #[cfg(unix)]
1298 {
1299 let mut seen = std::collections::BTreeSet::new();
1300 report.measured_allocated_file_bytes = Some(0);
1301 report.estimated_released_allocated_bytes = Some(0);
1302 report.proven_reclaimable_allocated_bytes = Some(0);
1303 for (_, _, path, _) in &targets {
1304 report.measured_allocated_file_bytes = report
1305 .measured_allocated_file_bytes
1306 .zip(allocated_inventory(&path.join("target"), &mut seen).ok())
1307 .map(|(a, b)| a.saturating_add(b));
1308 }
1309 }
1310 let mut remaining = report.measured_bytes;
1311 let pressure_paths: Vec<_> = targets.iter().map(|t| t.2.clone()).collect();
1312 let mut deletion_started = false;
1313 let mut oversized_started = false;
1314 for (updated, id, path, _) in targets {
1315 if disk.worktree_limit_bytes == 0
1316 || remaining <= disk.worktree_limit_bytes
1317 && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
1318 {
1319 report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
1320 continue;
1321 }
1322 if oversized_started || report.freed_bytes >= disk.worktree_reclaim_bytes {
1323 report
1324 .retained
1325 .push(format!("{id}: per-pass byte budget spent"));
1326 continue;
1327 }
1328 let result = async {
1329 ensure!(
1330 crate::clean::due(now, updated, disk.fold_grace_secs),
1331 "execution activity within grace period: {updated} (now {now}, grace {}s)", disk.fold_grace_secs
1332 );
1333 let _owner = crate::pr_head::owner_lock(home, &id)?;
1334 let state = RunState::load_under(&id, home)?;
1335 ensure!(
1336 state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
1337 == Liveness::Dead,
1338 "run live or ownership unknown"
1339 );
1340 ensure!(crate::clean::due(now, state.output_activity(), disk.fold_grace_secs), "execution activity after inventory: {}", state.output_activity());
1341 let _driver = crate::driver::maintenance(home, &id)?;
1342 let _outputs = OutputLease::take(&path, true)?;
1343 ensure!(
1344 worktrees(&state).await?.contains(&path),
1345 "worktree registration changed"
1346 );
1347 let target = validate_cargo_target(&path, true).await
1348 .with_context(|| format!("stage validate-target: {}", path.join("target").display()))?;
1349 let _cargo = cargo_locks(&target)
1350 .with_context(|| format!("stage cargo-locks: {}", target.display()))?;
1351 validate_cargo_target(&path, true).await
1354 .with_context(|| format!("stage validate-target (locked): {}", target.display()))?;
1355 let lease_meta = _outputs._file.metadata().context("output lease activity: metadata")?;
1356 if lease_meta.len() > 0 {
1357 ensure!(lease_meta.len() == 1, "output lease activity witness is unreadable or incomplete for {}", path.display());
1358 let lease_at = Timestamp::try_from(lease_meta.modified().context("output lease activity: modified")?)?;
1359 ensure!(crate::clean::due(now, lease_at, disk.fold_grace_secs),
1360 "output lease activity within grace period: {lease_at} (now {now}, grace {}s)", disk.fold_grace_secs);
1361 }
1362 output_age(&target, now, disk.fold_grace_secs)
1363 .with_context(|| format!("stage output-age: {}", target.display()))?;
1364 let mut outputs = declared_outputs(&target)
1365 .with_context(|| format!("stage declared-outputs: {}", target.display()))?;
1366 let incremental = match crate::incremental::candidates(&path) {
1367 Ok(candidates) => candidates,
1368 Err(error) => {
1369 report.retained.push(format!("{id}: incremental cache retained: {error:#}"));
1370 crate::incremental::Reclaim::default()
1371 }
1372 };
1373 let incremental_files = incremental.files.clone();
1374 outputs.extend(incremental_files.iter().cloned());
1375 outputs.sort();
1376 outputs.dedup();
1377 ensure!(!outputs.is_empty(), "no dep-info-proven outputs or provenance-proven incremental caches; legacy incremental/build and unknown files retained");
1378 let sizes = outputs.iter().map(|p| std::fs::metadata(p).map(|m| m.len())
1379 .with_context(|| format!("stage measure: {}", p.display())))
1380 .collect::<Result<Vec<_>>>()?;
1381 let current_size = sizes.iter().fold(0u64, |bytes, size| bytes.saturating_add(*size));
1382 report.proven_reclaimable_bytes = report.proven_reclaimable_bytes.saturating_add(current_size);
1383 let mut allocation = ReleasedAllocation::of(&outputs)?;
1384 report.proven_reclaimable_allocated_bytes = report.proven_reclaimable_allocated_bytes
1385 .zip(allocation.reclaimable_allocated_bytes()).map(|(a, b)| a.saturating_add(b));
1386 let exceeds_budget = current_size > disk.worktree_reclaim_bytes.saturating_sub(report.freed_bytes);
1387 ensure!(!deletion_started || !exceeds_budget,
1391 "declared outputs exceed remaining per-pass byte budget");
1392 let ledger = home.join("runs").join(&id).join("build-outputs.json");
1395 std::fs::write(
1396 &ledger,
1397 serde_json::to_vec(
1398 &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","identity":if target.join("CACHEDIR.TAG").exists() { "cache-tag" } else { "rustc-profile-dep-info" },"files":outputs}),
1399 )?,
1400 ).with_context(|| format!("stage ledger: {}", ledger.display()))?;
1401 if !incremental_files.is_empty() {
1404 report.retained.push(format!("{id}: incremental proof does not authorize retirement"));
1405 } else if let Err(error) = record_residue(home, &id, &target, &outputs) {
1406 report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
1407 }
1408 ensure!(worktrees(&state).await?.contains(&path),
1409 "worktree registration changed before deletion");
1410 deletion_started = true;
1411 oversized_started = exceeds_budget;
1412 measurements.remove(&target);
1413 for (output, size) in outputs.iter().zip(sizes) {
1414 std::fs::remove_file(output).with_context(|| format!("stage remove: {}", output.display()))?;
1415 report.estimated_released_allocated_bytes = report.estimated_released_allocated_bytes
1416 .zip(allocation.unlinked(output)).map(|(a, b)| a.saturating_add(b));
1417 report.freed_bytes = report.freed_bytes.saturating_add(size);
1419 remaining = remaining.saturating_sub(size);
1420 }
1421 incremental.finish().context("finish incremental reclamation")?;
1422 report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
1423 Ok::<_, anyhow::Error>(())
1424 }
1425 .await;
1426 if let Err(e) = result {
1427 report.retained.push(format!(
1428 "{id}/{}: {e:#}",
1429 path.file_name().unwrap_or_default().to_string_lossy()
1430 ));
1431 }
1432 }
1433 measurements.retain(|path, _| observed.contains(path));
1434 if let Ok(body) = serde_json::to_vec(&measurements)
1435 && let Err(e) = std::fs::write(&measurements_path, body)
1436 {
1437 report.failures.push(format!("output measurements: {e}"));
1438 }
1439 copies::evict(home, disk, now, &mut report).await;
1440 retire_empty(home, disk, now, &mut report).await;
1441 report.protected_or_unproven_bytes = report
1442 .measured_bytes
1443 .saturating_sub(report.proven_reclaimable_bytes);
1444 let pressure_remaining = pressure_paths
1445 .iter()
1446 .any(|path| crate::disk::free_bytes(path).map_or(true, |free| free < disk.min_free_bytes));
1447 if !report.failures.is_empty()
1448 || disk.worktree_limit_bytes > 0
1449 && (remaining > disk.worktree_limit_bytes || pressure_remaining)
1450 {
1451 let details = report
1452 .retained
1453 .iter()
1454 .filter(|reason| {
1455 !reason.contains("within policy")
1456 && !reason.ends_with("unmanifested files and Cargo metadata retained")
1457 })
1458 .take(4)
1459 .chain(report.failures.iter().take(2))
1460 .cloned()
1461 .collect::<Vec<_>>()
1462 .join("; ");
1463 crate::notices::raise_in(
1464 home,
1465 crate::notices::Notice::warn(
1466 "disk:retention",
1467 format!(
1468 "Retention remains blocked by the disk cap, free-space floor or inspection failures (logical inventory {}B, proven reclaimable {}B, protected or unproven {}B, unlinked {}B). {} Retained/held targets: {details}. Inspect the cleanup report for active owners, retained recovery sources and failures.",
1469 report.measured_bytes,
1470 report.proven_reclaimable_bytes,
1471 report.protected_or_unproven_bytes,
1472 report.freed_bytes,
1473 if report.proven_reclaimable_bytes == 0 {
1474 "No inspected contents have deletion authority."
1475 } else {
1476 "Authorized reclamation was insufficient; remaining protections and budgets still apply."
1477 }
1478 ),
1479 ),
1480 );
1481 }
1482 report.observed_home_free_change_bytes = free_before
1483 .zip(crate::disk::free_bytes(home).ok())
1484 .map(|(before, after)| {
1485 (i128::from(after) - i128::from(before))
1486 .clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
1487 });
1488 let store = home.join("retention.json");
1489 if let Ok(body) = serde_json::to_vec_pretty(&report) {
1490 let tmp = home.join("retention.json.tmp");
1491 if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
1492 tracing::warn!("retention report could not be saved: {e}");
1493 }
1494 }
1495 tracing::info!(
1496 measured_bytes = report.measured_bytes,
1497 freed_bytes = report.freed_bytes,
1498 retired = report.retired,
1499 retained = report.retained.len(),
1500 failures = report.failures.len(),
1501 "retained worktree cleanup"
1502 );
1503 for reason in report.retained.iter().chain(&report.failures) {
1504 tracing::debug!(%reason, "worktree retained");
1505 }
1506 report
1507}
1508
1509async fn retire_empty(
1510 home: &Path,
1511 disk: &crate::config::Disk,
1512 now: Timestamp,
1513 report: &mut Report,
1514) {
1515 if !disk.auto_fold {
1516 return;
1517 }
1518 let queue = crate::queue::Queue::at(home.join("queue"));
1519 let questions = crate::ask::Questions::at(home.join("questions"));
1520 let open = questions.list_for_effects();
1521 for listed in queue.list() {
1522 let Ok(_claim) = queue.claim(&listed.id) else {
1523 continue;
1524 };
1525 let Ok(task) = queue.get(&listed.id) else {
1526 continue;
1527 };
1528 if open
1529 .iter()
1530 .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
1531 {
1532 report.retained.push(format!(
1533 "{}: task has an unanswered owner decision",
1534 task.id
1535 ));
1536 continue;
1537 }
1538 let sources: Vec<_> = task
1539 .runs
1540 .iter()
1541 .collect::<std::collections::BTreeSet<_>>()
1542 .into_iter()
1543 .cloned()
1544 .collect();
1545 let mut useful = Vec::new();
1546 let mut selected = Vec::new();
1547 let mut public = Vec::new();
1548 let mut unreadable = false;
1549 for id in &sources {
1550 let Ok(state) = RunState::load_under(id, home) else {
1551 unreadable = true;
1552 continue;
1553 };
1554 if state.released() || state.retired_to.is_some() {
1555 continue;
1556 }
1557 if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
1558 selected.push(state.clone());
1559 }
1560 if state.pr.is_some() && state.pr_taken_over_by.is_none() {
1561 public.push(state.clone());
1562 }
1563 if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
1564 useful.push(state);
1565 }
1566 }
1567 let pool = if !selected.is_empty() {
1571 selected
1572 } else if !public.is_empty() {
1573 public
1574 } else {
1575 useful
1576 };
1577 if unreadable || pool.len() != 1 {
1578 report.retained.push(format!(
1579 "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
1580 task.id,
1581 sources.len()
1582 ));
1583 continue;
1584 }
1585 let authority = &pool[0];
1586 if authority.candidates.is_empty() {
1587 continue;
1588 }
1589 let latest = &authority.id;
1590 for id in &sources {
1591 if id == latest || report.retired >= 5 {
1592 continue;
1593 }
1594 if let Ok(state) = RunState::load_under(id, home)
1595 && state.retired_to.as_ref() == Some(latest)
1596 && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
1597 {
1598 continue;
1599 }
1600 let result = async {
1601 ensure!(
1602 !task
1603 .resume_override
1604 .as_ref()
1605 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
1606 "operator-pinned recovery run"
1607 );
1608 ensure!(
1609 !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
1610 "selected replacement recovery source"
1611 );
1612 let _owner = crate::pr_head::owner_lock(home, id)?;
1613 let mut state = RunState::load_under(id, home)?;
1614 ensure!(
1615 state.repo == authority.repo,
1616 "repository differs from recovery authority"
1617 );
1618 ensure!(
1619 !crate::pr_recovery::explicit_sources_retained(home, id),
1620 "explicitly retained recovery source"
1621 );
1622 ensure!(
1623 state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
1624 == Liveness::Dead,
1625 "run live or ownership unknown"
1626 );
1627 let _driver = crate::driver::maintenance(home, id)?;
1628 ensure!(
1629 crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
1630 "within grace period"
1631 );
1632 ensure!(
1633 !open.iter().any(|q| q.run == *id && q.status.open()),
1634 "unanswered owner decision"
1635 );
1636 ensure!(
1637 state.pr.is_none()
1638 && state.pr_head.is_none()
1639 && state.pr_recovery.is_none()
1640 && state.land_approval.is_none()
1641 && state.merge_decision.is_none()
1642 && state.reviews.is_empty()
1643 && state.recovery.is_none()
1644 && state.candidate_preservation.is_empty()
1645 && state.interrupted_implementations.is_empty()
1646 && !state.released(),
1647 "PR ownership or recovery checkpoint"
1648 );
1649 let proof = prove_empty(home, &state, latest).await?;
1650 commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
1651 for path in &proof.paths {
1654 ensure!(
1655 git::worktree_remove_clean(&state.repo, path).await?,
1656 "Git refused clean worktree removal"
1657 );
1658 }
1659 Ok::<_, anyhow::Error>(())
1660 }
1661 .await;
1662 match result {
1663 Ok(()) => report.retired += 1,
1664 Err(e) => report.retained.push(format!("{id}: {e:#}")),
1665 }
1666 }
1667 }
1668}
1669
1670struct EmptyProof {
1671 paths: Vec<PathBuf>,
1672 _outputs: Vec<OutputLease>,
1673 _cargo: Vec<File>,
1674}
1675
1676pub(crate) async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
1679 let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
1680 let mut files = Vec::new();
1681 let mut blobs = Vec::new();
1682 for record in tree.split('\0').filter(|s| !s.is_empty()) {
1683 let (header, name) = record.split_once('\t').context("unknown tree record")?;
1684 let parts: Vec<_> = header.split_whitespace().collect();
1685 ensure!(
1686 parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
1687 "unsupported tracked source kind"
1688 );
1689 let path = worktree.join(name);
1690 let meta = std::fs::symlink_metadata(&path)?;
1691 ensure!(
1692 meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
1693 "noncanonical tracked source"
1694 );
1695 #[cfg(unix)]
1696 {
1697 use std::os::unix::fs::PermissionsExt as _;
1698 ensure!(
1699 (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
1700 "tracked executable mode differs from original base"
1701 );
1702 }
1703 path.to_str().context("non UTF-8 source")?;
1704 files.push(git::path_arg(&path));
1705 blobs.push(parts[2].to_owned());
1706 }
1707 for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
1708 let mut args = vec!["hash-object", "--no-filters", "--"];
1709 args.extend(paths.iter().map(String::as_str));
1710 let actual = git::git(worktree, &args).await?;
1711 ensure!(
1712 actual.lines().eq(expected.iter().map(String::as_str)),
1713 "tracked bytes differ from original base"
1714 );
1715 }
1716 Ok(())
1717}
1718
1719async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
1720 let id = &state.id;
1721 let paths = worktrees(state).await?;
1722 let mut guards = Vec::new();
1723 let mut cargo_guards = Vec::new();
1724 for path in &paths {
1725 guards.push(OutputLease::take(path, true)?);
1726 ensure!(
1727 git::rev_parse(path, "HEAD").await? == state.base_commit,
1728 "unique or unrecorded commits"
1729 );
1730 ensure!(
1731 git::git(
1732 path,
1733 &[
1734 "--no-optional-locks",
1735 "status",
1736 "--porcelain",
1737 "--untracked-files=all"
1738 ]
1739 )
1740 .await?
1741 .is_empty(),
1742 "tracked or untracked source changes"
1743 );
1744 ensure!(
1745 git::git(
1746 path,
1747 &[
1748 "--no-optional-locks",
1749 "status",
1750 "--porcelain",
1751 "--untracked-files=all",
1752 "--ignored",
1753 "--",
1754 ".",
1755 ":(exclude)target"
1756 ]
1757 )
1758 .await?
1759 .is_empty(),
1760 "source changes, ignored files or build outputs"
1761 );
1762 tracked_base(path, &state.base_commit).await?;
1763 match std::fs::symlink_metadata(path.join("target")) {
1764 Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
1765 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1766 Err(error) => return Err(error.into()),
1767 }
1768 for checkpoint in [
1769 "rebase-merge",
1770 "rebase-apply",
1771 "sequencer",
1772 "BISECT_LOG",
1773 "REVERT_HEAD",
1774 "AUTO_MERGE",
1775 "MERGE_MSG",
1776 "CHERRY_PICK_HEAD",
1777 "MERGE_HEAD",
1778 "REBASE_HEAD",
1779 ] {
1780 let location = git::git(
1781 path,
1782 &[
1783 "rev-parse",
1784 "--path-format=absolute",
1785 "--git-path",
1786 checkpoint,
1787 ],
1788 )
1789 .await?;
1790 ensure!(
1791 matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
1792 "retained Git recovery metadata: {checkpoint}"
1793 );
1794 }
1795 let origin_path = git::git(
1798 path,
1799 &[
1800 "rev-parse",
1801 "--path-format=absolute",
1802 "--git-path",
1803 "ORIG_HEAD",
1804 ],
1805 )
1806 .await?;
1807 match std::fs::symlink_metadata(origin_path.trim()) {
1808 Ok(meta) => {
1809 ensure!(
1810 meta.is_file() && !meta.file_type().is_symlink(),
1811 "unknown ORIG_HEAD artifact"
1812 );
1813 git::rev_parse(path, "ORIG_HEAD")
1814 .await
1815 .context("unreadable ORIG_HEAD")?;
1816 }
1817 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1818 Err(e) => return Err(e.into()),
1819 }
1820 if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1821 ensure!(
1822 git::git_raw(
1823 path,
1824 &["merge-base", "--is-ancestor", &original, &state.base_commit]
1825 )
1826 .await?
1827 .ok(),
1828 "ORIG_HEAD retains commits outside the original base"
1829 );
1830 }
1831 let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1832 for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1833 ensure!(
1834 git::git_raw(
1835 path,
1836 &["merge-base", "--is-ancestor", head, &state.base_commit]
1837 )
1838 .await?
1839 .ok(),
1840 "worktree reflog retains commits outside the original base"
1841 );
1842 }
1843 for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1844 ensure!(
1845 !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1846 .await?
1847 .ok(),
1848 "unfinished Git operation"
1849 );
1850 }
1851 }
1852 if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1855 return Ok(EmptyProof {
1856 paths,
1857 _outputs: guards,
1858 _cargo: cargo_guards,
1859 });
1860 }
1861 ensure!(!paths.is_empty(), "no worktrees to prove empty");
1862 for candidate in &state.candidates {
1863 ensure!(
1864 (paths.contains(&candidate.worktree)
1865 || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1866 && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1867 "candidate recovery source"
1868 );
1869 }
1870 let pattern = format!("refs/heads/magi/{}/", state.short());
1871 let branches = git::git(
1872 &state.repo,
1873 &["for-each-ref", "--format=%(objectname)", &pattern],
1874 )
1875 .await?;
1876 ensure!(
1877 branches.lines().all(|head| head == state.base_commit),
1878 "unrecorded candidate branch changes"
1879 );
1880 Ok(EmptyProof {
1881 paths,
1882 _outputs: guards,
1883 _cargo: cargo_guards,
1884 })
1885}
1886
1887fn commit_retirement(
1892 home: &Path,
1893 queue: &crate::queue::Queue,
1894 expected: &crate::queue::Task,
1895 state: &mut RunState,
1896 authority: &str,
1897 sources: &[String],
1898 now: Timestamp,
1899) -> Result<()> {
1900 let _write = queue.lock_task(&expected.id)?;
1901 let current = queue.get(&expected.id)?;
1902 ensure!(
1903 serde_json::to_value(¤t)? == serde_json::to_value(expected)?,
1904 "task changed during retirement proof; retaining worktrees for the current owner decision"
1905 );
1906 ensure!(
1907 !current
1908 .resume_override
1909 .as_ref()
1910 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1911 && !current
1912 .replacement
1913 .as_ref()
1914 .is_some_and(|pin| pin.run == state.id),
1915 "current owner selected this recovery source"
1916 );
1917 ensure!(
1918 !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1919 "explicitly retained recovery source"
1920 );
1921 ensure!(
1922 !crate::ask::Questions::at(home.join("questions"))
1923 .list_for_effects()
1924 .iter()
1925 .any(|q| q.status.open()
1926 && (q.run == state.id
1927 || q.run == current.id
1928 || current.blocked_by.contains(&q.id))),
1929 "owner decision arrived during retirement proof"
1930 );
1931 let dir = home.join("recovery-lineages");
1932 std::fs::create_dir_all(&dir)?;
1933 let tmp = dir.join(format!("{}.tmp", current.id));
1934 let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1935 "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1936 std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1937 std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1938 state.retired_to = Some(authority.to_owned());
1939 state.event(
1940 "retention",
1941 format!(
1942 "empty attempt retired to {authority}; recovery sources: {}",
1943 sources.join(", ")
1944 ),
1945 );
1946 state.save_under(home)?;
1947 Ok(())
1949}
1950
1951pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1955 if min_free == 0 {
1956 return std::future::pending().await;
1957 }
1958 loop {
1959 for path in &paths {
1960 let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1961 match crate::disk::free_bytes(existing) {
1962 Ok(free) if free >= min_free / 2 => {}
1963 Ok(free) => {
1964 return format!(
1965 "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1966 );
1967 }
1968 Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1969 }
1970 }
1971 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1972 }
1973}
1974
1975#[cfg(all(test, unix))]
1976mod tests {
1977 use super::*;
1978 use crate::{
1979 config::{Config, Disk},
1980 queue::{Source, Task},
1981 run::{Candidate, RunStatus},
1982 };
1983
1984 struct Fx {
1985 _temp: tempfile::TempDir,
1986 home: PathBuf,
1987 repo: PathBuf,
1988 bay: PathBuf,
1989 base: String,
1990 }
1991 impl Fx {
1992 async fn new() -> Self {
1993 let temp = tempfile::tempdir().unwrap();
1994 let root = temp.path().canonicalize().unwrap();
1995 let home = root.join("home");
1996 let repo = root.join("repo");
1997 let bay = root.join("worktrees");
1998 std::fs::create_dir_all(&repo).unwrap();
1999 std::fs::create_dir_all(home.join("runs")).unwrap();
2000 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
2001 git::git(&repo, &["config", "user.name", "Fixture"])
2002 .await
2003 .unwrap();
2004 git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
2005 .await
2006 .unwrap();
2007 git::git(&repo, &["config", "commit.gpgsign", "false"])
2008 .await
2009 .unwrap();
2010 std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
2011 std::fs::write(repo.join("source"), "base\n").unwrap();
2012 git::git(&repo, &["add", "."]).await.unwrap();
2013 git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
2014 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
2015 Self {
2016 _temp: temp,
2017 home,
2018 repo,
2019 bay,
2020 base,
2021 }
2022 }
2023 async fn run(&self, tag: &str, useful: bool) -> RunState {
2024 let mut cfg = Config::default();
2025 cfg.graph.worktree_root = Some(self.bay.clone());
2026 let mut state = RunState::new(
2027 self.repo.clone(),
2028 "main".into(),
2029 self.base.clone(),
2030 "task".into(),
2031 cfg,
2032 );
2033 state.id = format!("20260101-000000-{tag}");
2034 state.status = RunStatus::Blocked;
2035 state.driver_exited = true;
2036 let wt = state.worktree_root().join("cand-A");
2037 let branch = format!("magi/{tag}/A");
2038 git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
2039 .await
2040 .unwrap();
2041 if useful {
2042 std::fs::write(wt.join("source"), tag).unwrap();
2043 git::git(&wt, &["add", "."]).await.unwrap();
2044 git::git(&wt, &["commit", "-m", "candidate change"])
2045 .await
2046 .unwrap();
2047 }
2048 state.candidates.push(Candidate {
2049 index: 0,
2050 label: 'A',
2051 agent: "fixture".into(),
2052 branch,
2053 worktree: wt,
2054 summary: String::new(),
2055 stat: String::new(),
2056 files: usize::from(useful),
2057 commits: usize::from(useful),
2058 empty: !useful,
2059 failed: None,
2060 verified_noop: None,
2061 duration_ms: 0,
2062 folded: false,
2063 });
2064 state.save_under(&self.home).unwrap();
2065 state
2066 }
2067 fn disk() -> Disk {
2068 Disk {
2069 worktree_limit_bytes: 1,
2070 worktree_reclaim_bytes: 1024 * 1024,
2071 fold_grace_secs: 0,
2072 min_free_bytes: 0,
2073 ..Disk::default()
2074 }
2075 }
2076 fn target(wt: &Path) {
2077 let target = wt.join("target");
2078 std::fs::create_dir_all(target.join("debug")).unwrap();
2079 let mut artifact = [1; 200];
2080 artifact[..4].copy_from_slice(b"\x7fELF");
2081 std::fs::write(target.join("debug/output"), artifact).unwrap();
2082 let output = target
2083 .join("debug/output")
2084 .display()
2085 .to_string()
2086 .replace('\\', "\\\\")
2087 .replace(' ', "\\ ");
2088 std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
2089 std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
2090 std::fs::write(
2091 target.join("CACHEDIR.TAG"),
2092 "Signature: 8a477f597d28d172789f06886806bc55\n",
2093 )
2094 .unwrap();
2095 std::fs::write(
2096 target.join(".rustc_info.json"),
2097 "{\"rustc_fingerprint\":123}",
2098 )
2099 .unwrap();
2100 }
2101 fn now() -> Timestamp {
2102 Timestamp::now() + jiff::SignedDuration::from_secs(2)
2103 }
2104 fn task(&self, runs: &[RunState]) -> Task {
2105 let mut task = Task::new(
2106 "fixture".into(),
2107 "task".into(),
2108 self.repo.clone(),
2109 Source::Human,
2110 );
2111 task.runs = runs.iter().map(|s| s.id.clone()).collect();
2112 crate::queue::Queue::at(self.home.join("queue"))
2113 .put(&mut task)
2114 .unwrap();
2115 task
2116 }
2117 }
2118
2119 #[cfg(unix)]
2120 #[tokio::test]
2121 async fn incremental_only_reclamation_preserves_legacy_and_unknown_data() {
2122 let f = Fx::new().await;
2123 let state = f.run("ca01", true).await;
2124 let wt = &state.candidates[0].worktree;
2125 let generation = crate::incremental::tests::generate(wt);
2126 assert!(!declared_outputs(&wt.join("target")).unwrap().is_empty());
2128 for dir in profile_output_dirs(&wt.join("target/debug"), true).unwrap() {
2129 for entry in std::fs::read_dir(dir).unwrap() {
2130 let path = entry.unwrap().path();
2131 if path.extension().is_some_and(|x| x == "d") {
2132 std::fs::remove_file(path).unwrap();
2133 }
2134 }
2135 }
2136 assert!(declared_outputs(&wt.join("target")).unwrap().is_empty());
2137 assert_eq!(
2138 sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes,
2139 0,
2140 "live generation is protected"
2141 );
2142 drop(generation);
2143 let candidates = crate::incremental::candidates(wt).unwrap();
2144 let logical: u64 = candidates
2145 .files
2146 .iter()
2147 .map(|p| std::fs::metadata(p).unwrap().len())
2148 .sum();
2149 let unknown = candidates.files[0].parent().unwrap().join("operator-note");
2150 std::fs::write(&unknown, "retain unknown data").unwrap();
2151 drop(candidates);
2152 let refused = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2153 assert_eq!(refused.freed_bytes, 0, "{refused:?}");
2154 assert!(
2155 refused
2156 .retained
2157 .iter()
2158 .any(|s| s.contains("provenance mismatch"))
2159 );
2160 std::fs::remove_file(&unknown).unwrap();
2161 let legacy = wt.join("target/debug/incremental/legacy-session");
2162 std::fs::create_dir(&legacy).unwrap();
2163 let candidates = crate::incremental::candidates(wt).unwrap();
2165 let bin = candidates
2166 .files
2167 .iter()
2168 .find(|p| p.extension().is_some_and(|x| x == "bin"))
2169 .unwrap();
2170 std::fs::copy(bin, legacy.join("query-cache.bin")).unwrap();
2171 drop(candidates);
2172 let cargo = cargo_locks(&wt.join("target")).unwrap();
2173 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2174 drop(cargo);
2175 let outputs = OutputLease::borrow(wt).unwrap().unwrap();
2176 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2177 drop(outputs);
2178 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2179 assert_eq!(report.freed_bytes, logical, "{report:?}");
2180 assert_eq!(report.proven_reclaimable_bytes, logical);
2181 assert!(report.measured_bytes > logical);
2182 assert_eq!(
2183 report.protected_or_unproven_bytes,
2184 report.measured_bytes - logical
2185 );
2186 #[cfg(unix)]
2187 {
2188 assert!(report.estimated_released_allocated_bytes.unwrap() > 0);
2189 assert!(
2190 report.measured_allocated_file_bytes.unwrap()
2191 >= report.estimated_released_allocated_bytes.unwrap()
2192 );
2193 }
2194 assert!(report.observed_home_free_change_bytes.is_some());
2195 assert!(legacy.join("query-cache.bin").is_file());
2196 assert!(wt.join("src/lib.rs").is_file());
2197 assert_eq!(report.retired, 0);
2198 }
2199
2200 #[cfg(unix)]
2201 #[tokio::test]
2202 async fn incremental_and_deps_links_account_for_retained_third_link() {
2203 use std::os::unix::fs::MetadataExt;
2204 let f = Fx::new().await;
2205 let state = f.run("ca02", true).await;
2206 let wt = &state.candidates[0].worktree;
2207 let generation = crate::incremental::tests::generate(wt);
2208 drop(generation);
2209 let candidates = crate::incremental::candidates(wt).unwrap();
2210 let objects = candidates
2211 .files
2212 .iter()
2213 .filter(|p| p.extension().is_some_and(|x| x == "o"))
2214 .take(2)
2215 .cloned()
2216 .collect::<Vec<_>>();
2217 assert_eq!(
2218 objects.len(),
2219 2,
2220 "fixture must produce real incremental objects"
2221 );
2222 let mut outputs = candidates.files.clone();
2223 drop(candidates);
2224 let target = wt.join("target");
2225 assert!(!declared_outputs(&target).unwrap().is_empty());
2226 let generated_dirs = profile_output_dirs(&target.join("debug"), true).unwrap();
2227 let mut output_dir = None;
2230 for dir in &generated_dirs {
2231 for entry in std::fs::read_dir(dir).unwrap() {
2232 let path = entry.unwrap().path();
2233 if path.extension().is_some_and(|x| x == "d") {
2234 std::fs::remove_file(path).unwrap();
2235 if dir != &generated_dirs[0] {
2236 output_dir = Some(dir.clone());
2237 }
2238 }
2239 }
2240 }
2241 let output_dir = output_dir.expect("Cargo generated no recognised output directory");
2242 for entry in std::fs::read_dir(&output_dir).unwrap() {
2246 let path = entry.unwrap().path();
2247 if path.is_file() {
2248 std::fs::remove_file(path).unwrap();
2249 }
2250 }
2251 let stem = if output_dir.file_name().is_some_and(|n| n == "out") {
2254 let unit = output_dir.parent().unwrap();
2255 let hash = unit.file_name().unwrap().to_string_lossy().into_owned();
2256 let name = std::fs::read_dir(unit.join("fingerprint"))
2257 .unwrap()
2258 .filter_map(|e| e.unwrap().file_name().into_string().ok())
2259 .find_map(|n| {
2260 n.strip_prefix("lib-")
2261 .filter(|n| !n.contains('.'))
2262 .map(str::to_owned)
2263 })
2264 .expect("Cargo unit has a lib fingerprint");
2265 format!("{}-{hash}", name.replace('-', "_"))
2266 } else {
2267 "link".to_owned()
2268 };
2269 let mut declared = Vec::new();
2272 for (index, object) in objects.iter().enumerate() {
2273 assert_eq!(std::fs::metadata(object).unwrap().nlink(), 1);
2274 let dep = output_dir.join(if index == 0 {
2275 stem.clone()
2276 } else {
2277 format!("lib{stem}")
2278 });
2279 std::fs::hard_link(object, &dep).unwrap();
2280 declared.push(dep.display().to_string());
2281 outputs.push(dep);
2282 }
2283 std::fs::write(
2284 output_dir.join(format!("{stem}.d")),
2285 format!("{}: src/lib.rs\n", declared.join(" ")),
2286 )
2287 .unwrap();
2288 let retained = wt.join("retained-input");
2289 std::fs::hard_link(&objects[1], &retained).unwrap();
2290 let sentinel = std::fs::read(&retained).unwrap();
2291 let logical: u64 = outputs
2292 .iter()
2293 .map(|p| std::fs::metadata(p).unwrap().len())
2294 .sum();
2295 let mut seen = std::collections::BTreeSet::new();
2296 let retained_meta = std::fs::metadata(&retained).unwrap();
2297 let retained_inode = (retained_meta.dev(), retained_meta.ino());
2298 let mut expected = 0u64;
2299 for output in &outputs {
2300 let m = std::fs::metadata(output).unwrap();
2301 let inode = (m.dev(), m.ino());
2302 if inode != retained_inode && seen.insert(inode) {
2303 expected += m.blocks() * 512;
2304 }
2305 }
2306 assert!(expected > 0);
2307 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2308 assert_eq!(report.proven_reclaimable_bytes, logical, "{report:?}");
2309 assert_eq!(report.freed_bytes, logical, "{report:?}");
2310 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(expected));
2311 assert_eq!(report.estimated_released_allocated_bytes, Some(expected));
2312 assert!(outputs.iter().all(|p| !p.exists()));
2313 assert_eq!(std::fs::read(retained).unwrap(), sentinel);
2314 assert!(wt.join("src/lib.rs").is_file());
2315 }
2316
2317 #[tokio::test]
2318 async fn retention_notice_explains_unproven_inventory_and_live_owner() {
2319 let f = Fx::new().await;
2320 let state = f.run("ca03", true).await;
2321 let wt = &state.candidates[0].worktree;
2322 Fx::target(wt);
2323 std::fs::remove_file(wt.join("target/debug/output")).unwrap();
2324 let unknown = wt.join("target/debug/operator-note");
2325 std::fs::write(&unknown, vec![1; 4096]).unwrap();
2326 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2327 assert_eq!(report.proven_reclaimable_bytes, 0);
2328 #[cfg(unix)]
2329 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(0));
2330 assert!(report.protected_or_unproven_bytes >= 4096);
2331 let message = || {
2332 crate::notices::Notices::at(f.home.join("notifications"))
2333 .list()
2334 .into_iter()
2335 .find(|n| n.key == "disk:retention")
2336 .unwrap()
2337 .message
2338 };
2339 let notice = message();
2340 assert!(notice.contains("proven reclaimable 0B"), "{notice}");
2341 assert!(notice.contains("protected or unproven"), "{notice}");
2342 assert!(
2343 notice.contains("legacy incremental/build and unknown files retained"),
2344 "{notice}"
2345 );
2346 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2347 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2348 assert_eq!(report.freed_bytes, 0);
2349 let notice = message();
2350 assert!(notice.contains(&state.id), "{notice}");
2351 assert!(
2352 notice.contains("lock") || notice.contains("owner"),
2353 "{notice}"
2354 );
2355 drop(owner);
2356 assert_eq!(std::fs::read(unknown).unwrap(), vec![1; 4096]);
2357 }
2358
2359 #[tokio::test]
2360 async fn real_cargo_build_holds_the_target_against_reclamation() {
2361 let f = Fx::new().await;
2362 let state = f.run("ca02", true).await;
2363 let wt = &state.candidates[0].worktree;
2364 Fx::target(wt);
2365 std::fs::create_dir(wt.join("src")).unwrap();
2366 std::fs::write(
2367 wt.join("Cargo.toml"),
2368 "[package]\nname='live-cache-fixture'\nversion='0.1.0'\nedition='2021'\n[workspace]\n",
2369 )
2370 .unwrap();
2371 std::fs::write(wt.join("src/lib.rs"), "pub fn answer() -> usize { 42 }\n").unwrap();
2372 std::fs::write(
2373 wt.join("build.rs"),
2374 r#"fn main() {
2375 println!("cargo:rerun-if-env-changed=FIXTURE_READY");
2376 let Ok(ready) = std::env::var("FIXTURE_READY") else { return };
2377 let release = std::env::var("FIXTURE_RELEASE").unwrap();
2378 std::fs::write(ready, "ready").unwrap();
2379 let start = std::time::Instant::now();
2380 while !std::path::Path::new(&release).exists() {
2381 assert!(start.elapsed().as_secs() < 120, "fixture was not released");
2382 std::thread::sleep(std::time::Duration::from_millis(10));
2383 }
2384 }"#,
2385 )
2386 .unwrap();
2387 let ready = wt.join("ready");
2388 let release = wt.join("release");
2389 let mut cargo = tokio::process::Command::new("cargo");
2390 cargo.env_clear();
2391 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2392 if let Some(value) = std::env::var_os(key) {
2393 cargo.env(key, value);
2394 }
2395 }
2396 cargo
2397 .current_dir(wt)
2398 .env("RUSTUP_TOOLCHAIN", "stable")
2399 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2400 .env("CARGO_INCREMENTAL", "1")
2401 .args(["build", "--offline", "--jobs", "1", "--target-dir"])
2402 .arg(wt.join("target"));
2403 let warm = cargo.output().await.unwrap();
2407 assert!(
2408 warm.status.success(),
2409 "fixture compilation failed: {}",
2410 String::from_utf8_lossy(&warm.stderr)
2411 );
2412 let log_path = f._temp.path().join("live-cargo.stderr");
2413 let mut child = cargo
2414 .env("FIXTURE_READY", &ready)
2415 .env("FIXTURE_RELEASE", &release)
2416 .stdout(std::process::Stdio::null())
2417 .stderr(std::fs::File::create(&log_path).unwrap())
2418 .kill_on_drop(true)
2419 .spawn()
2420 .unwrap();
2421 let readiness = tokio::time::timeout(std::time::Duration::from_secs(60), async {
2422 while !ready.exists() {
2423 if let Some(status) = child.try_wait().unwrap() {
2424 return Err(format!("fixture exited before readiness: {status}"));
2425 }
2426 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2427 }
2428 Ok(())
2429 })
2430 .await;
2431 let arrived = matches!(readiness, Ok(Ok(())));
2432 let report = if arrived {
2433 Some(sweep(&f.home, &Fx::disk(), Fx::now()).await)
2434 } else {
2435 None
2436 };
2437 std::fs::write(release, "release").unwrap();
2439 let status = tokio::time::timeout(std::time::Duration::from_secs(60), child.wait()).await;
2440 if status.is_err() {
2441 child.kill().await.unwrap();
2442 child.wait().await.unwrap();
2443 }
2444 assert!(
2445 arrived,
2446 "fixture build never reached its active build script: {readiness:?}\n{}",
2447 std::fs::read_to_string(&log_path).unwrap()
2448 );
2449 assert!(
2450 status.unwrap().unwrap().success(),
2451 "fixture build failed: {}",
2452 std::fs::read_to_string(&log_path).unwrap()
2453 );
2454 let report = report.unwrap();
2455 assert_eq!(report.freed_bytes, 0, "{report:?}");
2456 assert!(
2457 report
2458 .retained
2459 .iter()
2460 .any(|s| s.contains("Cargo build is active")),
2461 "{report:?}"
2462 );
2463 assert!(wt.join("target/debug/output").is_file());
2464 assert!(crate::disk::dir_size(&wt.join("target/debug/incremental")) > 0);
2465 }
2466
2467 #[cfg(unix)]
2468 #[test]
2469 fn allocated_accounting_deduplicates_links_and_requires_the_last_link() {
2470 let temp = tempfile::tempdir().unwrap();
2471 let a = temp.path().join("a");
2472 let b = temp.path().join("b");
2473 let outside = temp.path().join("retained-input");
2474 std::fs::write(&a, vec![1; 65536]).unwrap();
2475 std::fs::hard_link(&a, &b).unwrap();
2476 std::fs::hard_link(&a, &outside).unwrap();
2477 let mut seen = Default::default();
2478 let blocks = allocated_inventory(&temp.path().canonicalize().unwrap(), &mut seen).unwrap();
2479 assert!(blocks > 0);
2480 let mut accounting = ReleasedAllocation::of(&[a.clone(), b.clone()]).unwrap();
2481 std::fs::remove_file(&a).unwrap();
2482 assert_eq!(accounting.unlinked(&a), Some(0));
2483 std::fs::remove_file(&b).unwrap();
2484 assert_eq!(accounting.unlinked(&b), Some(0));
2485 assert_eq!(std::fs::read(&outside).unwrap().len(), 65536);
2486 let mut accounting = ReleasedAllocation::of(std::slice::from_ref(&outside)).unwrap();
2487 std::fs::remove_file(&outside).unwrap();
2488 assert_eq!(accounting.unlinked(&outside), Some(blocks));
2489 }
2490
2491 #[cfg(unix)]
2492 #[test]
2493 fn relative_admission_home_matches_absolute_home_and_retains_symlink_checks() {
2494 let temp = tempfile::tempdir_in(".").unwrap();
2495 let home = PathBuf::from(temp.path().file_name().unwrap()).join("state");
2496 assert!(home.is_relative());
2497 let paths = vec![std::env::current_dir().unwrap()];
2498 let gib = 1024 * 1024 * 1024;
2499 let reservation = OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).unwrap();
2500 assert!(
2501 OutputLease::admit_with(&home.canonicalize().unwrap(), &paths, 8 * gib, |_| Ok(
2502 9 * gib
2503 ))
2504 .is_err()
2505 );
2506 drop(reservation);
2507 assert!(OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).is_ok());
2508 let bad = temp.path().join("symlink-state");
2509 std::fs::create_dir(&bad).unwrap();
2510 std::os::unix::fs::symlink(
2511 home.canonicalize().unwrap().join("disk-admission"),
2512 bad.join("disk-admission"),
2513 )
2514 .unwrap();
2515 assert!(OutputLease::admit_with(&bad, &paths, 8 * gib, |_| Ok(9 * gib)).is_err());
2516 }
2517
2518 #[cfg(unix)]
2519 #[test]
2520 fn concurrent_disk_admission_reserves_once_per_volume_until_release() {
2521 let temp = tempfile::tempdir().unwrap();
2522 let home = temp.path().canonicalize().unwrap();
2523 let paths = vec![home.clone(), home.join("not-created-yet")];
2524 let gib = 1024 * 1024 * 1024;
2525 let floor = 8 * gib;
2526 let first = OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap();
2527 assert_eq!(first.len(), 1);
2528 let blocked =
2529 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap_err();
2530 assert!(blocked.to_string().contains("active writer reservations"));
2531 let second =
2532 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + 2 * gib)).unwrap();
2533 drop(first);
2534 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_err());
2535 drop(second);
2536 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_ok());
2537 assert!(
2538 OutputLease::admit_with(&home, &paths, floor, |_| anyhow::bail!("unmeasured")).is_err()
2539 );
2540 }
2541
2542 #[cfg(unix)]
2543 #[test]
2544 fn simultaneous_writers_cannot_spend_the_same_volume_headroom() {
2545 let temp = tempfile::tempdir().unwrap();
2546 let home = temp.path().canonicalize().unwrap();
2547 let barrier = std::sync::Barrier::new(2);
2548 let gib = 1024 * 1024 * 1024;
2549 let results = std::thread::scope(|scope| {
2550 let threads: Vec<_> = (0..2)
2551 .map(|_| {
2552 scope.spawn(|| {
2553 barrier.wait();
2554 OutputLease::admit_with(&home, std::slice::from_ref(&home), 100, |_| {
2555 Ok(100 + gib)
2556 })
2557 })
2558 })
2559 .collect();
2560 threads
2561 .into_iter()
2562 .map(|thread| thread.join().unwrap())
2563 .collect::<Vec<_>>()
2564 });
2565 assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
2566 assert!(
2567 results
2568 .iter()
2569 .filter_map(|r| r.as_ref().err())
2570 .all(|e| e.to_string().contains("active writer reservations"))
2571 );
2572 }
2573
2574 #[tokio::test]
2575 async fn sweep_reclaims_real_cargo_outputs_and_preserves_source_and_unknown_files() {
2576 real_cargo_fixture(false).await;
2577 }
2578
2579 #[tokio::test]
2580 async fn missing_tag_from_precreated_target_reclaims_only_real_cargo_artifacts() {
2581 real_cargo_fixture(true).await;
2582 }
2583
2584 async fn real_cargo_fixture(precreate: bool) {
2585 let f = Fx::new().await;
2586 let state = f.run("ca00", true).await;
2587 let wt = &state.candidates[0].worktree;
2588 let source = "fn main() { println!(\"retained fixture\"); }\n";
2589 std::fs::create_dir(wt.join("src")).unwrap();
2590 std::fs::write(wt.join("src/main.rs"), source).unwrap();
2591 std::fs::write(
2592 wt.join("Cargo.toml"),
2593 "[package]\nname = \"retained-fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n",
2594 )
2595 .unwrap();
2596 if precreate {
2597 std::fs::create_dir_all(wt.join("target/magi-tools")).unwrap();
2598 }
2599 let mut cargo = std::process::Command::new("cargo");
2600 cargo.env_clear();
2601 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2602 if let Some(value) = std::env::var_os(key) {
2603 cargo.env(key, value);
2604 }
2605 }
2606 let build = cargo
2607 .current_dir(wt)
2608 .env("RUSTUP_TOOLCHAIN", "stable")
2609 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2610 .env("CARGO_INCREMENTAL", "0")
2611 .arg("--config")
2612 .arg(format!(
2613 "build.build-dir={}",
2614 serde_json::to_string(&wt.join("target")).unwrap()
2615 ))
2616 .args([
2617 "build",
2618 "--offline",
2619 "--release",
2620 "--jobs",
2621 "1",
2622 "--target-dir",
2623 ])
2624 .arg(wt.join("target"))
2625 .output()
2626 .unwrap();
2627 assert!(
2628 build.status.success(),
2629 "{}",
2630 String::from_utf8_lossy(&build.stderr)
2631 );
2632 assert_eq!(wt.join("target/CACHEDIR.TAG").exists(), !precreate);
2633 if precreate {
2634 let again = cargo.output().unwrap();
2636 assert!(
2637 again.status.success(),
2638 "{}",
2639 String::from_utf8_lossy(&again.stderr)
2640 );
2641 assert!(!wt.join("target/CACHEDIR.TAG").exists());
2642 assert!(cargo_target(wt).await.is_err());
2643 }
2644 git::git(wt, &["add", "Cargo.toml", "Cargo.lock", "src"])
2645 .await
2646 .unwrap();
2647 git::git(wt, &["commit", "-m", "real Cargo fixture"])
2648 .await
2649 .unwrap();
2650 let artifact = wt.join("target/release/retained-fixture");
2651 assert!(artifact.is_file());
2652 let unknown = wt.join("target/release/operator-note");
2653 std::fs::write(&unknown, "keep this note\n").unwrap();
2654
2655 let target = wt.join("target");
2656 let mut preserved = std::collections::BTreeMap::new();
2657 for relative in [
2658 "tmp/sentinel",
2659 "magi-tools/sentinel",
2660 "lifecycle-client-check.log",
2661 ] {
2662 let path = target.join(relative);
2663 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
2664 std::fs::write(&path, format!("retain {relative}\n")).unwrap();
2665 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2666 }
2667 let decoy = target.join("magi-tools/retained-fixture");
2670 std::fs::copy(&artifact, &decoy).unwrap();
2671 let dep_info = decoy.with_extension("d");
2672 std::fs::write(&dep_info, format!("{}: src/main.rs\n", decoy.display())).unwrap();
2673 for path in [
2674 decoy,
2675 dep_info,
2676 wt.join("Cargo.toml"),
2677 wt.join("Cargo.lock"),
2678 ] {
2679 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2680 }
2681 let supported_dirs = profile_output_dirs(&target.join("release"), true).unwrap();
2682 let proven = declared_outputs(&target).unwrap();
2683 assert!(
2684 proven
2685 .iter()
2686 .any(|path| path.parent() != Some(target.join("release").as_path())),
2687 "both uplifted and unit/deps binary references must be proven: {proven:?}"
2688 );
2689
2690 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2691 assert!(out.freed_bytes > 0, "{out:?}");
2692 for (path, bytes) in preserved {
2693 assert_eq!(std::fs::read(&path).unwrap(), bytes, "{}", path.display());
2694 }
2695 assert!(!artifact.exists(), "{out:?}");
2696 assert_eq!(
2697 std::fs::read_to_string(wt.join("src/main.rs")).unwrap(),
2698 source
2699 );
2700 assert_eq!(
2701 std::fs::read_to_string(&unknown).unwrap(),
2702 "keep this note\n"
2703 );
2704 assert_eq!(std::fs::read_to_string(wt.join("source")).unwrap(), "ca00");
2705 let ledger: serde_json::Value = serde_json::from_slice(
2706 &std::fs::read(
2707 f.home
2708 .join("runs")
2709 .join(&state.id)
2710 .join("build-outputs.json"),
2711 )
2712 .unwrap(),
2713 )
2714 .unwrap();
2715 assert_eq!(
2716 ledger["identity"],
2717 if precreate {
2718 "rustc-profile-dep-info"
2719 } else {
2720 "cache-tag"
2721 }
2722 );
2723 if precreate {
2724 assert!(!target.join("CACHEDIR.TAG").exists());
2725 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2726 }
2727 let deleted: Vec<PathBuf> = serde_json::from_value(ledger["files"].clone()).unwrap();
2728 assert!(deleted.contains(&artifact));
2729 assert_eq!(deleted, proven);
2730 for path in deleted {
2731 assert!(!path.exists());
2732 assert!(supported_dirs.contains(&path.parent().unwrap().to_owned()));
2733 }
2734 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2735 }
2736
2737 fn missing_tag_target(wt: &Path) {
2738 Fx::target(wt);
2739 let target = wt.join("target");
2740 std::fs::remove_file(target.join("CACHEDIR.TAG")).unwrap();
2741 for subdir in ["debug/deps", "debug/.fingerprint"] {
2742 std::fs::create_dir_all(target.join(subdir)).unwrap();
2743 }
2744 std::fs::write(target.join(".rustc_info.json"),
2745 r#"{"rustc_fingerprint":123,"outputs":{"1":{"success":true,"stdout":"rustc fixture","stderr":""}},"successes":{}}"#).unwrap();
2746 }
2747
2748 fn v2_target(wt: &Path) -> PathBuf {
2750 missing_tag_target(wt);
2751 let target = wt.join("target");
2752 std::fs::remove_dir(target.join("debug/deps")).unwrap();
2753 std::fs::remove_dir(target.join("debug/.fingerprint")).unwrap();
2754 let unit = target.join("debug/build/fixture/0123456789abcdef");
2755 let out = unit.join("out");
2756 std::fs::create_dir_all(&out).unwrap();
2757 std::fs::create_dir(unit.join("fingerprint")).unwrap();
2758 for name in [
2759 "lib-fixture",
2760 "lib-fixture.json",
2761 "dep-lib-fixture",
2762 "invoked.timestamp",
2763 ] {
2764 std::fs::write(unit.join("fingerprint").join(name), "fixture").unwrap();
2765 }
2766 let artifact = out.join("libfixture-0123456789abcdef.rlib");
2767 std::fs::copy(target.join("debug/output"), &artifact).unwrap();
2768 std::fs::write(
2769 out.join("fixture-0123456789abcdef.d"),
2770 format!("{}: source\n", artifact.display()),
2771 )
2772 .unwrap();
2773 artifact
2774 }
2775
2776 #[tokio::test]
2777 async fn missing_tag_v2_proves_only_compiler_units_and_preserves_retained_links() {
2778 let f = Fx::new().await;
2779 let state = f.run("bc01", true).await;
2780 let wt = &state.candidates[0].worktree;
2781 let artifact = v2_target(wt);
2782 let unknown = artifact.parent().unwrap().join("operator-note");
2783 std::fs::write(&unknown, "retain").unwrap();
2784 let script_unit = wt.join("target/debug/build/fixture/fedcba9876543210");
2786 let script = script_unit.join("out");
2787 std::fs::create_dir_all(&script).unwrap();
2788 std::fs::create_dir(script_unit.join("fingerprint")).unwrap();
2789 std::fs::write(script_unit.join("fingerprint/invoked.timestamp"), "fixture").unwrap();
2790 let decoy = script.join("libfixture-fedcba9876543210.rlib");
2791 std::fs::copy(&artifact, &decoy).unwrap();
2792 std::fs::write(
2793 script.join("fixture-fedcba9876543210.d"),
2794 format!("{}: source\n", decoy.display()),
2795 )
2796 .unwrap();
2797 let uplift = wt.join("target/debug/libfixture.rlib");
2799 std::fs::hard_link(&artifact, &uplift).unwrap();
2800 std::fs::write(
2801 wt.join("target/debug/fixture.d"),
2802 format!("{}: source\n", uplift.display()),
2803 )
2804 .unwrap();
2805 let bin_unit = wt.join("target/debug/build/fixture/1111111111111111");
2806 std::fs::create_dir_all(bin_unit.join("out")).unwrap();
2807 std::fs::create_dir(bin_unit.join("fingerprint")).unwrap();
2808 for name in [
2809 "bin-fixture",
2810 "bin-fixture.json",
2811 "dep-bin-fixture",
2812 "invoked.timestamp",
2813 ] {
2814 std::fs::write(bin_unit.join("fingerprint").join(name), "fixture").unwrap();
2815 }
2816 let bin = bin_unit.join("out/fixture");
2817 std::fs::hard_link(wt.join("target/debug/output"), &bin).unwrap();
2818 std::fs::write(
2819 bin.with_extension("d"),
2820 format!("{}: source\n", bin.display()),
2821 )
2822 .unwrap();
2823 let retained = wt.join("retained-link");
2824 std::fs::hard_link(&artifact, &retained).unwrap();
2825 use std::os::unix::fs::MetadataExt;
2826 let expected_allocation = std::fs::metadata(wt.join("target/debug/output"))
2827 .unwrap()
2828 .blocks()
2829 * 512;
2830 let sentinel = std::fs::read(&retained).unwrap();
2831 let first = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2832 assert_eq!(first.freed_bytes, 800, "{first:?}");
2833 assert!(!bin.exists());
2834 assert_eq!(
2835 first.estimated_released_allocated_bytes,
2836 Some(expected_allocation)
2837 );
2838 assert_eq!(std::fs::read(&retained).unwrap(), sentinel);
2839 assert!(retained.is_file());
2840 assert!(unknown.is_file());
2841 assert!(decoy.is_file());
2842 assert!(!artifact.exists());
2843 assert_eq!(std::fs::metadata(retained).unwrap().len(), 200);
2845 }
2846
2847 #[tokio::test]
2848 async fn missing_tag_v2_accepts_completed_build_script_run_units() {
2849 let f = Fx::new().await;
2850 let state = f.run("bc02", true).await;
2851 let wt = &state.candidates[0].worktree;
2852 let artifact = v2_target(wt);
2853 let run = wt.join("target/debug/build/fixture/2222222222222222");
2854 std::fs::create_dir_all(run.join("out")).unwrap();
2855 std::fs::create_dir(run.join("fingerprint")).unwrap();
2856 for name in [
2857 "run-build-script-build-script-build",
2858 "run-build-script-build-script-build.json",
2859 ] {
2860 std::fs::write(run.join("fingerprint").join(name), "fixture").unwrap();
2861 }
2862 let decoy = run.join("out/libdecoy-2222222222222222.rlib");
2864 std::fs::copy(&artifact, &decoy).unwrap();
2865 std::fs::write(
2866 run.join("out/decoy-2222222222222222.d"),
2867 format!("{}: source\n", decoy.display()),
2868 )
2869 .unwrap();
2870 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2871 assert!(report.freed_bytes > 0, "{report:?}");
2872 assert!(!artifact.exists());
2873 assert!(decoy.is_file());
2874 }
2875
2876 #[tokio::test]
2877 async fn missing_tag_v2_rejects_incomplete_mixed_and_symlinked_layouts() {
2878 for invalid in [
2879 "build",
2880 "deps",
2881 ".fingerprint",
2882 "out",
2883 "fingerprint",
2884 "hash",
2885 "mixed",
2886 "unit-fingerprint",
2887 "compiler-fingerprint",
2888 "absent-out",
2889 ] {
2890 let f = Fx::new().await;
2891 let state = f.run("bc02", true).await;
2892 let wt = &state.candidates[0].worktree;
2893 let artifact = v2_target(wt);
2894 let unit = artifact.parent().unwrap().parent().unwrap();
2895 let profile = wt.join("target/debug");
2896 match invalid {
2897 "mixed" => {
2898 std::fs::create_dir(profile.join("deps")).unwrap();
2899 std::fs::create_dir(profile.join(".fingerprint")).unwrap();
2900 }
2901 "hash" => {
2902 std::fs::rename(unit, unit.with_file_name("unknown")).unwrap();
2903 }
2904 "unit-fingerprint" => {
2905 std::fs::remove_file(unit.join("fingerprint/invoked.timestamp")).unwrap();
2906 }
2907 "compiler-fingerprint" => {
2908 std::fs::remove_file(unit.join("fingerprint/lib-fixture.json")).unwrap();
2909 }
2910 "absent-out" => {
2911 std::fs::remove_dir_all(unit.join("out")).unwrap();
2912 }
2913 name => {
2914 let path = if matches!(name, "out" | "fingerprint") {
2915 unit.join(name)
2916 } else {
2917 profile.join(name)
2918 };
2919 if path.exists() {
2920 std::fs::remove_dir_all(&path).unwrap();
2921 }
2922 std::os::unix::fs::symlink(&f.repo, &path).unwrap();
2923 }
2924 }
2925 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2926 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
2927 assert!(!report.retained.is_empty(), "{invalid}: {report:?}");
2928 }
2929 }
2930
2931 #[tokio::test]
2932 async fn bookkeeping_flushes_do_not_extend_artifact_grace_including_legacy_records() {
2933 for legacy in [false, true] {
2934 let f = Fx::new().await;
2935 let mut state = f.run("ac00", true).await;
2936 Fx::target(&state.candidates[0].worktree);
2937 let original = state.updated_at;
2938 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2939 if legacy {
2940 let path = f.home.join("runs").join(&state.id).join("run.json");
2941 let mut value = serde_json::to_value(&state).unwrap();
2942 value.as_object_mut().unwrap().remove("output_activity_at");
2943 std::fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
2944 state = RunState::load_under(&state.id, &f.home).unwrap();
2945 assert_eq!(state.output_activity(), original);
2946 }
2947 let age = state.output_activity();
2948 for offset in [30, 20, 10] {
2949 state
2950 .save_under_at(&f.home, now - jiff::SignedDuration::from_secs(offset))
2951 .unwrap();
2952 state = RunState::load_under(&state.id, &f.home).unwrap();
2953 assert_eq!(state.output_activity(), age);
2954 }
2955 let mut disk = Fx::disk();
2956 disk.fold_grace_secs = 3600;
2957 let report = sweep(&f.home, &disk, now).await;
2958 assert_eq!(report.freed_bytes, 200, "legacy={legacy}: {report:?}");
2959 }
2960 }
2961
2962 #[tokio::test]
2963 async fn recent_execution_outputs_and_released_output_leases_retain_artifacts() {
2964 for evidence in [
2965 "execution",
2966 "artifact",
2967 "dep-info",
2968 "fingerprint",
2969 "lease",
2970 "future",
2971 ] {
2972 let f = Fx::new().await;
2973 let mut state = f.run("ac01", true).await;
2974 let wt = state.candidates[0].worktree.clone();
2975 missing_tag_target(&wt);
2976 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2977 let recent = now - jiff::SignedDuration::from_secs(10);
2978 match evidence {
2979 "execution" => {
2980 state.note_output_activity(recent);
2981 state.save_under_at(&f.home, recent).unwrap();
2982 }
2983 "lease" => {
2984 let lease = OutputLease::borrow(&wt).unwrap().unwrap();
2985 let file = lease._file.try_clone().unwrap();
2986 drop(lease);
2987 file.set_modified(std::time::SystemTime::from(recent))
2988 .unwrap();
2989 }
2990 _ => {
2991 let relative = match evidence {
2992 "dep-info" => "debug/output.d",
2993 "fingerprint" => "debug/.fingerprint/invoked.timestamp",
2994 _ => "debug/output",
2995 };
2996 let path = wt.join("target").join(relative);
2997 let file = OpenOptions::new()
2998 .write(true)
2999 .create(true)
3000 .truncate(false)
3001 .open(path)
3002 .unwrap();
3003 let at = if evidence == "future" {
3004 now + jiff::SignedDuration::from_secs(10)
3005 } else {
3006 recent
3007 };
3008 file.set_modified(std::time::SystemTime::from(at)).unwrap();
3009 }
3010 }
3011 let mut disk = Fx::disk();
3012 disk.fold_grace_secs = 3600;
3013 let report = sweep(&f.home, &disk, now).await;
3014 assert_eq!(report.freed_bytes, 0, "{evidence}: {report:?}");
3015 assert!(
3016 report.retained.iter().any(|s| s.contains("grace period")
3017 || s.contains("execution activity after inventory")),
3018 "{evidence}: {report:?}"
3019 );
3020 let aged = now + jiff::SignedDuration::from_secs(7200);
3021 let report = sweep(&f.home, &disk, aged).await;
3022 assert_eq!(report.freed_bytes, 200, "{evidence}: {report:?}");
3023 }
3024 }
3025
3026 #[tokio::test]
3027 async fn missing_tag_proof_preserves_locks_unknown_files_and_retirement_boundary() {
3028 let f = Fx::new().await;
3029 let state = f.run("ac02", true).await;
3030 let wt = &state.candidates[0].worktree;
3031 missing_tag_target(wt);
3032 let target = wt.join("target");
3033 let unknown = target.join("debug/notes");
3034 std::fs::write(&unknown, "operator data").unwrap();
3035 for lock in [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"] {
3036 let path = target.join("debug").join(lock);
3037 let file = OpenOptions::new()
3038 .create(true)
3039 .truncate(false)
3040 .read(true)
3041 .write(true)
3042 .open(&path)
3043 .unwrap();
3044 crate::pr_head::try_lock_file(&file).unwrap();
3045 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3046 assert_eq!(report.freed_bytes, 0, "{lock}: {report:?}");
3047 assert!(
3048 report
3049 .retained
3050 .iter()
3051 .any(|s| s.contains(path.to_str().unwrap())),
3052 "{report:?}"
3053 );
3054 }
3055 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
3056 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3057 drop(owner);
3058 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
3059 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3060 drop(driver);
3061 let output = OutputLease::borrow(wt).unwrap().unwrap();
3062 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3063 drop(output);
3064 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3065 assert_eq!(report.freed_bytes, 200, "{report:?}");
3066 assert_eq!(std::fs::read_to_string(unknown).unwrap(), "operator data");
3067 assert!(!target.join("CACHEDIR.TAG").exists());
3068 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
3069 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
3070 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3071 }
3072
3073 #[tokio::test]
3074 async fn missing_tag_partial_reclamation_can_resume_without_adopting_residue() {
3075 let f = Fx::new().await;
3076 let state = f.run("ac04", true).await;
3077 let wt = &state.candidates[0].worktree;
3078 missing_tag_target(wt);
3079 let target = wt.join("target");
3080 let first = target.join("debug/output");
3081 let second = target.join("debug/second");
3082 std::fs::copy(&first, &second).unwrap();
3083 std::fs::write(
3084 second.with_extension("d"),
3085 format!("{}: source\n", second.display()),
3086 )
3087 .unwrap();
3088 let outputs = declared_outputs(&target).unwrap();
3089 assert_eq!(outputs.len(), 2);
3090 std::fs::write(f.home.join("runs").join(&state.id).join("build-outputs.json"),
3092 serde_json::to_vec(&serde_json::json!({"target":target,"files":outputs,"action":"reclaim-declared-outputs"})).unwrap()).unwrap();
3093 std::fs::remove_file(first).unwrap();
3094 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3095 assert_eq!(report.freed_bytes, 200, "{report:?}");
3096 assert!(!second.exists());
3097 assert!(second.with_extension("d").exists());
3098 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
3099 }
3100
3101 #[tokio::test]
3102 async fn missing_tag_proof_rejects_invalid_identity_structure_and_symlinks() {
3103 for invalid in [
3104 "identity",
3105 "deps",
3106 "fingerprint",
3107 "lock",
3108 "dep-info",
3109 "tag-symlink",
3110 "bad-tag",
3111 ] {
3112 let f = Fx::new().await;
3113 let state = f.run("ac03", true).await;
3114 let wt = &state.candidates[0].worktree;
3115 missing_tag_target(wt);
3116 let target = wt.join("target");
3117 match invalid {
3118 "identity" => std::fs::write(
3119 target.join(".rustc_info.json"),
3120 r#"{"rustc_fingerprint":123}"#,
3121 )
3122 .unwrap(),
3123 "deps" | "fingerprint" => {
3124 let path = target.join("debug").join(if invalid == "deps" {
3125 "deps"
3126 } else {
3127 ".fingerprint"
3128 });
3129 std::fs::remove_dir(&path).unwrap();
3130 std::os::unix::fs::symlink(&f.repo, path).unwrap();
3131 }
3132 "lock" => {
3133 std::fs::remove_file(target.join("debug/.cargo-lock")).unwrap();
3134 }
3135 "dep-info" => std::fs::write(target.join("debug/output.d"), "").unwrap(),
3136 "tag-symlink" => {
3137 std::os::unix::fs::symlink(target.join("absent"), target.join("CACHEDIR.TAG"))
3138 .unwrap()
3139 }
3140 "bad-tag" => std::fs::write(target.join("CACHEDIR.TAG"), "unknown").unwrap(),
3141 _ => unreachable!(),
3142 }
3143 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3144 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
3145 assert!(target.join("debug/output").is_file());
3146 }
3147 }
3148
3149 #[tokio::test]
3150 async fn sweep_budget_oversized_target_makes_progress_without_a_second_deletion() {
3151 let f = Fx::new().await;
3152 let first = f.run("ba00", true).await;
3153 let second = f.run("ba01", true).await;
3154 for state in [&first, &second] {
3155 Fx::target(&state.candidates[0].worktree);
3156 }
3157 let mut disk = Fx::disk();
3158 disk.worktree_reclaim_bytes = 100;
3159 let out = sweep(&f.home, &disk, Fx::now()).await;
3160 assert_eq!(out.freed_bytes, 200, "{out:?}");
3161 assert!(
3162 out.retained
3163 .contains(&format!("{}: per-pass byte budget spent", second.id)),
3164 "{out:?}"
3165 );
3166 assert!(
3167 !first.candidates[0]
3168 .worktree
3169 .join("target/debug/output")
3170 .exists()
3171 );
3172 assert!(
3173 second.candidates[0]
3174 .worktree
3175 .join("target/debug/output")
3176 .exists()
3177 );
3178 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
3179 }
3180
3181 #[tokio::test]
3182 async fn sweep_budget_pressure_prioritizes_largest_over_oldest() {
3183 let f = Fx::new().await;
3184 let small = f.run("bb00", true).await;
3185 let large = f.run("bb01", true).await;
3186 for state in [&small, &large] {
3187 Fx::target(&state.candidates[0].worktree);
3188 }
3189 let output = large.candidates[0].worktree.join("target/debug/output");
3190 std::fs::OpenOptions::new()
3191 .write(true)
3192 .open(&output)
3193 .unwrap()
3194 .set_len(1000)
3195 .unwrap();
3196 let mut disk = Fx::disk();
3197 disk.worktree_reclaim_bytes = 300;
3198 disk.min_free_bytes = u64::MAX;
3199 disk.worktree_limit_bytes = u64::MAX;
3201 let out = sweep(&f.home, &disk, Fx::now()).await;
3202 assert_eq!(out.freed_bytes, 1000, "{out:?}");
3203 assert!(!output.exists());
3204 assert!(
3205 small.candidates[0]
3206 .worktree
3207 .join("target/debug/output")
3208 .exists()
3209 );
3210 }
3211
3212 #[tokio::test]
3213 async fn sweep_budget_zero_remaining_and_inventory_are_distinct() {
3214 let f = Fx::new().await;
3215 let first = f.run("bc00", true).await;
3216 let second = f.run("bc01", true).await;
3217 for state in [&first, &second] {
3218 Fx::target(&state.candidates[0].worktree);
3219 }
3220 std::fs::create_dir_all(
3222 first.candidates[0]
3223 .worktree
3224 .join("target/debug/incremental"),
3225 )
3226 .unwrap();
3227 std::fs::write(
3228 first.candidates[0]
3229 .worktree
3230 .join("target/debug/incremental/cache"),
3231 [0; 1000],
3232 )
3233 .unwrap();
3234 let mut disk = Fx::disk();
3235 disk.worktree_reclaim_bytes = 0;
3236 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3237 disk.worktree_reclaim_bytes = 300;
3238 let out = sweep(&f.home, &disk, Fx::now()).await;
3239 assert_eq!(out.freed_bytes, 200, "{out:?}");
3240 assert!(
3241 !first.candidates[0]
3242 .worktree
3243 .join("target/debug/output")
3244 .exists()
3245 );
3246 assert!(
3247 second.candidates[0]
3248 .worktree
3249 .join("target/debug/output")
3250 .exists()
3251 );
3252 assert!(
3253 out.retained
3254 .iter()
3255 .any(|r| r.contains("declared outputs exceed"))
3256 );
3257 }
3258
3259 #[tokio::test]
3260 async fn sweep_budget_exception_preserves_grace_liveness_and_locks() {
3261 let f = Fx::new().await;
3262 let mut protected = f.run("bd00", true).await;
3263 let later = f.run("bd01", true).await;
3264 for state in [&protected, &later] {
3265 Fx::target(&state.candidates[0].worktree);
3266 }
3267 let mut disk = Fx::disk();
3268 disk.worktree_reclaim_bytes = 100;
3269 disk.fold_grace_secs = 3600;
3270 let out = sweep(&f.home, &disk, Fx::now()).await;
3271 assert_eq!(out.freed_bytes, 0);
3272 assert!(
3273 out.retained
3274 .iter()
3275 .any(|r| r.contains("within grace period"))
3276 );
3277 disk.fold_grace_secs = 0;
3278 protected.driver_exited = false;
3279 protected.driver_pid = Some(std::process::id());
3280 protected.driver_started_at = crate::proc::process_started_at(std::process::id());
3281 protected
3282 .save_under_at(&f.home, protected.updated_at)
3283 .unwrap();
3284 let out = sweep(&f.home, &disk, Fx::now()).await;
3285 assert!(
3286 out.retained
3287 .iter()
3288 .any(|r| r.contains("run live or ownership unknown"))
3289 );
3290 assert_eq!(
3291 out.freed_bytes, 200,
3292 "live target must not consume exception: {out:?}"
3293 );
3294 let output = protected.candidates[0].worktree.join("target/debug/output");
3295 assert!(output.exists());
3296 protected.driver_exited = true;
3297 protected.save_under(&f.home).unwrap();
3298 let owner = crate::pr_head::owner_lock(&f.home, &protected.id).unwrap();
3299 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3300 assert!(output.exists());
3301 drop(owner);
3302 let cargo = cargo_locks(&protected.candidates[0].worktree.join("target")).unwrap();
3303 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3304 assert!(output.exists());
3305 drop(cargo);
3306 let lease = OutputLease::take(&protected.candidates[0].worktree, true).unwrap();
3307 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3308 assert!(output.exists());
3309 drop(lease);
3310 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
3311 assert!(!output.exists());
3312 }
3313
3314 #[tokio::test]
3315 async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
3316 for remove_root in [true, false] {
3317 let f = Fx::new().await;
3318 let stale = Fx::new().await;
3319 let mut missing = stale.run("aa00", false).await;
3320 Fx::target(&missing.candidates[0].worktree);
3321 missing.save_under(&f.home).unwrap();
3322 if remove_root {
3323 std::fs::remove_dir_all(missing.worktree_root()).unwrap();
3324 std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
3327 } else {
3328 std::fs::remove_dir_all(&stale.repo).unwrap();
3329 }
3330 let healthy = f.run("aa01", false).await;
3331 Fx::target(&healthy.candidates[0].worktree);
3332
3333 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3334 assert!(report.failures.is_empty(), "{report:?}");
3335 assert_eq!(report.freed_bytes, 200, "{report:?}");
3336 assert!(
3337 !healthy.candidates[0]
3338 .worktree
3339 .join("target/debug/output")
3340 .exists()
3341 );
3342 if !remove_root {
3343 assert!(
3345 missing.candidates[0]
3346 .worktree
3347 .join("target/debug/output")
3348 .exists()
3349 );
3350 }
3351 }
3352 }
3353
3354 #[tokio::test]
3355 async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
3356 let f = Fx::new().await;
3357 let mut missing = f.run("aa00", false).await;
3358 Fx::target(&missing.candidates[0].worktree);
3359 missing.repo = missing.candidates[0].worktree.clone();
3360 missing.save_under(&f.home).unwrap();
3361 std::fs::remove_dir_all(&missing.repo).unwrap();
3362 assert!(missing.worktree_root().is_dir());
3363 assert!(f.repo.is_dir());
3364
3365 let healthy = f.run("aa01", false).await;
3366 Fx::target(&healthy.candidates[0].worktree);
3367
3368 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3369 assert!(report.failures.is_empty(), "{report:?}");
3370 assert_eq!(report.freed_bytes, 200, "{report:?}");
3371 assert!(
3372 !healthy.candidates[0]
3373 .worktree
3374 .join("target/debug/output")
3375 .exists()
3376 );
3377 }
3378
3379 #[tokio::test]
3380 async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
3381 let f = Fx::new().await;
3382 let broken = Fx::new().await;
3383 let mut invalid = broken.run("aa00", false).await;
3384 Fx::target(&invalid.candidates[0].worktree);
3385 invalid.save_under(&f.home).unwrap();
3386 std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
3387 let healthy = f.run("aa01", false).await;
3388 Fx::target(&healthy.candidates[0].worktree);
3389
3390 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3391 assert_eq!(report.failures.len(), 1, "{report:?}");
3392 assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
3393 assert_eq!(report.freed_bytes, 200, "{report:?}");
3394 assert!(
3395 !healthy.candidates[0]
3396 .worktree
3397 .join("target/debug/output")
3398 .exists()
3399 );
3400 assert!(
3401 invalid.candidates[0]
3402 .worktree
3403 .join("target/debug/output")
3404 .exists()
3405 );
3406 }
3407
3408 async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
3409 let path = reconcile::run(&f.home, &task.id, &source.id, None)
3410 .await
3411 .unwrap();
3412 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
3413 }
3414
3415 #[tokio::test]
3416 async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
3417 use crate::{
3418 chat_control::{Operation, dispatch},
3419 talk::{self, Talks},
3420 };
3421 let f = Fx::new().await;
3422 let source = f.run("ae00", true).await;
3423 let clean = f.run("ae01", false).await;
3424 let queue = crate::queue::Queue::at(f.home.join("queue"));
3425 let talks = Talks::at(f.home.join("talks"));
3426 let open_talk = |id: &str| -> talk::Talk {
3427 let mut conversation = serde_json::from_value(serde_json::json!({
3428 "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
3429 "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
3430 "seat":crate::agent::SeatState::new("talk","worker",1)
3431 }))
3432 .unwrap();
3433 talks.put(&mut conversation).unwrap();
3434 conversation
3435 };
3436 let conversation = open_talk("talk-owner");
3437 let other = open_talk("talk-other");
3438 let task = f.task(&[source.clone(), clean.clone()]);
3440 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3441 let operation = |apply: Option<&str>| Operation::ReconcileRetention {
3442 task: task.id.clone(),
3443 authority: source.id.clone(),
3444 apply: apply.map(str::to_owned),
3445 };
3446 let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
3448 .await
3449 .unwrap();
3450 let legacy: serde_json::Value =
3451 serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
3452 let legacy_id = legacy["id"].as_str().unwrap();
3453 assert!(
3454 dispatch(
3455 &queue,
3456 &f.home,
3457 &talks,
3458 &conversation.id,
3459 operation(Some(legacy_id))
3460 )
3461 .await
3462 .is_err()
3463 );
3464 let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
3465 .await
3466 .unwrap();
3467 assert!(summary.contains("Complete review summary"));
3468 assert!(summary.contains(&source.id) && summary.contains(&clean.id));
3469 let root = f.home.join("retention-reconciliations");
3470 let displayed: serde_json::Value =
3471 serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
3472 let id = displayed["plan"].as_str().unwrap();
3473 let dir = root.join(id);
3474 let plan: serde_json::Value =
3475 serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
3476 assert_eq!(plan["version"], 2);
3477 assert!(clean.candidates[0].worktree.exists());
3478 assert!(!dir.join("approved.json").exists());
3479 assert!(
3481 dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
3482 .await
3483 .is_err()
3484 );
3485 assert!(!dir.join("approved.json").exists());
3486 let error = reconcile::interrupt_chat_after_fence(
3489 &f.home,
3490 &task,
3491 id,
3492 reconcile::ChatRequest {
3493 talks: &talks,
3494 conversation: &conversation.id,
3495 },
3496 )
3497 .await
3498 .unwrap_err();
3499 assert!(
3500 error.to_string().contains("simulated interruption"),
3501 "{error:#}"
3502 );
3503 assert!(clean.candidates[0].worktree.exists());
3504 assert!(dir.join("approved.json").exists());
3505 dispatch(
3506 &queue,
3507 &f.home,
3508 &talks,
3509 &conversation.id,
3510 operation(Some(id)),
3511 )
3512 .await
3513 .unwrap();
3514 assert!(!clean.candidates[0].worktree.exists());
3515 assert!(source.candidates[0].worktree.exists());
3516 let audit: serde_json::Value =
3517 serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
3518 assert_eq!(audit["version"], 2);
3519 assert_eq!(
3520 audit["owner"],
3521 serde_json::json!({"conversation": conversation.id})
3522 );
3523 dispatch(
3524 &queue,
3525 &f.home,
3526 &talks,
3527 &conversation.id,
3528 operation(Some(id)),
3529 )
3530 .await
3531 .unwrap();
3532 }
3533
3534 #[tokio::test]
3535 async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
3536 let f = Fx::new().await;
3537 let source = f.run("af00", true).await;
3538 let mut clean = f.run("af01", false).await;
3539 let mut dirty = f.run("af02", false).await;
3540 for state in [&mut clean, &mut dirty] {
3541 state.status = RunStatus::Prep;
3542 state.prep_complete = Some(false);
3543 state.candidates[0].empty = false;
3544 state.save_under(&f.home).unwrap();
3545 }
3546 std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
3547 let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
3548 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3549 let plan = reconciliation_plan(&f, &task, &source).await;
3550 let selected: Vec<_> = plan["candidates"]
3551 .as_array()
3552 .unwrap()
3553 .iter()
3554 .filter(|e| e["selected"] == true)
3555 .collect();
3556 assert_eq!(selected.len(), 1, "{plan:#}");
3557 assert_eq!(selected[0]["run"], clean.id);
3558 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3559 .await
3560 .unwrap();
3561 assert!(!clean.candidates[0].worktree.exists());
3562 assert!(dirty.candidates[0].worktree.exists());
3563 assert!(
3564 RunState::load_under(&dirty.id, &f.home)
3565 .unwrap()
3566 .retired_to
3567 .is_none()
3568 );
3569 }
3570
3571 #[tokio::test]
3572 async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
3573 let f = Fx::new().await;
3574 let mut source = f.run("aa00", true).await;
3575 let alternative = f.run("aa01", true).await;
3576 let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
3577 .await
3578 .unwrap();
3579 source.reviewed_commits = Some(vec![head.clone()]);
3580 source.save_under(&f.home).unwrap();
3581 let mut runs = vec![source.clone(), alternative.clone()];
3582 for i in 2..14 {
3583 runs.push(f.run(&format!("aa{i:02}"), false).await);
3584 }
3585 let mut task = f.task(&runs);
3586 task.pr_recovery = Some(crate::pr_recovery::Selection {
3587 url: "https://example.invalid/pull/1".into(),
3588 branch: "selected".into(),
3589 head: head.clone(),
3590 source_head: Some(head.clone()),
3591 source_run: Some(source.id.clone()),
3592 source_proof: None,
3593 disposition: Some(crate::pr_recovery::Disposition::Land),
3594 instruction: task.instruction.clone(),
3595 approved_at: Timestamp::now(),
3596 });
3597 source.pr_recovery = task.pr_recovery.clone();
3598 source.save_under(&f.home).unwrap();
3599 runs[0] = source.clone();
3600 crate::queue::Queue::at(f.home.join("queue"))
3601 .put(&mut task)
3602 .unwrap();
3603 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3604 let task_before = serde_json::to_value(&task).unwrap();
3605 let mut report = Report::default();
3606 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3607 assert_eq!(
3608 report.retired, 0,
3609 "automatic cleanup must still refuse blanket markers"
3610 );
3611 assert!(
3612 report
3613 .retained
3614 .iter()
3615 .any(|reason| reason.contains("explicitly retained"))
3616 );
3617 let mut total = 0;
3618 for expected in [5, 5, 2, 0] {
3619 let plan = reconciliation_plan(&f, &task, &source).await;
3620 let selected: Vec<_> = plan["candidates"]
3621 .as_array()
3622 .unwrap()
3623 .iter()
3624 .filter(|e| e["selected"] == true)
3625 .collect();
3626 assert_eq!(selected.len(), expected, "{plan:#}");
3627 for e in &selected {
3628 assert!(
3629 RunState::load_under(e["run"].as_str().unwrap(), &f.home)
3630 .unwrap()
3631 .retired_to
3632 .is_none()
3633 );
3634 }
3635 let operation = plan["id"].as_str().unwrap();
3636 for _ in 0..2 {
3637 reconcile::run(&f.home, &task.id, &source.id, Some(operation))
3638 .await
3639 .unwrap();
3640 }
3641 total += expected;
3642 }
3643 assert_eq!(total, 12);
3644 for run in &runs {
3645 let dir = f.home.join("runs").join(&run.id);
3646 assert!(dir.join("run.json").exists());
3647 assert!(dir.join("retain-recovery-sources").exists());
3648 assert!(
3649 git::branch_exists(&f.repo, &run.candidates[0].branch)
3650 .await
3651 .unwrap()
3652 );
3653 }
3654 assert!(source.candidates[0].worktree.exists());
3655 assert!(alternative.candidates[0].worktree.exists());
3656 assert_eq!(
3657 git::rev_parse(&f.repo, &source.candidates[0].branch)
3658 .await
3659 .unwrap(),
3660 head
3661 );
3662 assert_eq!(
3663 serde_json::to_value(
3664 crate::queue::Queue::at(f.home.join("queue"))
3665 .get(&task.id)
3666 .unwrap()
3667 )
3668 .unwrap(),
3669 task_before
3670 );
3671 assert_eq!(
3672 runs.iter()
3673 .filter(|r| RunState::load_under(&r.id, &f.home)
3674 .unwrap()
3675 .retired_to
3676 .is_none())
3677 .count(),
3678 2
3679 );
3680 }
3681
3682 #[tokio::test]
3683 async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
3684 let f = Fx::new().await;
3685 let source = f.run("ab00", true).await;
3686 let mut runs = vec![source.clone()];
3687 for i in 1..9 {
3688 runs.push(f.run(&format!("ab{i:02}"), false).await);
3689 }
3690 std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
3691 std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
3692 std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
3693 std::fs::write(
3694 runs[3].candidates[0].worktree.join("scratch/source"),
3695 "ignored source",
3696 )
3697 .unwrap();
3698 std::fs::write(
3699 runs[4].worktree_root().join("unknown"),
3700 "unregistered source",
3701 )
3702 .unwrap();
3703 let checkpoint = git::git(
3704 &runs[5].candidates[0].worktree,
3705 &[
3706 "rev-parse",
3707 "--path-format=absolute",
3708 "--git-path",
3709 "sequencer",
3710 ],
3711 )
3712 .await
3713 .unwrap();
3714 std::fs::create_dir(checkpoint.trim()).unwrap();
3715 runs[6].driver_exited = false;
3716 runs[6].driver_pid = Some(std::process::id());
3717 runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
3718 runs[6].save_under(&f.home).unwrap();
3719 runs[7].driver_exited = false;
3720 runs[7].driver_pid = None;
3721 runs[7].driver_claimed = None; runs[7].save_under(&f.home).unwrap();
3723 assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
3724
3725 let hidden = &runs[8].candidates[0].worktree;
3726 git::git(hidden, &["update-index", "--assume-unchanged", "source"])
3727 .await
3728 .unwrap();
3729 std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
3730 assert!(
3731 git::git(hidden, &["status", "--porcelain"])
3732 .await
3733 .unwrap()
3734 .is_empty()
3735 );
3736 let task = f.task(&runs);
3737 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3738 let plan = reconciliation_plan(&f, &task, &source).await;
3739 assert!(
3740 plan["candidates"]
3741 .as_array()
3742 .unwrap()
3743 .iter()
3744 .all(|e| e["selected"] == false),
3745 "{plan:#}"
3746 );
3747 for run in &runs {
3748 assert!(run.candidates[0].worktree.exists());
3749 }
3750 }
3751
3752 #[tokio::test]
3753 async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
3754 let f = Fx::new().await;
3755 let source = f.run("ac00", true).await;
3756 let empty = f.run("ac01", false).await;
3757 let task = f.task(&[source.clone(), empty.clone()]);
3758 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3759 let plan = reconciliation_plan(&f, &task, &source).await;
3760 std::fs::write(
3761 f.home.join("runs").join(&source.id).join("new-report"),
3762 "new review checkpoint",
3763 )
3764 .unwrap();
3765 assert!(
3766 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3767 .await
3768 .is_err()
3769 );
3770 assert!(empty.candidates[0].worktree.exists());
3771 assert!(
3772 RunState::load_under(&empty.id, &f.home)
3773 .unwrap()
3774 .retired_to
3775 .is_none()
3776 );
3777 let marker = f
3778 .home
3779 .join("runs")
3780 .join(&empty.id)
3781 .join("retain-recovery-sources");
3782 std::fs::write(&marker, "different owner").unwrap();
3783 let plan = reconciliation_plan(&f, &task, &source).await;
3784 assert!(
3785 plan["candidates"]
3786 .as_array()
3787 .unwrap()
3788 .iter()
3789 .all(|e| e["selected"] == false)
3790 );
3791 }
3792
3793 #[tokio::test]
3794 async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
3795 for partial in [false, true] {
3796 let f = Fx::new().await;
3797 let source = f.run("ab10", true).await;
3798 let first = f.run("ab11", false).await;
3799 let second = f.run("ab12", false).await;
3800 let task = f.task(&[source.clone(), first.clone(), second.clone()]);
3801 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3802 let plan = reconciliation_plan(&f, &task, &source).await;
3803 let id = plan["id"].as_str().unwrap();
3804 let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
3805 std::fs::write(&staging, b"").unwrap();
3807 assert!(
3808 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3809 .await
3810 .is_err()
3811 );
3812 assert!(staging.exists());
3813 std::fs::remove_file(&staging).unwrap();
3814 let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
3815 .await
3816 .unwrap_err();
3817 assert!(
3818 error.to_string().contains("before run state rename"),
3819 "{error:#}"
3820 );
3821 assert!(
3822 RunState::load_under(&first.id, &f.home)
3823 .unwrap()
3824 .retired_to
3825 .is_some()
3826 );
3827 assert!(
3828 RunState::load_under(&second.id, &f.home)
3829 .unwrap()
3830 .retired_to
3831 .is_none()
3832 );
3833 assert!(first.candidates[0].worktree.exists());
3834 assert!(second.candidates[0].worktree.exists());
3835 let interrupted = std::fs::read(&staging).unwrap();
3836 std::fs::write(&staging, b"unrelated recovery source").unwrap();
3838 assert!(
3839 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3840 .await
3841 .is_err()
3842 );
3843 assert_eq!(
3844 std::fs::read(&staging).unwrap(),
3845 b"unrelated recovery source"
3846 );
3847 std::fs::write(&staging, interrupted).unwrap();
3848 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3849 .await
3850 .unwrap();
3851 assert!(!staging.exists());
3852 assert!(!first.candidates[0].worktree.exists());
3853 assert!(!second.candidates[0].worktree.exists());
3854 assert_eq!(
3855 RunState::load_under(&second.id, &f.home)
3856 .unwrap()
3857 .retired_to,
3858 Some(source.id)
3859 );
3860 }
3861 }
3862
3863 #[tokio::test]
3864 async fn reconciliation_resumes_only_its_exact_durable_fence() {
3865 let f = Fx::new().await;
3866 let source = f.run("ad00", true).await;
3867 let empty = f.run("ad01", false).await;
3868 let task = f.task(&[source.clone(), empty.clone()]);
3869 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3870 let plan = reconciliation_plan(&f, &task, &source).await;
3871 let id = plan["id"].as_str().unwrap();
3872 assert!(
3873 reconcile::interrupt_after_fence(&f.home, &task, id)
3874 .await
3875 .is_err()
3876 );
3877 assert!(empty.candidates[0].worktree.exists());
3878 assert_eq!(
3879 RunState::load_under(&empty.id, &f.home)
3880 .unwrap()
3881 .retired_to
3882 .as_ref(),
3883 Some(&source.id)
3884 );
3885 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3886 .await
3887 .unwrap();
3888 assert!(!empty.candidates[0].worktree.exists());
3889 git::worktree_add_branch(
3891 &f.repo,
3892 &empty.candidates[0].worktree,
3893 "restore-empty",
3894 &f.base,
3895 )
3896 .await
3897 .unwrap();
3898 assert!(
3900 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3901 .await
3902 .is_err()
3903 );
3904 assert!(empty.candidates[0].worktree.exists());
3905 let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
3906 changed.prep_failure = Some("new owner edit".into());
3907 changed.save_under(&f.home).unwrap();
3908 assert!(
3909 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3910 .await
3911 .is_err()
3912 );
3913 }
3914
3915 #[tokio::test]
3916 async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
3917 let f = Fx::new().await;
3918 let source = f.run("ae00", true).await;
3919 let mut prep = f.run("ae01", false).await;
3920 let candidate = prep.candidates.pop().unwrap();
3921 prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
3922 prep.save_under(&f.home).unwrap();
3923 let reset = f.run("ae02", false).await;
3924 let wt = &reset.candidates[0].worktree;
3925 std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
3926 git::git(wt, &["add", "."]).await.unwrap();
3927 git::git(wt, &["commit", "-m", "unique implementation"])
3928 .await
3929 .unwrap();
3930 git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
3931 let unknown = f.run("ae03", false).await;
3932 let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
3933 let mut raw: serde_json::Value =
3934 serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
3935 raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
3936 std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
3937 let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
3938 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3939 let plan = reconciliation_plan(&f, &task, &source).await;
3940 let selected: Vec<_> = plan["candidates"]
3941 .as_array()
3942 .unwrap()
3943 .iter()
3944 .filter(|e| e["selected"] == true)
3945 .collect();
3946 assert_eq!(selected.len(), 1, "{plan:#}");
3947 assert_eq!(selected[0]["run"], prep.id);
3948 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3949 .await
3950 .unwrap();
3951 assert!(!candidate.worktree.exists());
3952 assert!(reset.candidates[0].worktree.exists());
3953 assert!(unknown.candidates[0].worktree.exists());
3954 }
3955
3956 #[tokio::test]
3957 async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
3958 let f = Fx::new().await;
3959 let mut state = f.run("aaaa", true).await;
3960 let candidate = state.candidates[0].worktree.clone();
3961 std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
3962 Fx::target(&candidate);
3963 for seat in ["review-1", "review-2"] {
3964 let wt = state.worktree_root().join(seat);
3965 git::worktree_add_detached(&f.repo, &wt, &f.base)
3966 .await
3967 .unwrap();
3968 Fx::target(&wt);
3969 }
3970 state.status = RunStatus::Landing;
3971 state.save_under(&f.home).unwrap();
3972 let second = Fx::new().await;
3973 let other = second.run("bbbb", true).await;
3974 let mut imported = other.clone();
3976 imported.save_under(&f.home).unwrap();
3977 Fx::target(&other.candidates[0].worktree);
3978 let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
3979 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3980 assert!(
3981 candidate.join("target/debug/output").exists(),
3982 "active writer protected"
3983 );
3984 assert!(
3985 !state
3986 .worktree_root()
3987 .join("review-1/target/debug/output")
3988 .exists()
3989 );
3990 assert!(
3991 !state
3992 .worktree_root()
3993 .join("review-2/target/debug/output")
3994 .exists()
3995 );
3996 assert!(
3997 !other.candidates[0]
3998 .worktree
3999 .join("target/debug/output")
4000 .exists()
4001 );
4002 assert!(out.freed_bytes >= 600);
4003 drop(guard);
4004 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4005 assert!(out.freed_bytes >= 200);
4006 assert_eq!(
4007 std::fs::read_to_string(candidate.join("source")).unwrap(),
4008 "uncommitted work"
4009 );
4010 assert!(
4011 git::branch_exists(&f.repo, &state.candidates[0].branch)
4012 .await
4013 .unwrap()
4014 );
4015 }
4016
4017 #[tokio::test]
4018 async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
4019 let f = Fx::new().await;
4020 let state = f.run("cc01", false).await;
4021 let wt = &state.candidates[0].worktree;
4022 Fx::target(wt);
4023 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
4024 let dir = wt.join("target").join(dir);
4025 std::fs::create_dir_all(&dir).unwrap();
4026 std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
4027 }
4028 let manifest = wt.join("target/debug/output.d");
4030 let output = wt.join("target/debug/output");
4031 std::fs::write(
4032 &manifest,
4033 format!(
4034 "{}: {}\n",
4035 output.display(),
4036 wt.join("target/debug/notes.txt").display()
4037 ),
4038 )
4039 .unwrap();
4040 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4041 assert_eq!(report.freed_bytes, 200);
4042 assert!(!output.exists());
4043 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
4044 assert_eq!(
4045 std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
4046 "user evidence"
4047 );
4048 }
4049 assert!(manifest.exists());
4051 assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
4052 std::fs::write(&output, "user note replacing an old binary").unwrap();
4053 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4054 assert_eq!(report.freed_bytes, 0);
4055 assert_eq!(
4056 std::fs::read_to_string(&output).unwrap(),
4057 "user note replacing an old binary"
4058 );
4059 }
4060
4061 #[tokio::test]
4062 async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
4063 let f = Fx::new().await;
4064 let state = f.run("cc02", false).await;
4065 let wt = &state.candidates[0].worktree;
4066 Fx::target(wt);
4067 let mut disk = Fx::disk();
4068 disk.worktree_limit_bytes = u64::MAX;
4069 let now = Fx::now();
4070 let first = sweep(&f.home, &disk, now).await;
4071 std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
4072 assert_eq!(
4073 sweep(&f.home, &disk, now).await.measured_bytes,
4074 first.measured_bytes
4075 );
4076 assert!(
4077 sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
4078 .await
4079 .measured_bytes
4080 > first.measured_bytes
4081 );
4082 disk.worktree_limit_bytes = 0;
4083 assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
4084 }
4085
4086 #[tokio::test]
4087 async fn unfinished_source_fold_reports_its_retention_reason() {
4088 let f = Fx::new().await;
4089 let mut source = f.run("cc03", true).await;
4090 let task = f.task(&[source.clone()]);
4091 let error = crate::graph::fold_run(&mut source, true, &f.home)
4092 .await
4093 .unwrap_err();
4094 assert!(error.to_string().contains(&task.id));
4095 assert!(source.candidates[0].worktree.exists());
4096 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4097 let error = crate::graph::fold_run(&mut source, true, &f.home)
4098 .await
4099 .unwrap_err();
4100 assert!(error.to_string().contains("explicitly retained"));
4101 }
4102
4103 #[tokio::test]
4104 async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
4105 let f = Fx::new().await;
4106 let mut source = f.run("cc04", true).await;
4107 let mut task = f.task(&[source.clone()]);
4108 task.hold_manual(Some("queue settlement pending".into()));
4109 let queue = crate::queue::Queue::at(f.home.join("queue"));
4110 queue.put(&mut task).unwrap();
4111 source.merge = Some(crate::run::MergeOutcome {
4112 mode: crate::config::MergeMode::Pr,
4113 ok: true,
4114 detail: "confirmed merged".into(),
4115 empty: false,
4116 });
4117 for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
4119 source.status = status;
4120 assert!(
4121 crate::graph::fold_run(&mut source, true, &f.home)
4122 .await
4123 .is_err()
4124 );
4125 assert!(source.candidates[0].worktree.exists());
4126 }
4127 git::git(
4128 &f.repo,
4129 &["merge", "--ff-only", &source.candidates[0].branch],
4130 )
4131 .await
4132 .unwrap();
4133 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
4134 .await
4135 .unwrap();
4136 source.status = RunStatus::Merged;
4137 source.save_under(&f.home).unwrap();
4138 assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
4139 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4140 assert!(
4141 crate::graph::fold_run(&mut source, true, &f.home)
4142 .await
4143 .unwrap_err()
4144 .to_string()
4145 .contains("explicitly retained")
4146 );
4147 std::fs::remove_file(
4150 f.home
4151 .join("runs")
4152 .join(&source.id)
4153 .join("retain-recovery-sources"),
4154 )
4155 .unwrap();
4156 let (folded, unreadable) = crate::clean::fold_due(
4157 &f.home.join("runs"),
4158 &f.home,
4159 &f.bay,
4160 &Fx::disk(),
4161 Fx::now(),
4162 )
4163 .await
4164 .unwrap();
4165 assert_eq!((folded, unreadable), (1, 0));
4166 assert!(!source.candidates[0].worktree.exists());
4167 let evidence = f
4168 .home
4169 .join("runs")
4170 .join(&source.id)
4171 .join("artifacts/fold-evidence");
4172 let names: Vec<_> = std::fs::read_dir(evidence)
4173 .unwrap()
4174 .map(|entry| entry.unwrap().file_name())
4175 .collect();
4176 assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
4177 assert_eq!(
4178 queue.get(&task.id).unwrap().status,
4179 crate::queue::TaskStatus::Held
4180 );
4181 }
4182
4183 #[tokio::test]
4184 async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
4185 for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
4186 let f = Fx::new().await;
4187 let mut state = f.run("cc06", true).await;
4188 let candidate = state.candidates[0].clone();
4189 let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
4190 std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
4191 git::rescue_commit(&candidate.worktree, "second candidate change")
4192 .await
4193 .unwrap();
4194 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
4195 state.tally = Some(crate::run::Tally {
4196 first_choice: Default::default(),
4197 borda: Default::default(),
4198 winner: 'A',
4199 rankings: 0,
4200 unanimous_initial: true,
4201 deliberated: false,
4202 changed_votes: 0,
4203 unanimous_final: true,
4204 tie_break: None,
4205 judges: 0,
4206 present: 0,
4207 quorum: 0,
4208 met_quorum: true,
4209 uncontested: Some("fixture".into()),
4210 });
4211 let reviewer = state.worktree_root().join("review-1");
4214 git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
4215 .await
4216 .unwrap();
4217 std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
4220 git::rescue_commit(&f.repo, "advance before squash")
4221 .await
4222 .unwrap();
4223 let published = if variant == "partial" {
4224 &intermediate
4225 } else {
4226 &head
4227 };
4228 git::git(&f.repo, &["merge", "--squash", published])
4229 .await
4230 .unwrap();
4231 git::git(&f.repo, &["commit", "-m", "squash candidate"])
4232 .await
4233 .unwrap();
4234 let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
4235 assert!(
4236 !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
4237 .await
4238 .unwrap()
4239 .ok()
4240 );
4241 std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
4242 git::rescue_commit(&f.repo, "advance after squash")
4243 .await
4244 .unwrap();
4245 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
4246 .await
4247 .unwrap();
4248 if matches!(variant, "checkpoint" | "reviewer") {
4249 std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
4251 git::rescue_commit(&reviewer, "unpublished checkpoint")
4252 .await
4253 .unwrap();
4254 if variant == "checkpoint" {
4255 git::git(&reviewer, &["reset", "--hard", &intermediate])
4256 .await
4257 .unwrap();
4258 }
4259 }
4260 if variant == "loser" {
4261 let mut loser = candidate.clone();
4262 loser.label = 'B';
4263 loser.branch = "magi/cc06/B".into();
4264 loser.worktree = state.worktree_root().join("cand-B");
4265 git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
4266 .await
4267 .unwrap();
4268 std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
4269 git::rescue_commit(&loser.worktree, "independent loser")
4270 .await
4271 .unwrap();
4272 state.candidates.push(loser);
4273 }
4274 state.status = RunStatus::Merged;
4275 state.merge = Some(crate::run::MergeOutcome {
4276 mode: crate::config::MergeMode::Pr,
4277 ok: true,
4278 detail: "candidate A squash merged".into(),
4279 empty: false,
4280 });
4281 state.save_under(&f.home).unwrap();
4282 let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
4283 let result = crate::graph::fold_run(&mut state, true, &f.home).await;
4284 result.unwrap();
4285 assert!(
4286 !candidate.worktree.exists() && !reviewer.exists(),
4287 "{variant}"
4288 );
4289 assert!(
4290 !git::branch_exists(&f.repo, &candidate.branch)
4291 .await
4292 .unwrap()
4293 );
4294 let artifacts = f
4295 .home
4296 .join("runs")
4297 .join(&state.id)
4298 .join("artifacts/fold-evidence");
4299 assert_eq!(
4300 artifacts.join(format!("{head}.patch")).exists(),
4301 variant == "partial",
4302 "only a partially published winner needs evidence: {variant}"
4303 );
4304 assert_eq!(
4305 artifacts.join(format!("{intermediate}.patch")).exists(),
4306 variant == "partial",
4307 "squash proof also covers intermediate reviewer tips: {variant}"
4308 );
4309 let patches: Vec<_> = std::fs::read_dir(&artifacts)
4310 .unwrap()
4311 .map(|entry| entry.unwrap().path())
4312 .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
4313 .collect();
4314 let expected = match variant {
4315 "partial" => 2,
4316 "loser" | "reviewer" => 1,
4317 _ => 0,
4318 };
4319 assert_eq!(patches.len(), expected, "{variant}");
4320 if variant == "loser" {
4321 assert!(
4322 std::fs::read_to_string(&patches[0])
4323 .unwrap()
4324 .contains("losing candidate")
4325 );
4326 }
4327 assert!(artifacts.join("manifest.json").exists());
4328 if variant == "reviewer" {
4329 assert!(
4330 std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
4331 .unwrap()
4332 .contains("reviewer checkpoint")
4333 );
4334 }
4335 crate::graph::fold_run(&mut state, true, &f.home)
4336 .await
4337 .unwrap();
4338 }
4339 }
4340
4341 #[tokio::test]
4342 async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
4343 for missing in [true, false] {
4344 let f = Fx::new().await;
4345 let mut state = f.run("cc05", true).await;
4346 git::git(
4349 &f.repo,
4350 &["update-ref", "refs/remotes/origin/main", &f.base],
4351 )
4352 .await
4353 .unwrap();
4354 let candidate = state.candidates[0].clone();
4355 if !missing {
4356 std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
4357 git::rescue_commit(&f.repo, "advance base before rebase")
4358 .await
4359 .unwrap();
4360 git::git(&candidate.worktree, &["rebase", "main"])
4361 .await
4362 .unwrap();
4363 git::sync_to_head(&candidate.worktree).await.unwrap();
4366 }
4367 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
4368 state.reviewed_commits = Some(Vec::new());
4370 state.candidates[0].worktree = state.worktree_root().join("under-review");
4371 git::git(
4372 &f.repo,
4373 &[
4374 "worktree",
4375 "move",
4376 candidate.worktree.to_str().unwrap(),
4377 state.candidates[0].worktree.to_str().unwrap(),
4378 ],
4379 )
4380 .await
4381 .unwrap();
4382 let reviewer = state.worktree_root().join("review-1");
4383 git::worktree_add_detached(&f.repo, &reviewer, &head)
4384 .await
4385 .unwrap();
4386 std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
4389 git::rescue_commit(&reviewer, "old review history")
4390 .await
4391 .unwrap();
4392 git::git(&reviewer, &["reset", "--hard", &head])
4393 .await
4394 .unwrap();
4395 if missing {
4396 std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
4397 }
4398 let run_dir = f.home.join("runs").join(&state.id);
4399 let artifacts = run_dir.join("artifacts");
4400 std::fs::create_dir_all(&artifacts).unwrap();
4401 std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
4402 if missing {
4403 std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
4404 }
4405 state.status = RunStatus::Merged;
4406 state.merge = Some(crate::run::MergeOutcome {
4407 mode: crate::config::MergeMode::Pr,
4408 ok: true,
4409 detail: "confirmed squash merge".into(),
4410 empty: false,
4411 });
4412 state.save_under(&f.home).unwrap();
4413 let owned = state.candidates[0].worktree.clone();
4415 state.candidates[0].worktree = f.home.join("missing-external");
4416 assert!(
4417 crate::graph::fold_run(&mut state, true, &f.home)
4418 .await
4419 .unwrap_err()
4420 .to_string()
4421 .contains("outside the inventoried")
4422 );
4423 state.candidates[0].worktree = owned;
4424 let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
4426 assert!(
4427 crate::graph::fold_run(&mut state, true, &f.home)
4428 .await
4429 .is_err()
4430 );
4431 assert!(reviewer.exists());
4432 drop(guard);
4433 let evidence = artifacts.join("fold-evidence");
4435 std::fs::write(&evidence, "blocked").unwrap();
4436 assert!(
4437 crate::graph::fold_run(&mut state, true, &f.home)
4438 .await
4439 .is_err()
4440 );
4441 assert!(reviewer.exists());
4442 std::fs::remove_file(&evidence).unwrap();
4443 std::fs::create_dir_all(&evidence).unwrap();
4444 let oversized = evidence.join("capacity-fixture");
4445 std::fs::File::create(&oversized)
4446 .unwrap()
4447 .set_len(64 * 1024 * 1024)
4448 .unwrap();
4449 assert!(
4450 crate::graph::fold_run(&mut state, true, &f.home)
4451 .await
4452 .is_err()
4453 );
4454 assert!(reviewer.exists());
4455 std::fs::remove_file(oversized).unwrap();
4456 std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
4459 std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
4460 assert!(
4461 crate::graph::fold_run(&mut state, true, &f.home)
4462 .await
4463 .is_err()
4464 );
4465 std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
4466 let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
4467 .await
4468 .unwrap();
4469 std::fs::write(&checkpoint, &head).unwrap();
4470 assert!(
4471 crate::graph::fold_run(&mut state, true, &f.home)
4472 .await
4473 .is_err()
4474 );
4475 std::fs::remove_file(checkpoint).unwrap();
4476 let (folded, unreadable) = crate::clean::fold_due(
4477 &f.home.join("runs"),
4478 &f.home,
4479 &f.bay,
4480 &Fx::disk(),
4481 Fx::now(),
4482 )
4483 .await
4484 .unwrap();
4485 if folded != 1 {
4486 panic!(
4487 "normal cleanup failed: {:#}",
4488 crate::graph::fold_run(&mut state, true, &f.home)
4489 .await
4490 .unwrap_err()
4491 );
4492 }
4493 assert_eq!((folded, unreadable), (1, 0));
4494 assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
4495 state = RunState::load_under(&state.id, &f.home).unwrap();
4496 assert!(state.candidates[0].folded);
4497 let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
4498 crate::graph::fold_run(&mut state, true, &f.home)
4499 .await
4500 .unwrap();
4501 assert_eq!(
4502 std::fs::read(evidence.join("manifest.json")).unwrap(),
4503 manifest
4504 );
4505 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
4506 .await
4507 .unwrap();
4508 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
4509 assert!(
4510 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
4511 .await
4512 .unwrap()
4513 .ok()
4514 );
4515 let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
4518 let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
4519 .as_str()
4520 .unwrap();
4521 let recovered = f.home.join("recovered");
4522 std::fs::create_dir_all(&recovered).unwrap();
4523 git::git(&recovered, &["init"]).await.unwrap();
4524 std::fs::write(
4525 recovered.join("source"),
4526 std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
4527 )
4528 .unwrap();
4529 git::git(
4530 &recovered,
4531 &[
4532 "apply",
4533 evidence.join(format!("{head}.patch")).to_str().unwrap(),
4534 ],
4535 )
4536 .await
4537 .unwrap();
4538 assert_eq!(
4539 std::fs::read_to_string(recovered.join("source")).unwrap(),
4540 "cc05"
4541 );
4542 assert!(
4543 !std::fs::read(evidence.join(format!("{head}.patch")))
4544 .unwrap()
4545 .is_empty()
4546 );
4547 assert_eq!(
4548 std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
4549 "review findings"
4550 );
4551 if missing {
4552 assert_eq!(
4553 std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
4554 "original snapshot"
4555 );
4556 } else {
4557 assert!(!artifacts.join("cand-A.patch").exists());
4558 }
4559 assert!(run_dir.join("run.json").exists());
4560 }
4561 }
4562
4563 #[tokio::test]
4564 async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
4565 use std::os::unix::fs::PermissionsExt as _;
4566 let f = Fx::new().await;
4567 let source = f.run("dd03", true).await;
4568 let empty = f.run("dd04", false).await;
4569 let task = f.task(&[source, empty.clone()]);
4570 let entered = f.repo.join("proof-entered");
4571 let release = f.repo.join("proof-release");
4572 let hook = f.repo.join("proof-monitor");
4573 std::fs::write(&hook, format!(
4574 "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
4575 entered.display(), release.display())).unwrap();
4576 std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
4577 git::git(
4578 &f.repo,
4579 &["config", "core.fsmonitor", hook.to_str().unwrap()],
4580 )
4581 .await
4582 .unwrap();
4583 struct ReleaseProof(PathBuf);
4584 impl Drop for ReleaseProof {
4585 fn drop(&mut self) {
4586 let _ = std::fs::write(&self.0, "continue");
4587 }
4588 }
4589 let _release_on_panic = ReleaseProof(release.clone());
4590 let home = f.home.clone();
4591 let sweep = tokio::spawn(async move {
4592 let mut report = Report::default();
4593 retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
4594 report
4595 });
4596 let watchdog = std::time::Duration::from_secs(60);
4599 tokio::time::timeout(watchdog, async {
4600 while !entered.exists() {
4601 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4602 }
4603 })
4604 .await
4605 .expect("Git retirement proof did not enter the blocking hook");
4606 let queue = crate::queue::Queue::at(f.home.join("queue"));
4607 let mut selected = queue.get(&task.id).unwrap();
4608 selected.resume_override = Some(crate::queue::OperatorResume {
4609 question_id: "during-slow-proof".into(),
4610 at: Fx::now(),
4611 conductor_rehold: None,
4612 forced: false,
4613 pinned_run: Some(empty.id.clone()),
4614 });
4615 let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
4616 let updated_while_blocked = match tokio::time::timeout(watchdog, &mut update).await {
4620 Ok(result) => {
4621 result.unwrap().unwrap();
4622 true
4623 }
4624 Err(_) => {
4625 std::fs::write(&release, "continue").unwrap();
4626 update.await.unwrap().unwrap();
4627 false
4628 }
4629 };
4630 std::fs::write(release, "continue").unwrap();
4631 let report = tokio::time::timeout(watchdog, sweep)
4632 .await
4633 .expect("Git retirement proof did not finish after release")
4634 .unwrap();
4635 assert!(
4636 updated_while_blocked,
4637 "task writers must not wait behind Git retirement proof"
4638 );
4639 assert_eq!(report.retired, 0);
4640 assert!(report.retained.iter().any(|r| r.contains("task changed")));
4641 assert!(empty.candidates[0].worktree.exists());
4642 assert!(
4643 RunState::load_under(&empty.id, &f.home)
4644 .unwrap()
4645 .retired_to
4646 .is_none()
4647 );
4648 }
4649
4650 #[tokio::test]
4651 async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
4652 let f = Fx::new().await;
4653 let source = f.run("dd01", true).await;
4654 let mut empty = f.run("dd02", false).await;
4655 let expected = f.task(&[source.clone(), empty.clone()]);
4656 let queue = crate::queue::Queue::at(f.home.join("queue"));
4657 let _claim = queue.claim(&expected.id).unwrap();
4658 let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
4660 let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
4661 let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
4662 let mut selected = queue.get(&expected.id).unwrap();
4663 selected.resume_override = Some(crate::queue::OperatorResume {
4664 question_id: "new-owner-decision".into(),
4665 at: Fx::now(),
4666 conductor_rehold: None,
4667 forced: false,
4668 pinned_run: Some(empty.id.clone()),
4669 });
4670 queue.put(&mut selected).unwrap();
4671 let error = commit_retirement(
4672 &f.home,
4673 &queue,
4674 &expected,
4675 &mut empty,
4676 &source.id,
4677 &expected.runs,
4678 Fx::now(),
4679 )
4680 .unwrap_err();
4681 assert!(error.to_string().contains("task changed"));
4682 assert!(
4683 RunState::load_under(&empty.id, &f.home)
4684 .unwrap()
4685 .retired_to
4686 .is_none()
4687 );
4688 assert!(empty.candidates[0].worktree.exists());
4689 assert_eq!(
4690 queue
4691 .get(&expected.id)
4692 .unwrap()
4693 .resume_override
4694 .unwrap()
4695 .pinned_run,
4696 Some(empty.id.clone())
4697 );
4698 assert!(
4699 !f.home
4700 .join("recovery-lineages")
4701 .join(format!("{}.json", expected.id))
4702 .exists()
4703 );
4704 assert!(
4706 commit_retirement(
4707 &f.home,
4708 &queue,
4709 &selected,
4710 &mut empty,
4711 &source.id,
4712 &selected.runs,
4713 Fx::now()
4714 )
4715 .unwrap_err()
4716 .to_string()
4717 .contains("selected this recovery source")
4718 );
4719 }
4720
4721 #[tokio::test]
4722 async fn cargo_owners_and_driver_resumes_are_protected_with_retained_root_data() {
4723 let f = Fx::new().await;
4724 let state = f.run("cccc", false).await;
4725 let wt = &state.candidates[0].worktree;
4726 Fx::target(wt);
4727 std::fs::write(wt.join("target/notes"), "user file").unwrap();
4728 std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
4729 assert!(
4730 !declared_outputs(&wt.join("target"))
4731 .unwrap()
4732 .contains(&wt.join("target/debug/link"))
4733 );
4734 std::fs::remove_file(wt.join("target/debug/link")).unwrap();
4735 let cargo = cargo_locks(&wt.join("target")).unwrap();
4736 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4737 assert_eq!(out.freed_bytes, 0);
4738 assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
4739 drop(cargo);
4740 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
4741 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4742 assert_eq!(out.freed_bytes, 0);
4743 drop(driver);
4744 let exclusive = OutputLease::take(wt, true).unwrap();
4745 assert!(
4746 OutputLease::borrow(wt).is_err(),
4747 "resume cannot start a writer during cleanup"
4748 );
4749 drop(exclusive);
4750 assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
4751 assert_eq!(
4752 std::fs::read(wt.join("target/notes")).unwrap(),
4753 b"user file"
4754 );
4755 }
4756
4757 #[tokio::test]
4758 async fn root_leftovers_allow_eviction_but_keep_empty_attempts_registered() {
4759 for leftover in ["tmp", "magi-tools", "lifecycle-client-check.log"] {
4760 let f = Fx::new().await;
4761 let source = f.run("aa10", true).await;
4762 let empty = f.run("aa11", false).await;
4763 f.task(&[source, empty.clone()]);
4764 let wt = &empty.candidates[0].worktree;
4765 Fx::target(wt);
4766 let path = wt.join("target").join(leftover);
4767 if leftover.ends_with(".log") {
4768 std::fs::write(&path, "keep log\n").unwrap();
4769 } else {
4770 std::fs::create_dir(&path).unwrap();
4772 }
4773 let mut disk = Fx::disk();
4774 disk.auto_fold = true;
4775 let report = sweep(&f.home, &disk, Fx::now()).await;
4776 assert_eq!(report.freed_bytes, 200, "{report:?}");
4777 assert_eq!(report.retired, 0, "{report:?}");
4778 assert!(!wt.join("target/debug/output").exists());
4779 assert!(!residue_proof(&f.home, &empty.id, wt).unwrap().exists());
4780 assert!(path.exists());
4781 if leftover.ends_with(".log") {
4782 assert_eq!(std::fs::read(&path).unwrap(), b"keep log\n");
4783 }
4784 assert_eq!(std::fs::read(wt.join("source")).unwrap(), b"base\n");
4785 assert!(worktrees(&empty).await.unwrap().contains(wt));
4786 assert!(
4787 git::branch_exists(&f.repo, &empty.candidates[0].branch)
4788 .await
4789 .unwrap()
4790 );
4791 assert!(
4792 RunState::load_under(&empty.id, &f.home)
4793 .unwrap()
4794 .retired_to
4795 .is_none()
4796 );
4797 }
4798 }
4799
4800 #[tokio::test]
4801 async fn residue_proof_rejects_new_unknown_root_entries_including_empty_directories() {
4802 let f = Fx::new().await;
4803 let state = f.run("aa12", false).await;
4804 let wt = &state.candidates[0].worktree;
4805 Fx::target(wt);
4806 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4807 assert_eq!(report.freed_bytes, 200, "{report:?}");
4808 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4809 for name in ["tmp", "magi-tools", "log"] {
4810 let path = wt.join("target").join(name);
4811 std::fs::create_dir(&path).unwrap();
4812 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
4813 std::fs::remove_dir(path).unwrap();
4814 }
4815 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4816 }
4817
4818 #[tokio::test]
4819 async fn target_identity_and_known_profiles_still_fail_closed_with_root_leftovers() {
4820 for invalid in [
4821 "tag",
4822 "identity",
4823 "tracked",
4824 "profile-file",
4825 "profile-symlink",
4826 "identity-symlink",
4827 ] {
4828 let f = Fx::new().await;
4829 let state = f.run("aa13", false).await;
4830 let wt = &state.candidates[0].worktree;
4831 Fx::target(wt);
4832 std::fs::create_dir(wt.join("target/tmp")).unwrap();
4833 let target = wt.join("target");
4834 match invalid {
4835 "tag" => std::fs::write(target.join("CACHEDIR.TAG"), "not Cargo").unwrap(),
4836 "identity" => std::fs::write(target.join(".rustc_info.json"), "{}").unwrap(),
4837 "tracked" => {
4838 git::git(wt, &["add", "-f", "target/debug/output"])
4839 .await
4840 .unwrap();
4841 }
4842 "profile-file" => {
4843 std::fs::write(target.join("release"), "keep").unwrap();
4844 }
4845 "profile-symlink" => {
4846 std::os::unix::fs::symlink(target.join("debug"), target.join("release"))
4847 .unwrap();
4848 }
4849 "identity-symlink" => {
4850 std::fs::rename(target.join(".rustc_info.json"), target.join("saved-info"))
4851 .unwrap();
4852 std::os::unix::fs::symlink(
4853 target.join("saved-info"),
4854 target.join(".rustc_info.json"),
4855 )
4856 .unwrap();
4857 }
4858 _ => unreachable!(),
4859 }
4860 let before = std::fs::read(target.join("debug/output")).unwrap();
4861 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4862 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
4863 assert_eq!(std::fs::read(target.join("debug/output")).unwrap(), before);
4864 assert!(target.join("tmp").is_dir());
4865 }
4866 }
4867
4868 #[tokio::test]
4869 async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
4870 let f = Fx::new().await;
4871 let source = f.run("0000", true).await;
4872 let empty = f.run("0001", false).await;
4873 let changed = f.run("0002", false).await;
4874 let unknown = f.run("0003", false).await;
4875 let unproven = f.run("0004", false).await;
4876 f.task(&[
4877 source.clone(),
4878 empty.clone(),
4879 changed.clone(),
4880 unknown.clone(),
4881 unproven.clone(),
4882 ]);
4883 for run in [&empty, &changed, &unknown] {
4884 Fx::target(&run.candidates[0].worktree);
4885 }
4886 let reviewer = empty.worktree_root().join("review-1");
4887 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4888 .await
4889 .unwrap();
4890 Fx::target(&reviewer);
4891 let mut disk = Fx::disk();
4892 disk.auto_fold = false;
4893 let report = sweep(&f.home, &disk, Fx::now()).await;
4894 assert_eq!(report.freed_bytes, 800, "{report:?}");
4895 let wt = &empty.candidates[0].worktree;
4896 assert!(wt.join("target/debug/output.d").exists());
4897 assert!(!wt.join("target/debug/output").exists());
4898 let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
4899 let proof: CargoResidue =
4900 serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
4901 assert_eq!(
4902 proof.files[Path::new("debug/output.d")],
4903 std::fs::read(wt.join("target/debug/output.d")).unwrap()
4904 );
4905 std::fs::write(
4906 changed.candidates[0].worktree.join("target/debug/output.d"),
4907 "owner notes",
4908 )
4909 .unwrap();
4910 std::fs::write(
4911 unknown.candidates[0].worktree.join("target/debug/notes"),
4912 "keep",
4913 )
4914 .unwrap();
4915 Fx::target(&unproven.candidates[0].worktree);
4916 std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
4917 disk.auto_fold = true;
4918 let locks = cargo_locks(&wt.join("target")).unwrap();
4919 let mut report = Report::default();
4920 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4921 assert_eq!(report.retired, 0, "{report:?}");
4922 assert!(
4923 report
4924 .retained
4925 .iter()
4926 .any(|reason| reason.contains("Cargo build"))
4927 );
4928 drop(locks);
4929 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4930 assert_eq!(report.retired, 1, "{report:?}");
4931 assert!(!wt.exists());
4932 assert!(!reviewer.exists());
4933 assert!(
4934 residue_proof(&f.home, &empty.id, &reviewer)
4935 .unwrap()
4936 .exists()
4937 );
4938 assert!(
4939 proof_path.exists(),
4940 "metadata evidence outlives the worktree"
4941 );
4942 assert_eq!(
4943 RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
4944 Some(source.id)
4945 );
4946 for run in [&changed, &unknown, &unproven] {
4947 assert!(run.candidates[0].worktree.exists());
4948 assert!(
4949 RunState::load_under(&run.id, &f.home)
4950 .unwrap()
4951 .retired_to
4952 .is_none()
4953 );
4954 }
4955 }
4956
4957 #[tokio::test]
4958 async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
4959 let f = Fx::new().await;
4960 let source = f.run("0000", true).await;
4961 let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
4962 .await
4963 .unwrap();
4964 let mut runs = vec![source.clone()];
4965 for n in 1..=12 {
4966 runs.push(f.run(&format!("{n:04}"), false).await);
4967 }
4968 let dirty = runs[1].candidates[0].worktree.clone();
4969 std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
4970 let ignored = runs[2].candidates[0].worktree.clone();
4971 std::fs::create_dir(ignored.join("scratch")).unwrap();
4972 std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
4973 runs[1].candidates.clear();
4976 runs[1].save_under(&f.home).unwrap();
4977 let task = f.task(&runs);
4978 let mut total = 0;
4979 for _ in 0..4 {
4980 let mut report = Report::default();
4981 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4982 assert!(report.retired <= 5);
4983 total += report.retired;
4984 }
4985 assert_eq!(total, 10);
4986 assert!(dirty.exists());
4987 assert!(ignored.exists());
4988 assert_eq!(
4989 git::rev_parse(&f.repo, &source.candidates[0].branch)
4990 .await
4991 .unwrap(),
4992 source_head
4993 );
4994 let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
4995 assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
4996 assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
4997 let mut released = task.clone();
4998 released.release();
4999 assert_eq!(
5000 crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
5001 RunState::load_under(id, &f.home)
5002 }),
5003 Some(source.id.clone())
5004 );
5005
5006 assert!(
5007 f.home
5008 .join("runs")
5009 .join(&retired.id)
5010 .join("run.json")
5011 .exists()
5012 );
5013 assert!(
5014 f.home
5015 .join("recovery-lineages")
5016 .join(format!("{}.json", task.id))
5017 .exists()
5018 );
5019 }
5020
5021 #[tokio::test]
5022 async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
5023 let f = Fx::new().await;
5024 let a = f.run("dddd", true).await;
5025 let b = f.run("eeee", true).await;
5026 let empty = f.run("ffff", false).await;
5027 let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
5028 assert!(
5029 crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
5030 "unfinished queue history protects older source branches from ordinary folding"
5031 );
5032 let mut prior = a.clone();
5033 assert!(
5034 crate::graph::fold_run(&mut prior, true, &f.home)
5035 .await
5036 .unwrap_err()
5037 .to_string()
5038 .contains(&task.id)
5039 );
5040 let mut report = Report::default();
5041 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5042 assert_eq!(report.retired, 0);
5043 assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
5044 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
5045 Fx::target(&a.candidates[0].worktree);
5046 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5047 assert!(
5048 report.freed_bytes > 0,
5049 "source retention does not retain regenerable outputs"
5050 );
5051 assert!(a.candidates[0].worktree.exists());
5052 assert!(b.candidates[0].worktree.exists());
5053 assert!(empty.candidates[0].worktree.exists());
5054 }
5055
5056 #[tokio::test]
5057 async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
5058 let f = Fx::new().await;
5059 let authority = f.run("p000", true).await;
5060 let question_run = f.run("p001", false).await;
5061 let mut checkpoint = f.run("p002", false).await;
5062 checkpoint.land_approval = Some(crate::run::LandApproval {
5063 question: "pending".into(),
5064 head: f.base.clone(),
5065 });
5066 checkpoint.save_under(&f.home).unwrap();
5067 let retained = f.run("p003", false).await;
5068 std::fs::write(
5069 f.home
5070 .join("runs")
5071 .join(&retained.id)
5072 .join("retain-recovery-sources"),
5073 "owner",
5074 )
5075 .unwrap();
5076 let mut partial = f.run("p004", false).await;
5077 let reviewer = partial.worktree_root().join("review-1");
5078 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
5079 .await
5080 .unwrap();
5081 partial.retired_to = Some(authority.id.clone());
5083 partial.save_under(&f.home).unwrap();
5084 assert!(
5085 git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
5086 .await
5087 .unwrap()
5088 );
5089 let git_checkpoint = f.run("p005", false).await;
5090 let location = git::git(
5091 &git_checkpoint.candidates[0].worktree,
5092 &[
5093 "rev-parse",
5094 "--path-format=absolute",
5095 "--git-path",
5096 "sequencer",
5097 ],
5098 )
5099 .await
5100 .unwrap();
5101 std::fs::create_dir(location.trim()).unwrap();
5102 let mut preservation = f.run("p006", false).await;
5103 preservation
5104 .candidate_preservation
5105 .insert('A', "pending preservation".into());
5106 preservation.save_under(&f.home).unwrap();
5107 let task = f.task(&[
5108 preservation.clone(),
5109 git_checkpoint.clone(),
5110 authority.clone(),
5111 question_run.clone(),
5112 checkpoint.clone(),
5113 retained.clone(),
5114 partial.clone(),
5115 ]);
5116 let questions = crate::ask::Questions::at(f.home.join("questions"));
5117 let mut question = crate::ask::Question::new(
5118 question_run.id.clone(),
5119 "implement".into(),
5120 "fixture".into(),
5121 "owner decision".into(),
5122 String::new(),
5123 vec!["keep".into()],
5124 );
5125 questions.put(&mut question).unwrap();
5126 let mut report = Report::default();
5127 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5128 assert_eq!(report.retired, 1, "{report:?}");
5129 assert!(!reviewer.exists());
5130 for state in [
5131 &question_run,
5132 &checkpoint,
5133 &retained,
5134 &git_checkpoint,
5135 &preservation,
5136 ] {
5137 assert!(state.candidates[0].worktree.exists());
5138 assert!(
5139 RunState::load_under(&state.id, &f.home)
5140 .unwrap()
5141 .retired_to
5142 .is_none()
5143 );
5144 }
5145 assert!(questions.get(&question.id).unwrap().status.open());
5146 assert_eq!(
5147 crate::queue::Queue::at(f.home.join("queue"))
5148 .get(&task.id)
5149 .unwrap()
5150 .status,
5151 task.status
5152 );
5153 }
5154
5155 #[tokio::test]
5156 async fn preservation_schema_migration_remains_distinct_from_retirement() {
5157 let f = Fx::new().await;
5158 let mut state = f.run("p007", false).await;
5159 state.schema = 15;
5160 state.save_under(&f.home).unwrap();
5161 let native = RunState::load_under(&state.id, &f.home).unwrap();
5162 assert_eq!(native.schema, crate::run::SCHEMA);
5163 assert!(native.candidate_preservation.is_empty());
5164 assert!(native.retired_to.is_none());
5165 state.schema = 14;
5166 state.save_under(&f.home).unwrap();
5167 let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
5168 assert!(legacy.candidate_preservation.contains_key(&'A'));
5169 legacy.save_under(&f.home).unwrap();
5170 let restored = RunState::load_under(&state.id, &f.home).unwrap();
5171 assert_eq!(
5172 restored.candidate_preservation,
5173 legacy.candidate_preservation
5174 );
5175 }
5176
5177 #[tokio::test]
5178 async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
5179 let f = Fx::new().await;
5180 let cache = f.repo.join("cache");
5181 let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
5182 std::fs::create_dir_all(&cache).unwrap();
5183 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
5184 assert!(
5185 crate::cache::maintenance_prune(&f.home, &cache, 1)
5186 .unwrap()
5187 .is_none()
5188 );
5189 drop(guard);
5190 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
5191 }
5192
5193 #[tokio::test]
5194 async fn launcher_death_does_not_release_live_tool_output_reservations() {
5195 let f = Fx::new().await;
5196 let run = f.run("abcd", false).await;
5197 let wt = &run.candidates[0].worktree;
5198 let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
5199 helper
5200 .quiet()
5201 .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
5202 .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
5203 .env("MAGI_TEST_WORKTREE", wt)
5204 .stdin(std::process::Stdio::null())
5205 .stdout(std::process::Stdio::null())
5206 .stderr(std::process::Stdio::null())
5207 .kill_on_drop(true);
5208 assert!(
5209 tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
5210 .await
5211 .unwrap()
5212 .unwrap()
5213 .success()
5214 );
5215 let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
5216 .unwrap()
5217 .trim()
5218 .parse()
5219 .unwrap();
5220 struct KillTool(i32);
5221 impl Drop for KillTool {
5222 fn drop(&mut self) {
5223 unsafe { libc::kill(self.0, libc::SIGKILL) };
5225 }
5226 }
5227 let tool = KillTool(pid);
5228 assert!(OutputLease::take(wt, true).is_err());
5229 assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
5230 drop(tool);
5231 tokio::time::timeout(std::time::Duration::from_secs(5), async {
5232 loop {
5233 if OutputLease::take(wt, true).is_ok()
5234 && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
5235 {
5236 break;
5237 }
5238 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5239 }
5240 })
5241 .await
5242 .unwrap();
5243 }
5244
5245 #[tokio::test]
5246 async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
5247 let f = Fx::new().await;
5248 let run = f.run("abcd", false).await;
5249 let wt = &run.candidates[0].worktree;
5250 let cache = f.repo.join("cache");
5251 std::fs::create_dir(&cache).unwrap();
5252 for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
5253 let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
5254 let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
5255 cmd.quiet()
5256 .args(["--exact", "seat_process::tests::orphan_tool_helper"])
5257 .env("MAGI_TEST_ORPHAN_PID", &pid_file)
5258 .stdin(std::process::Stdio::null())
5259 .stdout(std::process::Stdio::null())
5260 .stderr(std::process::Stdio::null())
5261 .kill_on_drop(true);
5262 if clear_owner {
5263 cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
5264 } else {
5265 cmd.env("MAGI_TEST_CLOSE_FDS", "1");
5266 }
5267 let gib = 1024 * 1024 * 1024;
5268 let mut leases = vec![
5269 OutputLease::borrow(wt).unwrap().unwrap(),
5270 OutputLease::cache(&f.home, &cache, false).unwrap(),
5271 ];
5272 leases.extend(
5273 OutputLease::admit_with(
5274 &f.home,
5275 std::slice::from_ref(wt),
5276 8 * gib,
5277 |_| Ok(9 * gib),
5278 )
5279 .unwrap(),
5280 );
5281 let mut running = crate::seat_process::Running::spawn(cmd, None, leases).unwrap();
5282 assert!(
5283 tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
5284 .await
5285 .unwrap()
5286 .unwrap()
5287 .success()
5288 );
5289 let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
5290 struct KillTool(Option<i32>);
5291 impl Drop for KillTool {
5292 fn drop(&mut self) {
5293 if let Some(pid) = self.0 {
5295 unsafe { libc::kill(pid, libc::SIGKILL) };
5296 }
5297 }
5298 }
5299 let mut tool = KillTool(Some(pid));
5300 assert!(OutputLease::take(wt, true).is_err());
5304 let finish = if cancel {
5305 drop(running);
5306 None
5307 } else {
5308 Some(tokio::spawn(async move { running.finish().await }))
5309 };
5310 if clear_owner {
5311 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
5312 assert!(crate::proc::pid_alive(pid as u32));
5313 assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
5314 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
5315 assert!(
5316 OutputLease::admit_with(&f.home, std::slice::from_ref(wt), 8 * gib, |_| Ok(
5317 9 * gib
5318 ))
5319 .is_err(),
5320 "orphan still owns its disk reservation"
5321 );
5322 assert!(
5323 crate::cache::maintenance_prune(&f.home, &cache, 0)
5324 .unwrap()
5325 .is_none()
5326 );
5327 if let Some(finish) = &finish {
5328 assert!(!finish.is_finished());
5329 }
5330 unsafe { libc::kill(pid, libc::SIGKILL) };
5332 }
5333 if let Some(finish) = finish {
5334 tokio::time::timeout(std::time::Duration::from_secs(5), finish)
5335 .await
5336 .unwrap()
5337 .unwrap();
5338 }
5339 tokio::time::timeout(std::time::Duration::from_secs(5), async {
5340 loop {
5341 if OutputLease::take(wt, true).is_ok()
5342 && OutputLease::cache(&f.home, &cache, true).is_ok()
5343 && OutputLease::admit_with(
5344 &f.home,
5345 std::slice::from_ref(wt),
5346 8 * gib,
5347 |_| Ok(9 * gib),
5348 )
5349 .is_ok()
5350 {
5351 if !clear_owner {
5353 assert!(!crate::proc::pid_alive(pid as u32));
5354 }
5355 break;
5356 }
5357 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5358 }
5359 })
5360 .await
5361 .unwrap();
5362 tool.0 = None;
5363 }
5364 }
5365
5366 #[tokio::test]
5367 async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
5368 let f = Fx::new().await;
5369 let run = f.run("abcd", false).await;
5370 let wt = &run.candidates[0].worktree;
5371 let cache = f.repo.join("cache");
5372 std::fs::create_dir(&cache).unwrap();
5373 let pid_file = f.repo.join("child-pid");
5374 let mut cmd = tokio::process::Command::new("sh");
5375 cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
5376 .quiet()
5377 .env("CHILD_PID_FILE", &pid_file)
5378 .process_group(0)
5379 .kill_on_drop(true);
5380 let running = crate::seat_process::Running::spawn(
5381 cmd,
5382 None,
5383 vec![
5384 OutputLease::borrow(wt).unwrap().unwrap(),
5385 OutputLease::cache(&f.home, &cache, false).unwrap(),
5386 ],
5387 )
5388 .unwrap();
5389 tokio::time::timeout(std::time::Duration::from_secs(5), async {
5390 while !pid_file.exists() {
5391 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5392 }
5393 })
5394 .await
5395 .unwrap();
5396 assert!(OutputLease::take(wt, true).is_err());
5397 assert!(
5398 crate::cache::maintenance_prune(&f.home, &cache, 0)
5399 .unwrap()
5400 .is_none()
5401 );
5402 let pid: u32 = std::fs::read_to_string(pid_file)
5403 .unwrap()
5404 .trim()
5405 .parse()
5406 .unwrap();
5407 drop(running);
5408 tokio::time::timeout(std::time::Duration::from_secs(10), async {
5409 loop {
5410 if let Ok(guard) = OutputLease::take(wt, true) {
5411 assert!(
5412 !crate::proc::pid_alive(pid),
5413 "output lock released before tool death"
5414 );
5415 drop(guard);
5416 break;
5417 }
5418 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5419 }
5420 })
5421 .await
5422 .unwrap();
5423 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
5424 }
5425
5426 #[tokio::test]
5427 async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
5428 let f = Fx::new().await;
5429 let reason = tokio::time::timeout(
5430 std::time::Duration::from_secs(2),
5431 emergency(vec![f.repo.clone()], u64::MAX),
5432 )
5433 .await
5434 .unwrap();
5435 assert!(reason.contains("disk pressure"));
5436 assert!(
5437 tokio::time::timeout(
5438 std::time::Duration::from_millis(10),
5439 emergency(vec![f.repo.clone()], 0)
5440 )
5441 .await
5442 .is_err()
5443 );
5444 assert_eq!(
5445 std::fs::read_to_string(f.repo.join("source")).unwrap(),
5446 "base\n"
5447 );
5448 }
5449
5450 #[tokio::test]
5451 async fn automatic_requeues_cannot_refund_failed_input_generation() {
5452 let f = Fx::new().await;
5453 let mut run = f.run("1234", false).await;
5454 run.prep_failure = Some("cherry-pick conflict".into());
5455 let mut task = f.task(std::slice::from_ref(&run));
5456 let inputs = crate::queue::recovery_inputs(&run.sources);
5457 for _ in 0..108 {
5458 task.recovery_failed(&run);
5459 task.requeue();
5460 assert_eq!(task.attempts, 0);
5461 }
5462 let bytes = serde_json::to_vec(&task).unwrap();
5463 let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
5464 assert!(
5465 restored
5466 .recovery_exhausted(&run.base_commit, &inputs, 2)
5467 .is_some()
5468 );
5469 assert!(
5470 restored
5471 .recovery_exhausted("different-base", &inputs, 2)
5472 .is_none()
5473 );
5474 let original_base = run.base_commit.clone();
5475 run.base_commit = "different-base".into();
5476 restored.recovery_failed(&run);
5477 assert!(
5478 restored
5479 .recovery_exhausted(&original_base, &inputs, 2)
5480 .is_some(),
5481 "returning to old inputs does not replenish their budget"
5482 );
5483 assert!(
5484 restored
5485 .recovery_exhausted(&run.base_commit, &inputs, 2)
5486 .is_none()
5487 );
5488 restored.release_operator("test-owner", None);
5489 assert!(
5490 restored
5491 .recovery_exhausted(&run.base_commit, &inputs, 2)
5492 .is_none()
5493 );
5494 }
5495 async fn inactive_source(f: &Fx, tag: &str, useful: bool) -> RunState {
5496 let mut state = f.run(tag, useful).await;
5497 state.status = RunStatus::Superseded;
5498 state.save_under(&f.home).unwrap();
5499 state
5500 }
5501
5502 fn historical_review(state: &mut RunState, head: &str) {
5503 state.reviews.push(
5504 serde_json::from_value(serde_json::json!({
5505 "round": state.reviews.len() + 1, "head": head, "reviews": []
5506 }))
5507 .unwrap(),
5508 );
5509 }
5510
5511 async fn reviewer(f: &Fx, state: &RunState, head: &str) -> PathBuf {
5512 let path = state.worktree_root().join("review-1");
5513 git::worktree_add_detached(&f.repo, &path, head)
5514 .await
5515 .unwrap();
5516 path
5517 }
5518
5519 #[tokio::test]
5520 async fn source_eviction_separates_run_authority_from_read_seats() {
5521 for status in [RunStatus::Superseded, RunStatus::Ready, RunStatus::Merged] {
5522 let f = Fx::new().await;
5523 let mut state = inactive_source(&f, "ef01", true).await;
5524 state.status = status;
5525 state.pr = Some(
5526 serde_json::from_value(serde_json::json!({
5527 "url":"https://example.invalid/pull/1", "number":1, "state":"merged",
5528 "checks":"green", "round":1, "rounds":1
5529 }))
5530 .unwrap(),
5531 );
5532 state.pr_head = Some(state.candidates[0].branch.clone());
5533 state.pr_merged_head = Some(f.base.clone());
5534 state.recovery = Some(
5535 serde_json::from_value(serde_json::json!({
5536 "from_run": "20260101-000000-abcd", "generation": 1,
5537 "mode": "full-review"
5538 }))
5539 .unwrap(),
5540 );
5541 state.pr_recovery = Some(
5542 serde_json::from_value(serde_json::json!({
5543 "url": "https://example.invalid/pull/1", "branch": "public",
5544 "head": f.base, "source_run": state.id,
5545 "instruction": "keep implementation", "approved_at": Fx::now()
5546 }))
5547 .unwrap(),
5548 );
5549 state.save_under(&f.home).unwrap();
5550 let review = reviewer(&f, &state, &f.base).await;
5551 let mut other = inactive_source(&f, "ef02", true).await;
5552 other.pr_recovery = state.pr_recovery.clone();
5553 other.save_under(&f.home).unwrap();
5554 let mut task = f.task(&[state.clone(), other]);
5555 task.pr_recovery = state.pr_recovery.clone();
5556 crate::queue::Queue::at(f.home.join("queue"))
5557 .put(&mut task)
5558 .unwrap();
5559 let run_path = f.home.join("runs").join(&state.id).join("run.json");
5560 let before = std::fs::read(&run_path).unwrap();
5561 let task_before = serde_json::to_value(&task).unwrap();
5562 let mut report = Report::default();
5563 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5564 assert_eq!(report.evicted_copies, 1, "{status:?}: {report:?}");
5565 assert!(!review.exists());
5566 assert!(state.candidates[0].worktree.exists());
5567 assert!(
5568 git::branch_exists(&f.repo, &state.candidates[0].branch)
5569 .await
5570 .unwrap()
5571 );
5572 assert_eq!(std::fs::read(&run_path).unwrap(), before);
5573 assert_eq!(
5574 serde_json::to_value(
5575 crate::queue::Queue::at(f.home.join("queue"))
5576 .get(&task.id)
5577 .unwrap()
5578 )
5579 .unwrap(),
5580 task_before
5581 );
5582 }
5583 }
5584
5585 #[tokio::test]
5586 async fn source_eviction_retains_own_paths_and_unfinished_continuations() {
5587 for kind in ["own-path", "active-seat", "paused-session"] {
5588 let f = Fx::new().await;
5589 let mut state = inactive_source(&f, "ef03", true).await;
5590 let review = reviewer(&f, &state, &f.base).await;
5591 let expected = match kind {
5592 "own-path" => {
5593 state.attachments.push(review.join("source"));
5594 "run.attachments[0]"
5595 }
5596 "active-seat" => {
5597 state.active.insert(
5598 "review-1".into(),
5599 serde_json::from_value(serde_json::json!({
5600 "node":"review", "started_at":Fx::now(), "timeout_secs":60
5601 }))
5602 .unwrap(),
5603 );
5604 "continuation lease"
5605 }
5606 _ => {
5607 state.status = RunStatus::Blocked;
5608 state.parked = true;
5609 let mut seat = crate::agent::SeatState::new("review-1", "fixture", 1);
5610 seat.turns = 2;
5611 seat.captured_session = Some("saved-session".into());
5612 state.seats.insert("review-1".into(), seat);
5613 "resume contract"
5614 }
5615 };
5616 state.save_under(&f.home).unwrap();
5617 let mut report = Report::default();
5618 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5619 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5620 assert!(review.exists());
5621 assert!(
5622 report.retained.iter().any(|r| r.contains(expected)),
5623 "{kind}: {report:?}"
5624 );
5625 restore_execution_seats(&f.home, &state).await.unwrap();
5628 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), f.base);
5629 }
5630 }
5631
5632 #[tokio::test]
5633 async fn source_eviction_distinguishes_historical_cwd_from_question_dependencies() {
5634 for kind in ["waiting", "lease", "historical", "unknown"] {
5635 let f = Fx::new().await;
5636 let state = inactive_source(&f, "ef06", true).await;
5637 let review = reviewer(&f, &state, &f.base).await;
5638 let store = crate::ask::Questions::at(f.home.join("questions"));
5639 let mut question = crate::ask::Question::new(
5640 "20260101-000000-ffff".into(),
5641 "review".into(),
5642 "review-1".into(),
5643 "fixture".into(),
5644 "fixture".into(),
5645 Vec::new(),
5646 );
5647 question.cwd = Some(review.to_string_lossy().into_owned());
5648 if kind != "waiting" {
5649 question.status = crate::ask::QuestionStatus::Abandoned;
5650 }
5651 store.put(&mut question).unwrap();
5652 if kind == "lease" {
5653 store.beat(&question.id, crate::ask::WaiterKind::Daemon);
5654 }
5655 if kind == "unknown" {
5656 let path = store.path_of(&question.id);
5657 let mut raw: serde_json::Value =
5658 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5659 raw["future_cwd_authority"] = serde_json::Value::Null;
5660 std::fs::write(path, serde_json::to_vec(&raw).unwrap()).unwrap();
5661 }
5662 let mut report = Report::default();
5663 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5664 assert_eq!(
5665 report.evicted_copies,
5666 usize::from(kind == "historical"),
5667 "{kind}: {report:?}"
5668 );
5669 assert_eq!(review.exists(), kind != "historical");
5670 if kind == "waiting" || kind == "lease" {
5671 assert!(
5672 report
5673 .retained
5674 .iter()
5675 .any(|r| r.contains("question") && r.contains("cwd")),
5676 "{report:?}"
5677 );
5678 }
5679 }
5680 }
5681
5682 #[tokio::test]
5683 async fn source_eviction_failed_execution_setup_keeps_its_checkpoint() {
5684 let f = Fx::new().await;
5685 let mut state = inactive_source(&f, "ef07", true).await;
5686 let review = reviewer(&f, &state, &f.base).await;
5687 let mut report = Report::default();
5688 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5689 assert_eq!(report.evicted_copies, 1, "{report:?}");
5690 state.config.worktree.setup = vec![crate::config::SetupStep {
5691 run: Some("printf preserved > source; exit 1".into()),
5692 ..Default::default()
5693 }];
5694 state.save_under(&f.home).unwrap();
5695 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5696 let marker = f
5697 .home
5698 .join("runs")
5699 .join(&state.id)
5700 .join("artifacts/restore-review-1.pending");
5701 assert!(marker.exists());
5702 assert_eq!(
5703 std::fs::read_to_string(review.join("source")).unwrap(),
5704 "preserved"
5705 );
5706 assert!(
5707 restore_execution_seats(&f.home, &state)
5708 .await
5709 .unwrap_err()
5710 .to_string()
5711 .contains("interrupted execution restoration")
5712 );
5713 let mut retry = Report::default();
5714 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5715 assert_eq!(retry.evicted_copies, 0, "{retry:?}");
5716 assert!(
5717 retry
5718 .retained
5719 .iter()
5720 .any(|r| r.contains("restoration checkpoint")),
5721 "{retry:?}"
5722 );
5723 assert!(marker.exists());
5724 assert_eq!(
5725 std::fs::read_to_string(review.join("source")).unwrap(),
5726 "preserved"
5727 );
5728 }
5729
5730 #[tokio::test]
5731 async fn source_eviction_retains_borrowed_or_promisor_anchor_stores() {
5732 for kind in [
5733 "alternates",
5734 "promisor-config",
5735 "promisor-pack",
5736 "objects-symlink",
5737 ] {
5738 let f = Fx::new().await;
5739 let state = inactive_source(&f, "ef11", true).await;
5740 let review = reviewer(&f, &state, &f.base).await;
5741 let common = crate::repo_lock::common_dir(&f.repo)
5742 .await
5743 .unwrap()
5744 .unwrap();
5745 match kind {
5746 "alternates" => {
5747 let borrowed = f._temp.path().join("borrowed-objects");
5748 std::fs::create_dir(&borrowed).unwrap();
5749 std::fs::write(
5750 common.join("objects/info/alternates"),
5751 format!("{}\n", borrowed.display()),
5752 )
5753 .unwrap();
5754 }
5755 "promisor-config" => {
5756 git::git(&f.repo, &["config", "remote.origin.promisor", "true"])
5757 .await
5758 .unwrap();
5759 }
5760 "promisor-pack" => {
5761 std::fs::write(common.join("objects/pack/fixture.promisor"), "").unwrap();
5762 }
5763 _ => {
5764 let borrowed = f._temp.path().join("borrowed-objects");
5765 std::fs::rename(common.join("objects"), &borrowed).unwrap();
5766 std::os::unix::fs::symlink(borrowed, common.join("objects")).unwrap();
5767 }
5768 }
5769 let mut report = Report::default();
5770 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5771 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5772 assert!(review.exists());
5773 assert!(
5774 report.retained.iter().any(|r| r.contains("source anchor")),
5775 "{kind}: {report:?}"
5776 );
5777 }
5778 }
5779
5780 #[tokio::test]
5781 async fn source_eviction_bounds_long_shared_history_and_requires_its_retained_anchor() {
5782 let mut f = Fx::new().await;
5783 crate::fold_evidence::fixtures::shared_history(&f.repo, &f.base, 5000).await;
5784 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5785 assert_eq!(
5786 git::git(&f.repo, &["rev-list", "--count", &f.base])
5787 .await
5788 .unwrap(),
5789 "5001"
5790 );
5791 let state = inactive_source(&f, "ef10", true).await;
5792 let review = reviewer(&f, &state, &f.base).await;
5793 std::fs::write(review.join("source"), "bounded review delta").unwrap();
5794 git::git(&review, &["add", "."]).await.unwrap();
5795 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5796 let head = git::git(
5797 &review,
5798 &["commit-tree", &tree, "-p", &f.base, "-m", "review delta"],
5799 )
5800 .await
5801 .unwrap();
5802 git::git(&review, &["checkout", "--detach", &head])
5803 .await
5804 .unwrap();
5805 let run_file = f.home.join("runs").join(&state.id).join("run.json");
5806 let run_before = std::fs::read(&run_file).unwrap();
5807 let mut report = Report::default();
5808 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5809 assert_eq!(report.evicted_copies, 1, "{report:?}");
5810 assert!(!review.exists());
5811 assert!(state.candidates[0].worktree.exists());
5812 assert!(
5813 git::branch_exists(&f.repo, &state.candidates[0].branch)
5814 .await
5815 .unwrap()
5816 );
5817 assert_eq!(std::fs::read(&run_file).unwrap(), run_before);
5818 let archive = f
5819 .home
5820 .join("runs")
5821 .join(&state.id)
5822 .join("artifacts/source-evidence")
5823 .join(format!("{head}-{}", f.base));
5824 let manifest: serde_json::Value =
5825 serde_json::from_slice(&std::fs::read(archive.join("manifest.json")).unwrap()).unwrap();
5826 assert_eq!(manifest["anchor"], f.base);
5827 assert!(
5828 std::fs::metadata(archive.join("objects.pack"))
5829 .unwrap()
5830 .len()
5831 < 16 * 1024
5832 );
5833 let pin = format!("refs/magi/source-anchors/{}", f.base);
5834 assert_eq!(git::rev_parse(&f.repo, &pin).await.unwrap(), f.base);
5835 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5836 .await
5837 .unwrap();
5838 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5839 assert!(
5840 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5841 .await
5842 .unwrap()
5843 .ok()
5844 );
5845 let other = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5846 .await
5847 .unwrap();
5848 for kind in ["missing", "moved", "symbolic", "symlink"] {
5850 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5851 .await
5852 .unwrap();
5853 match kind {
5854 "moved" => {
5855 git::git(&f.repo, &["update-ref", &pin, &other])
5856 .await
5857 .unwrap();
5858 }
5859 "symbolic" => {
5860 git::git(&f.repo, &["symbolic-ref", &pin, "refs/heads/main"])
5861 .await
5862 .unwrap();
5863 }
5864 "symlink" => {
5865 let common = crate::repo_lock::common_dir(&f.repo)
5866 .await
5867 .unwrap()
5868 .unwrap();
5869 let target = f._temp.path().join("operator-anchor");
5870 std::fs::write(&target, format!("{}\n", f.base)).unwrap();
5871 let file = common.join(&pin);
5872 std::fs::create_dir_all(file.parent().unwrap()).unwrap();
5873 std::os::unix::fs::symlink(target, file).unwrap();
5874 }
5875 _ => {}
5876 }
5877 assert!(
5878 restore_execution_seats(&f.home, &state).await.is_err(),
5879 "{kind}"
5880 );
5881 assert!(!review.exists());
5882 assert!(
5883 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5884 .await
5885 .unwrap()
5886 .ok()
5887 );
5888 if kind == "symlink" {
5889 let common = crate::repo_lock::common_dir(&f.repo)
5890 .await
5891 .unwrap()
5892 .unwrap();
5893 std::fs::remove_file(common.join(&pin)).unwrap();
5894 } else {
5895 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5896 .await
5897 .unwrap();
5898 }
5899 git::git(&f.repo, &["update-ref", &pin, &f.base])
5900 .await
5901 .unwrap();
5902 }
5903 assert!(
5906 git::worktree_remove_clean(&f.repo, &state.candidates[0].worktree)
5907 .await
5908 .unwrap()
5909 );
5910 git::git(
5911 &f.repo,
5912 &[
5913 "update-ref",
5914 "-d",
5915 &format!("refs/heads/{}", state.candidates[0].branch),
5916 ],
5917 )
5918 .await
5919 .unwrap();
5920 git::git(&f.repo, &["update-ref", "-d", "refs/heads/main"])
5921 .await
5922 .unwrap();
5923 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5924 .await
5925 .unwrap();
5926 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5927 assert_eq!(
5928 git::git(&f.repo, &["for-each-ref", "--format=%(refname)"])
5929 .await
5930 .unwrap(),
5931 pin
5932 );
5933 assert_eq!(
5934 git::git(&f.repo, &["rev-list", "--count", &f.base])
5935 .await
5936 .unwrap(),
5937 "5001"
5938 );
5939 restore_execution_seats(&f.home, &state).await.unwrap();
5940 assert_eq!(
5941 git::git(&review, &["rev-list", "--count", "HEAD"])
5942 .await
5943 .unwrap(),
5944 "5002"
5945 );
5946 assert_eq!(
5947 git::git(&f.repo, &["rev-parse", "--is-shallow-repository"])
5948 .await
5949 .unwrap(),
5950 "false"
5951 );
5952 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5953 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5954 let patch = seat_diff(&f.home, &state.id, &head).await.unwrap();
5955 assert!(String::from_utf8_lossy(&patch).contains("bounded review delta"));
5956 std::fs::remove_dir_all(&f.repo).unwrap();
5957 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), patch);
5958 let inspection = f._temp.path().join("shared-history-inspection");
5959 restore_seat(&f.home, &state.id, &head, &inspection)
5960 .await
5961 .unwrap();
5962 assert_eq!(git::rev_parse(&inspection, "HEAD").await.unwrap(), head);
5963 assert_eq!(
5964 git::git(&inspection, &["rev-parse", "--is-shallow-repository"])
5965 .await
5966 .unwrap(),
5967 "true"
5968 );
5969 assert_eq!(
5970 git::git(&inspection, &["log", "--format=%H"])
5971 .await
5972 .unwrap(),
5973 format!("{head}\n{}", f.base)
5974 );
5975 assert!(!inspection.join(".git/objects/info/alternates").exists());
5976 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5977 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5978 }
5979
5980 #[tokio::test]
5981 async fn source_eviction_retains_tree_only_archives_and_rejects_import_before_writes() {
5982 let f = Fx::new().await;
5983 let state = inactive_source(&f, "ef08", true).await;
5984 let review = reviewer(&f, &state, &f.base).await;
5985 let tree = git::rev_parse(&review, "HEAD^{tree}").await.unwrap();
5986 let parent = git::git(&review, &["commit-tree", &tree, "-m", "U0"])
5987 .await
5988 .unwrap();
5989 let head = git::git(&review, &["commit-tree", &tree, "-p", &parent, "-m", "U1"])
5990 .await
5991 .unwrap();
5992 git::git(&review, &["checkout", "--detach", &head])
5993 .await
5994 .unwrap();
5995 let archive =
5996 crate::fold_evidence::fixtures::tree_only_archive(&state, &f.home, &head).await;
5997 let mut report = Report::default();
5998 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5999 assert_eq!(report.evicted_copies, 0, "{report:?}");
6000 assert!(review.exists());
6001 assert!(
6002 report
6003 .retained
6004 .iter()
6005 .any(|r| r.contains("incomplete archived commit ancestry")),
6006 "{report:?}"
6007 );
6008 assert!(git::worktree_remove_clean(&f.repo, &review).await.unwrap());
6010 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6011 .await
6012 .unwrap();
6013 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6014 for oid in [&parent, &head] {
6015 assert!(
6016 !git::git_raw(&f.repo, &["cat-file", "-e", oid])
6017 .await
6018 .unwrap()
6019 .ok()
6020 );
6021 }
6022 let patch = source_diff(&archive).await.unwrap();
6023 assert!(
6024 crate::fold_evidence::import_snapshot(&archive, &f.repo)
6025 .await
6026 .is_err()
6027 );
6028 assert!(
6029 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
6030 .await
6031 .unwrap()
6032 .ok()
6033 );
6034 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
6035 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6036 let inspection = f._temp.path().join("legacy-tree-inspection");
6037 restore_source(&archive, &inspection).await.unwrap();
6038 assert_eq!(git::git(&inspection, &["write-tree"]).await.unwrap(), tree);
6039 assert!(
6040 !git::git_raw(&inspection, &["rev-parse", "--verify", "HEAD"])
6041 .await
6042 .unwrap()
6043 .ok()
6044 );
6045 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
6046 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
6047 assert_eq!(source_diff(&archive).await.unwrap(), patch);
6048 assert!(state.candidates[0].worktree.exists());
6049 }
6050
6051 #[tokio::test]
6052 async fn source_eviction_retains_ancestry_above_expanded_budget() {
6053 let f = Fx::new().await;
6054 let state = inactive_source(&f, "ef09", true).await;
6055 let review = reviewer(&f, &state, &f.base).await;
6056 std::fs::File::create(review.join("ancestor-only"))
6057 .unwrap()
6058 .set_len(64 * 1024 * 1024 + 1)
6059 .unwrap();
6060 git::git(&review, &["add", "."]).await.unwrap();
6061 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
6062 let parent = git::git(
6063 &review,
6064 &["commit-tree", &parent_tree, "-m", "large ancestor"],
6065 )
6066 .await
6067 .unwrap();
6068 std::fs::remove_file(review.join("ancestor-only")).unwrap();
6069 git::git(&review, &["add", "."]).await.unwrap();
6070 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6071 let head = git::git(
6072 &review,
6073 &[
6074 "commit-tree",
6075 &tree,
6076 "-p",
6077 &parent,
6078 "-m",
6079 "small final tree",
6080 ],
6081 )
6082 .await
6083 .unwrap();
6084 git::git(&review, &["checkout", "--detach", &head])
6085 .await
6086 .unwrap();
6087 let mut report = Report::default();
6088 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6089 assert_eq!(report.evicted_copies, 0, "{report:?}");
6090 assert!(
6091 report
6092 .retained
6093 .iter()
6094 .any(|r| r.contains("expanded object limit exceeded")),
6095 "{report:?}"
6096 );
6097 assert!(review.exists());
6098 assert!(state.candidates[0].worktree.exists());
6099 }
6100
6101 #[tokio::test]
6102 async fn source_eviction_restores_execution_after_gc_and_preserves_generations() {
6103 let f = Fx::new().await;
6104 let state = inactive_source(&f, "ef04", true).await;
6105 let review = reviewer(&f, &state, &f.base).await;
6106 std::fs::write(review.join("ancestor-only"), "ancestral content").unwrap();
6107 git::git(&review, &["add", "."]).await.unwrap();
6108 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
6109 let parent = git::git(
6110 &review,
6111 &["commit-tree", &parent_tree, "-m", "unrelated parent"],
6112 )
6113 .await
6114 .unwrap();
6115 std::fs::remove_file(review.join("ancestor-only")).unwrap();
6116 std::fs::write(review.join("source"), "unrelated review checkpoint").unwrap();
6117 git::git(&review, &["add", "."]).await.unwrap();
6118 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6119 let head = git::git(
6120 &review,
6121 &[
6122 "commit-tree",
6123 &tree,
6124 "-p",
6125 &parent,
6126 "-m",
6127 "unrelated review checkpoint",
6128 ],
6129 )
6130 .await
6131 .unwrap();
6132 git::git(&review, &["checkout", "--detach", &head])
6133 .await
6134 .unwrap();
6135 let mut report = Report::default();
6136 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6137 assert_eq!(report.evicted_copies, 1, "{report:?}");
6138 let ledger = f
6139 .home
6140 .join("runs")
6141 .join(&state.id)
6142 .join("artifacts/eviction-review-1.json");
6143 let original = std::fs::read(&ledger).unwrap();
6144 let diff = seat_diff(&f.home, &state.id, &head).await.unwrap();
6145 assert!(String::from_utf8_lossy(&diff).contains("unrelated review checkpoint"));
6146 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6147 .await
6148 .unwrap();
6149 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6150 assert!(
6151 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
6152 .await
6153 .unwrap()
6154 .ok()
6155 );
6156 assert!(
6157 !git::git_raw(&f.repo, &["cat-file", "-e", &parent])
6158 .await
6159 .unwrap()
6160 .ok()
6161 );
6162 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
6163 assert!(
6164 seat_heads(&f.home, &state.id)
6165 .await
6166 .unwrap()
6167 .contains_key(&head)
6168 );
6169 restore_execution_seats(&f.home, &state).await.unwrap();
6170 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), head);
6171 assert_eq!(
6172 git::git(&review, &["log", "--format=%H"]).await.unwrap(),
6173 format!("{head}\n{parent}")
6174 );
6175 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
6176 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6177 let inspection = f._temp.path().join("complete-history-inspection");
6178 restore_seat(&f.home, &state.id, &head, &inspection)
6179 .await
6180 .unwrap();
6181 assert_eq!(
6182 git::git(&inspection, &["log", "--format=%H"])
6183 .await
6184 .unwrap(),
6185 format!("{head}\n{parent}")
6186 );
6187 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
6188 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
6189
6190 assert_eq!(git::git(&review, &["write-tree"]).await.unwrap(), tree);
6191 assert!(
6192 !git::git_raw(&review, &["symbolic-ref", "-q", "HEAD"])
6193 .await
6194 .unwrap()
6195 .ok()
6196 );
6197 assert_eq!(
6198 std::fs::read_to_string(review.join("source")).unwrap(),
6199 "unrelated review checkpoint"
6200 );
6201 std::fs::write(review.join("operator-notes"), "keep").unwrap();
6202 assert!(restore_execution_seats(&f.home, &state).await.is_err());
6203 assert_eq!(
6204 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
6205 "keep"
6206 );
6207 std::fs::remove_file(review.join("operator-notes")).unwrap();
6208 restore_execution_seats(&f.home, &state).await.unwrap();
6209 let mut second = Report::default();
6210 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
6211 assert_eq!(second.evicted_copies, 1, "{second:?}");
6212 assert_eq!(std::fs::read(&ledger).unwrap(), original);
6213 assert!(ledger.with_file_name("eviction-review-1-2.json").exists());
6214 let mut legacy: serde_json::Value = serde_json::from_slice(&original).unwrap();
6217 legacy["version"] = 1.into();
6218 legacy.as_object_mut().unwrap().remove("base");
6219 std::fs::write(&ledger, serde_json::to_vec(&legacy).unwrap()).unwrap();
6220 let mut advanced = state.clone();
6221 advanced.base_commit = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6222 .await
6223 .unwrap();
6224 advanced.save_under(&f.home).unwrap();
6225 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
6226 let raw: serde_json::Value = serde_json::from_slice(&original).unwrap();
6228 let archive = PathBuf::from(raw["evidence"].as_str().unwrap());
6229 let patch = std::fs::read(archive.join("change.patch")).unwrap();
6230 std::fs::write(archive.join("change.patch"), "corrupt").unwrap();
6231 assert!(restore_execution_seats(&f.home, &state).await.is_err());
6232 assert!(!review.exists());
6233 std::fs::write(archive.join("change.patch"), patch).unwrap();
6234 std::fs::create_dir(&review).unwrap();
6235 std::fs::write(review.join("operator-notes"), "keep").unwrap();
6236 assert!(restore_execution_seats(&f.home, &state).await.is_err());
6237 assert_eq!(
6238 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
6239 "keep"
6240 );
6241 }
6242
6243 #[tokio::test]
6244 async fn source_eviction_preserves_history_and_restores_binary_modes_after_gc() {
6245 use std::os::unix::fs::PermissionsExt;
6246 let mut f = Fx::new().await;
6247 std::fs::write(f.repo.join("binary"), b"old\0binary\xff").unwrap();
6248 git::git(&f.repo, &["add", "."]).await.unwrap();
6249 git::git(&f.repo, &["commit", "-m", "binary base"])
6250 .await
6251 .unwrap();
6252 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
6253 let mut state = inactive_source(&f, "ec01", true).await;
6254 let candidate = state.candidates[0].worktree.clone();
6255 let earlier = git::rev_parse(&candidate, "HEAD").await.unwrap();
6256 historical_review(&mut state, &earlier);
6257 let review = reviewer(&f, &state, &earlier).await;
6258 std::fs::remove_file(candidate.join("source")).unwrap();
6259 std::fs::write(candidate.join("binary"), b"new\0binary\xfe").unwrap();
6260 std::fs::write(candidate.join("script"), "run\n").unwrap();
6261 std::fs::set_permissions(
6262 candidate.join("script"),
6263 std::fs::Permissions::from_mode(0o755),
6264 )
6265 .unwrap();
6266 git::git(&candidate, &["add", "."]).await.unwrap();
6267 git::git(&candidate, &["commit", "-m", "binary deletion and mode"])
6268 .await
6269 .unwrap();
6270 let head = git::rev_parse(&candidate, "HEAD").await.unwrap();
6271 let tree = git::rev_parse(&candidate, "HEAD^{tree}").await.unwrap();
6272 let base_tree = git::rev_parse(&candidate, &format!("{}^{{tree}}", f.base))
6273 .await
6274 .unwrap();
6275 historical_review(&mut state, &head);
6276 historical_review(&mut state, &head);
6277 state.status = RunStatus::Superseded;
6278 state.save_under(&f.home).unwrap();
6279 let task = f.task(std::slice::from_ref(&state));
6280 let before = serde_json::to_value(&state).unwrap();
6281 git::git(&review, &["reset", "--hard", &head])
6282 .await
6283 .unwrap();
6284 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
6285 assert_eq!(report.evicted_copies, 1, "{report:?}");
6286 assert!(!review.exists());
6287 assert!(candidate.exists(), "selected implementation survives");
6288 assert!(report.source_unlinked_logical_bytes > 0);
6289 assert!(report.source_unlinked_allocated_bytes.is_some());
6290 assert!(report.source_evidence_added_allocated_bytes.is_some());
6291 assert_eq!(
6292 serde_json::to_value(RunState::load_under(&state.id, &f.home).unwrap()).unwrap(),
6293 before
6294 );
6295 assert_eq!(
6296 crate::queue::Queue::at(f.home.join("queue"))
6297 .get(&task.id)
6298 .unwrap()
6299 .runs,
6300 task.runs
6301 );
6302 let archive = f
6303 .home
6304 .join("runs")
6305 .join(&state.id)
6306 .join("artifacts/source-evidence")
6307 .join(format!("{head}-{}", f.base));
6308 let patch = source_diff(&archive).await.unwrap();
6309 assert!(String::from_utf8_lossy(&patch).contains("GIT binary patch"));
6310 assert!(String::from_utf8_lossy(&patch).contains("100755"));
6311 assert!(
6312 git::worktree_remove_clean(&f.repo, &candidate)
6313 .await
6314 .unwrap()
6315 );
6316 git::git(&f.repo, &["branch", "-D", &state.candidates[0].branch])
6317 .await
6318 .unwrap();
6319 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6320 .await
6321 .unwrap();
6322 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6323 assert!(
6324 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
6325 .await
6326 .unwrap()
6327 .ok()
6328 );
6329 std::fs::remove_dir_all(&f.repo).unwrap();
6331 assert_eq!(source_diff(&archive).await.unwrap(), patch);
6332 let restored = f._temp.path().join("restored");
6333 restore_source(&archive, &restored).await.unwrap();
6334 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
6335 assert_eq!(
6336 std::fs::read(restored.join("binary")).unwrap(),
6337 b"new\0binary\xfe"
6338 );
6339 assert!(!restored.join("source").exists());
6340 assert_ne!(
6341 std::fs::metadata(restored.join("script"))
6342 .unwrap()
6343 .permissions()
6344 .mode()
6345 & 0o111,
6346 0
6347 );
6348 assert!(restore_source(&archive, &restored).await.is_err());
6349 git::git(&restored, &["read-tree", "--reset", "-u", &base_tree])
6352 .await
6353 .unwrap();
6354 let patch_file = f._temp.path().join("saved.patch");
6355 std::fs::write(&patch_file, &patch).unwrap();
6356 git::git(
6357 &restored,
6358 &["apply", "--index", "--binary", patch_file.to_str().unwrap()],
6359 )
6360 .await
6361 .unwrap();
6362 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
6363 assert_eq!(
6364 std::fs::read(restored.join("binary")).unwrap(),
6365 b"new\0binary\xfe"
6366 );
6367 }
6368
6369 #[tokio::test]
6370 async fn source_eviction_archives_unrelated_tips_and_allows_lineage_references() {
6371 let f = Fx::new().await;
6372 let state = inactive_source(&f, "ec10", true).await;
6373 let review = reviewer(&f, &state, &f.base).await;
6374 let mut successor = inactive_source(&f, "ec11", true).await;
6375 successor.retired_to = Some(state.id.clone());
6376 successor.save_under(&f.home).unwrap();
6377 f.task(&[state.clone(), successor]);
6378
6379 let mut tips = Vec::new();
6380 for contents in ["reflog-only evidence", "ORIG_HEAD-only evidence"] {
6381 std::fs::write(review.join("source"), contents).unwrap();
6382 git::git(&review, &["add", "."]).await.unwrap();
6383 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6384 let tip = git::git(
6385 &review,
6386 &["commit-tree", &tree, "-p", &f.base, "-m", contents],
6387 )
6388 .await
6389 .unwrap();
6390 git::git(&review, &["reset", "--hard", &tip]).await.unwrap();
6391 tips.push(tip);
6392 git::git(&review, &["reset", "--hard", &f.base])
6393 .await
6394 .unwrap();
6395 }
6396 git::git(&review, &["reflog", "delete", "--rewrite", "HEAD@{1}"])
6398 .await
6399 .unwrap();
6400 git::git(&review, &["update-ref", "ORIG_HEAD", &tips[1]])
6401 .await
6402 .unwrap();
6403 let mut report = Report::default();
6404 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6405 assert_eq!(report.evicted_copies, 1, "{report:?}");
6406 assert!(!review.exists());
6407 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6408 .await
6409 .unwrap();
6410 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6411 for tip in &tips {
6412 assert!(
6413 !git::git_raw(&f.repo, &["cat-file", "-e", tip])
6414 .await
6415 .unwrap()
6416 .ok()
6417 );
6418 }
6419 std::fs::remove_dir_all(&f.repo).unwrap();
6421 let shown = seat_evidence(&f.home, &state.id).await.unwrap();
6422 for (i, tip) in tips.iter().enumerate() {
6423 assert!(
6424 shown
6425 .iter()
6426 .any(|row| row.contains(tip) && row.contains("patch_bytes="))
6427 );
6428 let destination = f._temp.path().join(format!("restore-history-{i}"));
6429 restore_seat(&f.home, &state.id, tip, &destination)
6430 .await
6431 .unwrap();
6432 assert_eq!(
6433 std::fs::read_to_string(destination.join("source")).unwrap(),
6434 ["reflog-only evidence", "ORIG_HEAD-only evidence"][i]
6435 );
6436 }
6437 assert!(
6438 restore_seat(
6439 &f.home,
6440 &state.id,
6441 "unknown",
6442 &f._temp.path().join("missing")
6443 )
6444 .await
6445 .is_err()
6446 );
6447 }
6448
6449 #[tokio::test]
6450 async fn source_eviction_revalidates_archive_immediately_before_unlink() {
6451 fn corrupt(path: &Path) -> Result<()> {
6452 let root = path.parent().unwrap();
6453 let marker = root.join("archive-to-corrupt");
6456 let archive = std::fs::read_to_string(&marker)?;
6457 std::fs::write(Path::new(&archive).join("change.patch"), "corrupt")?;
6458 Ok(())
6459 }
6460 let f = Fx::new().await;
6461 let state = inactive_source(&f, "ec12", true).await;
6462 let review = reviewer(&f, &state, &f.base).await;
6463 let archive = f
6464 .home
6465 .join("runs")
6466 .join(&state.id)
6467 .join("artifacts/source-evidence")
6468 .join(format!("{}-{}", f.base, f.base));
6469 std::fs::write(
6470 state.worktree_root().join("archive-to-corrupt"),
6471 archive.to_str().unwrap(),
6472 )
6473 .unwrap();
6474 let mut report = Report::default();
6475 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut report, corrupt).await;
6476 assert_eq!(report.evicted_copies, 0, "{report:?}");
6477 assert!(review.exists());
6478 assert!(
6479 report
6480 .retained
6481 .iter()
6482 .any(|r| r.contains("corrupt snapshot")),
6483 "{report:?}"
6484 );
6485 }
6486
6487 #[tokio::test]
6488 async fn source_eviction_fails_closed_for_dirty_ignored_borrowed_and_authoritative_copies() {
6489 for kind in [
6490 "dirty",
6491 "hidden",
6492 "ignored",
6493 "empty-directory",
6494 "untracked",
6495 "borrowed",
6496 "borrowed-symlink",
6497 "lease",
6498 "owner",
6499 "driver",
6500 "task-pin",
6501 "task-replacement",
6502 "malformed-orig-head",
6503 "grace",
6504 "resumable",
6505 "archive-symlink",
6506 "root-symlink",
6507 "checkpoint",
6508 "symlink",
6509 "retained",
6510 "unreadable",
6511 "unknown-field",
6512 "unknown-task-field",
6513 ] {
6514 let f = Fx::new().await;
6515 let mut state = inactive_source(&f, "ec02", true).await;
6516 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6517 .await
6518 .unwrap();
6519 historical_review(&mut state, &head);
6520 let review = reviewer(&f, &state, &head).await;
6521 let mut output = None;
6522 let mut owner = None;
6523 let mut driver = None;
6524 let mut disk = Fx::disk();
6525 match kind {
6526 "dirty" => std::fs::write(review.join("source"), "notes").unwrap(),
6527 "hidden" => {
6528 git::git(&review, &["update-index", "--assume-unchanged", "source"])
6529 .await
6530 .unwrap();
6531 std::fs::write(review.join("source"), "notes").unwrap();
6532 }
6533 "ignored" => {
6534 std::fs::create_dir(review.join("scratch")).unwrap();
6535 std::fs::write(review.join("scratch/notes"), "notes").unwrap();
6536 }
6537 "empty-directory" => std::fs::create_dir(review.join("owner-notes")).unwrap(),
6538 "untracked" => std::fs::write(review.join("notes"), "notes").unwrap(),
6539 "borrowed" => {
6540 let mut other = f.run("ec03", true).await;
6541 other.candidates[0].worktree = review.clone();
6542 other.save_under(&f.home).unwrap();
6543 }
6544 "borrowed-symlink" => {
6545 let alias = f._temp.path().join("borrowed-alias");
6546 std::os::unix::fs::symlink(&review, &alias).unwrap();
6547 let mut other = f.run("ec03", true).await;
6548 other.candidates[0].worktree = alias;
6549 other.save_under(&f.home).unwrap();
6550 }
6551 "lease" => output = OutputLease::borrow(&review).unwrap(),
6552 "owner" => owner = Some(crate::pr_head::owner_lock(&f.home, &state.id).unwrap()),
6553 "driver" => driver = Some(crate::driver::maintenance(&f.home, &state.id).unwrap()),
6554 "task-pin" => {
6555 let mut task = f.task(std::slice::from_ref(&state));
6556 task.resume_override = Some(crate::queue::OperatorResume {
6557 question_id: "fixture-pin".into(),
6558 at: Fx::now(),
6559 conductor_rehold: None,
6560 forced: true,
6561 pinned_run: Some(state.id.clone()),
6562 });
6563 crate::queue::Queue::at(f.home.join("queue"))
6564 .put(&mut task)
6565 .unwrap();
6566 }
6567 "task-replacement" => {
6568 let mut task = f.task(std::slice::from_ref(&state));
6569 task.replacement = Some(crate::merge_decision::Replacement {
6570 question: "fixture-replacement".into(),
6571 run: state.id.clone(),
6572 pr: "https://example.invalid/pull/1".into(),
6573 agent: "worker".into(),
6574 context: String::new(),
6575 });
6576 crate::queue::Queue::at(f.home.join("queue"))
6577 .put(&mut task)
6578 .unwrap();
6579 }
6580 "malformed-orig-head" => {
6581 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6582 .await
6583 .unwrap();
6584 std::fs::write(Path::new(&admin).join("ORIG_HEAD"), "not-an-object-id")
6585 .unwrap();
6586 }
6587 "grace" => disk.fold_grace_secs = 3600,
6588 "resumable" => state.status = RunStatus::Blocked,
6589 "archive-symlink" => {
6590 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6591 std::fs::create_dir_all(&artifacts).unwrap();
6592 let outside = f._temp.path().join("outside");
6593 std::fs::create_dir(&outside).unwrap();
6594 std::os::unix::fs::symlink(&outside, artifacts.join("source-evidence"))
6595 .unwrap();
6596 }
6597 "root-symlink" => {
6598 let root = state.worktree_root();
6599 let outside = f._temp.path().join("outside-root");
6600 std::fs::rename(&root, &outside).unwrap();
6601 std::os::unix::fs::symlink(&outside, &root).unwrap();
6602 }
6603 "checkpoint" => {
6604 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6605 .await
6606 .unwrap();
6607 std::fs::write(Path::new(&admin).join("MERGE_MSG"), "checkpoint").unwrap();
6608 }
6609 "symlink" => {
6610 std::fs::remove_file(review.join("source")).unwrap();
6611 std::os::unix::fs::symlink(".gitignore", review.join("source")).unwrap();
6612 }
6613 "retained" => std::fs::write(
6614 f.home
6615 .join("runs")
6616 .join(&state.id)
6617 .join("retain-recovery-sources"),
6618 "owner",
6619 )
6620 .unwrap(),
6621 "unreadable" => {
6622 std::fs::create_dir_all(f.home.join("runs/20260101-000000-ffff")).unwrap();
6623 }
6624 "unknown-field" | "unknown-task-field" => {}
6625 _ => unreachable!(),
6626 }
6627 state.save_under(&f.home).unwrap();
6628 if kind == "unknown-field" {
6629 let path = f.home.join("runs").join(&state.id).join("run.json");
6630 let mut raw: serde_json::Value =
6631 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6632 raw["future_source_owner"] = serde_json::json!({"path": review});
6633 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6634 }
6635 if kind == "unknown-task-field" {
6636 let task = f.task(std::slice::from_ref(&state));
6637 let path = crate::queue::Queue::at(f.home.join("queue")).path_of(&task.id);
6638 let mut raw: serde_json::Value =
6639 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6640 raw["future_source_owner"] = serde_json::json!({"path": review});
6641 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6642 }
6643 let mut report = Report::default();
6644 copies::evict(&f.home, &disk, Fx::now(), &mut report).await;
6645 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
6646 assert!(review.exists(), "{kind}: {report:?}");
6647 assert!(!report.retained.is_empty(), "{kind}: {report:?}");
6648 drop(output);
6649 drop(owner);
6650 drop(driver);
6651 }
6652 }
6653
6654 #[tokio::test]
6655 async fn source_eviction_reconciles_interruption_and_refuses_corrupt_archives() {
6656 let f = Fx::new().await;
6657 let state = inactive_source(&f, "ec04", true).await;
6658 let review = reviewer(&f, &state, &f.base).await;
6659 let mut report = Report::default();
6660 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6661 assert_eq!(report.evicted_copies, 1, "{report:?}");
6662 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6663 let ledger = artifacts.join("eviction-review-1.json");
6664 let mut record: serde_json::Value =
6665 serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6666 record["phase"] = "pending".into();
6667 std::fs::write(&ledger, serde_json::to_vec(&record).unwrap()).unwrap();
6668 let archive = PathBuf::from(record["evidence"].as_str().unwrap());
6669 let pack = std::fs::read(archive.join("objects.pack")).unwrap();
6670 std::fs::write(archive.join("objects.pack"), b"corrupt").unwrap();
6671 let mut blocked = Report::default();
6672 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6673 assert!(
6674 blocked
6675 .retained
6676 .iter()
6677 .any(|r| r.contains("corrupt snapshot")),
6678 "{blocked:?}"
6679 );
6680 assert_eq!(
6681 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6682 ["phase"],
6683 "pending"
6684 );
6685 assert!(
6686 restore_source(&archive, &f._temp.path().join("bad-restore"))
6687 .await
6688 .is_err()
6689 );
6690 std::fs::write(archive.join("objects.pack"), &pack).unwrap();
6691 let mut retried = Report::default();
6692 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6693 assert_eq!(
6694 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6695 ["phase"],
6696 "evicted"
6697 );
6698 assert_eq!(retried.evicted_copies, 0);
6699 assert!(!review.exists());
6700 let pending = archive.with_extension("pending");
6702 std::fs::rename(&archive, &pending).unwrap();
6703 assert_eq!(
6704 crate::fold_evidence::snapshot(&state, &f.home, &f.base, 64 * 1024 * 1024)
6705 .await
6706 .unwrap(),
6707 archive
6708 );
6709 assert!(!pending.exists());
6710 }
6711
6712 #[tokio::test]
6713 async fn source_eviction_reclaims_superseded_empty_attempt_without_retiring_lineage() {
6714 let f = Fx::new().await;
6715 let mut empty = inactive_source(&f, "ec05", false).await;
6716 historical_review(&mut empty, &f.base);
6717 empty.status = RunStatus::Superseded;
6718 empty.save_under(&f.home).unwrap();
6719 let source = inactive_source(&f, "ec06", true).await;
6720 let task = f.task(&[empty.clone(), source.clone()]);
6721 let before = serde_json::to_value(&empty).unwrap();
6722 let mut report = Report::default();
6723 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6724 assert_eq!(report.evicted_copies, 1, "{report:?}");
6725 assert!(!empty.candidates[0].worktree.exists());
6726 assert!(source.candidates[0].worktree.exists());
6727 assert!(
6728 git::branch_exists(&f.repo, &empty.candidates[0].branch)
6729 .await
6730 .unwrap()
6731 );
6732 assert_eq!(
6733 serde_json::to_value(RunState::load_under(&empty.id, &f.home).unwrap()).unwrap(),
6734 before
6735 );
6736 let mut retry = Report::default();
6737 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6738 assert_eq!(retry.removed_empty_roots, 1, "{retry:?}");
6739 assert!(!empty.worktree_root().exists());
6740 assert_eq!(
6741 crate::queue::Queue::at(f.home.join("queue"))
6742 .get(&task.id)
6743 .unwrap()
6744 .runs,
6745 task.runs
6746 );
6747 }
6748 #[tokio::test]
6749 async fn source_eviction_retries_pending_intent_and_exact_missing_registration() {
6750 fn interrupt(_: &Path) -> Result<()> {
6751 anyhow::bail!("fixture interruption before unlink")
6752 }
6753 fn vanish(path: &Path) -> Result<()> {
6754 std::fs::remove_dir_all(path)?;
6755 anyhow::bail!("fixture interruption after checkout unlink")
6756 }
6757 for stage in [
6758 "intent",
6759 "missing-registration",
6760 "changed-admin",
6761 "missing-archive",
6762 "wrong-identity",
6763 ] {
6764 let f = Fx::new().await;
6765 let state = inactive_source(&f, "ec07", true).await;
6766 let review = reviewer(&f, &state, &f.base).await;
6767 let admin = PathBuf::from(
6768 git::git(&review, &["rev-parse", "--absolute-git-dir"])
6769 .await
6770 .unwrap(),
6771 );
6772 let mut interrupted = Report::default();
6773 copies::evict_with(
6774 &f.home,
6775 &Fx::disk(),
6776 Fx::now(),
6777 &mut interrupted,
6778 if stage == "missing-registration" || stage == "changed-admin" {
6779 vanish
6780 } else {
6781 interrupt
6782 },
6783 )
6784 .await;
6785 assert_eq!(interrupted.evicted_copies, 0, "{stage}: {interrupted:?}");
6786 let ledger = f
6787 .home
6788 .join("runs")
6789 .join(&state.id)
6790 .join("artifacts/eviction-review-1.json");
6791 let record: serde_json::Value =
6792 serde_json::from_slice(&std::fs::read(&ledger).unwrap_or_else(|error| {
6793 panic!("{stage}: missing pending ledger: {error}; {interrupted:?}")
6794 }))
6795 .unwrap();
6796 assert_eq!(record["phase"], "pending");
6797 if stage == "changed-admin" {
6798 std::fs::write(admin.join("owner-note"), "keep").unwrap();
6799 }
6800 if stage == "missing-archive" {
6801 std::fs::remove_file(
6802 Path::new(record["evidence"].as_str().unwrap()).join("objects.pack"),
6803 )
6804 .unwrap();
6805 }
6806 if stage == "wrong-identity" {
6807 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6808 .await
6809 .unwrap();
6810 let other =
6811 crate::fold_evidence::snapshot(&state, &f.home, &head, 64 * 1024 * 1024)
6812 .await
6813 .unwrap();
6814 let original = Path::new(record["evidence"].as_str().unwrap());
6815 for name in [
6816 "manifest.json",
6817 "objects.pack",
6818 "change.patch",
6819 "change.index",
6820 ] {
6821 std::fs::copy(other.join(name), original.join(name)).unwrap();
6822 }
6823 }
6824 let mut retried = Report::default();
6825 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6826 let phase =
6827 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap())
6828 .unwrap()["phase"]
6829 .clone();
6830 if stage == "changed-admin" || stage == "missing-archive" || stage == "wrong-identity" {
6831 assert_eq!(phase, "pending", "{stage}: {retried:?}");
6832 assert!(!retried.retained.is_empty());
6833 if stage == "changed-admin" {
6834 assert!(admin.join("owner-note").exists());
6835 } else {
6836 assert!(review.exists());
6837 }
6838 } else {
6839 assert_eq!(phase, "evicted", "{stage}: {retried:?}");
6840 assert!(!review.exists());
6841 assert!(!admin.exists());
6842 }
6843 }
6844 }
6845 fn source_census(home: &Path) -> serde_json::Value {
6846 serde_json::from_slice(&std::fs::read(home.join("source-eviction-cursor")).unwrap())
6847 .unwrap()
6848 }
6849
6850 #[tokio::test]
6851 async fn source_eviction_scan_bounds_missing_root_checkpoints_and_eventually_reclaims() {
6852 let f = Fx::new().await;
6853 let state = inactive_source(&f, "ffff", true).await;
6854 let review = reviewer(&f, &state, &f.base).await;
6855 for i in 0..130 {
6856 let mut old = state.clone();
6857 old.id = format!("20260101-000000-{i:04x}");
6858 old.candidates.clear();
6859 old.save_under(&f.home).unwrap();
6860 if i == 0 {
6862 let artifacts = f.home.join("runs").join(&old.id).join("artifacts");
6863 std::fs::create_dir_all(&artifacts).unwrap();
6864 std::fs::write(artifacts.join("eviction-review-1.json"), "{}").unwrap();
6865 }
6866 }
6867 for cycle in 1..=2 {
6872 for (pass, expected_next) in [65, 129, 131].into_iter().enumerate() {
6873 let mut report = Report::default();
6874 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6875 assert!(report.failures.is_empty(), "{report:?}");
6876 assert_eq!(report.evicted_copies, usize::from(cycle == 1 && pass == 2));
6877 assert_eq!(review.exists(), cycle == 1 && pass < 2);
6878 let census = source_census(&f.home);
6879 assert_eq!(census["current"]["next"], expected_next);
6880 assert_eq!(census["current"]["completed_at"].is_null(), pass < 2);
6881 assert_eq!(census["current"]["number"], cycle);
6882 assert_eq!(
6883 census["current"]["runs"]["20260101-000000-0000"]["root"]["outcome"],
6884 "error"
6885 );
6886 assert_eq!(
6887 census["current"]["runs"]["20260101-000000-0001"]["root"]["detail"],
6888 "root absent and no eviction records; nothing to reclaim"
6889 );
6890 if pass < 2 {
6891 let next_id = format!("20260101-000000-{expected_next:04x}");
6892 assert_eq!(
6893 census["current"]["runs"][next_id]["root"]["outcome"],
6894 "not-yet-inspected"
6895 );
6896 }
6897 }
6898 }
6899 }
6900
6901 #[tokio::test]
6902 async fn source_eviction_scan_reaches_old_seats_despite_empty_roots_and_timestamp_churn() {
6903 let f = Fx::new().await;
6904 let mut state = inactive_source(&f, "ffff", true).await;
6905 let review = reviewer(&f, &state, &f.base).await;
6906 let mut history = Vec::new();
6909 for i in 0..130 {
6910 let mut old = state.clone();
6911 old.id = format!("20260101-000000-{i:04x}");
6912 old.candidates.clear();
6913 old.save_under(&f.home).unwrap();
6914 std::fs::create_dir_all(old.worktree_root()).unwrap();
6915 history.push(old);
6916 }
6917 state
6918 .save_under_at(
6919 &f.home,
6920 Timestamp::now() + jiff::SignedDuration::from_secs(10),
6921 )
6922 .unwrap();
6923 for pass in 0..3 {
6924 let mut report = Report::default();
6925 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6926 assert!(report.failures.is_empty(), "{report:?}");
6927 let census = source_census(&f.home);
6928 assert_eq!(
6929 census["current"]["next"].as_u64().unwrap(),
6930 ((pass + 1) * 64).min(131)
6931 );
6932 if pass < 2 {
6933 assert!(review.exists());
6934 assert!(census["current"]["completed_at"].is_null());
6935 assert_eq!(
6936 census["current"]["runs"][&state.id]["root"]["outcome"],
6937 "not-yet-inspected"
6938 );
6939 }
6940 for old in history.iter_mut().take(64) {
6943 old.updated_at = Timestamp::now();
6944 old.save_under(&f.home).unwrap();
6945 }
6946 if pass == 1 {
6947 let older = state.updated_at - jiff::SignedDuration::from_secs(100);
6954 for old in history.iter_mut().take(128) {
6955 if f.home.join("runs").join(&old.id).exists() {
6956 old.save_under_at(&f.home, older).unwrap();
6957 }
6958 }
6959 history[127]
6960 .save_under_at(&f.home, older - jiff::SignedDuration::from_secs(100))
6961 .unwrap();
6962 let mut legacy: Vec<_> = history
6963 .iter()
6964 .filter(|r| f.home.join("runs").join(&r.id).exists())
6965 .cloned()
6966 .collect();
6967 legacy.push(state.clone());
6968 legacy.sort_by_key(|r| r.updated_at);
6969 let index = legacy.iter().position(|r| r.id == history[127].id).unwrap();
6970 let len = legacy.len();
6971 legacy.rotate_left((index + 1) % len);
6972 assert!(legacy.iter().take(64).all(|r| r.id != state.id));
6973 }
6974 if pass == 0 {
6975 std::fs::remove_dir_all(f.home.join("runs").join(&history[64].id)).unwrap();
6976 let mut added = state.clone();
6977 added.id = "20260101-000000-fffe".into();
6978 added.candidates.clear();
6979 added.save_under(&f.home).unwrap();
6980 }
6981 }
6982 assert!(!review.exists());
6983 let census = source_census(&f.home);
6984 assert!(!census["current"]["completed_at"].is_null());
6985 assert_eq!(
6986 census["current"]["runs"][&state.id]["seats"]["review-1"]["outcome"],
6987 "evicted"
6988 );
6989 assert_eq!(
6990 census["current"]["runs"][&history[64].id]["root"]["outcome"],
6991 "error"
6992 );
6993 let mut next = Report::default();
6994 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
6995 let wrapped = source_census(&f.home);
6996 assert_eq!(wrapped["current"]["number"], 2);
6997 assert_eq!(wrapped["previous"], census["current"]);
6998 assert!(
6999 wrapped["current"]["runs"]
7000 .get("20260101-000000-fffe")
7001 .is_some()
7002 );
7003 }
7004
7005 #[tokio::test]
7006 async fn source_eviction_scan_resumes_within_run_and_bounds_failed_seats() {
7007 fn fail_one(path: &Path) -> Result<()> {
7008 ensure!(
7009 path.file_name().unwrap() != "review-2",
7010 "injected checkpoint failure"
7011 );
7012 Ok(())
7013 }
7014 let f = Fx::new().await;
7015 let state = inactive_source(&f, "ef20", true).await;
7016 let mut paths = Vec::new();
7017 for i in 1..=8 {
7018 let path = state.worktree_root().join(format!("review-{i}"));
7019 git::worktree_add_detached(&f.repo, &path, &f.base)
7020 .await
7021 .unwrap();
7022 paths.push(path);
7023 }
7024 std::fs::write(paths[2].join("owner-data"), "retain me").unwrap();
7025 let mut first = Report::default();
7026 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut first, fail_one).await;
7027 assert_eq!(first.evicted_copies, 3, "{first:?}");
7028 let census = source_census(&f.home);
7029 let seats = &census["current"]["runs"][&state.id]["seats"];
7030 assert_eq!(census["current"]["after_seat"], "review-5");
7031 assert_eq!(seats["review-2"]["outcome"], "error");
7032 assert_eq!(seats["review-2"]["stage"], "unlink");
7033 assert_eq!(seats["review-3"]["outcome"], "protected");
7034 assert_eq!(seats["review-6"]["outcome"], "not-yet-inspected");
7035 assert!(first.source_unlinked_logical_bytes > 0);
7036 assert!(first.source_evidence_added_logical_bytes > 0);
7037 let mut second = Report::default();
7038 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
7039 assert_eq!(second.evicted_copies, 3, "{second:?}");
7040 assert!(paths[1].exists(), "failed seat waits until next cycle");
7041 assert!(paths[2].exists());
7042 assert!(!source_census(&f.home)["current"]["completed_at"].is_null());
7043 let mut third = Report::default();
7044 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut third).await;
7045 assert_eq!(third.evicted_copies, 1, "{third:?}");
7046 assert!(!paths[1].exists());
7047 assert_eq!(
7048 std::fs::read_to_string(paths[2].join("owner-data")).unwrap(),
7049 "retain me"
7050 );
7051 let shown = seat_inspections(&f.home, &state.id).unwrap().join("\n");
7052 assert!(
7053 shown.contains("cycle=1 complete=true review-2 error stage=unlink"),
7054 "{shown}"
7055 );
7056 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
7057 let destination = f.bay.join("restored-scan-seat");
7058 restore_seat(&f.home, &state.id, &f.base, &destination)
7059 .await
7060 .unwrap();
7061 assert_eq!(git::rev_parse(&destination, "HEAD").await.unwrap(), f.base);
7062 }
7063
7064 #[tokio::test]
7065 async fn source_eviction_scan_continues_after_archive_failure_and_busy_output() {
7066 let f = Fx::new().await;
7067 let failed = inactive_source(&f, "ef30", true).await;
7068 let failed_seat = reviewer(&f, &failed, &f.base).await;
7069 let evidence = f
7070 .home
7071 .join("runs")
7072 .join(&failed.id)
7073 .join("artifacts/source-evidence");
7074 std::fs::create_dir_all(&evidence).unwrap();
7075 std::fs::File::create(evidence.join("reserved-evidence"))
7076 .unwrap()
7077 .set_len(64 * 1024 * 1024)
7078 .unwrap();
7079 let state = inactive_source(&f, "ef31", true).await;
7080 let busy = reviewer(&f, &state, &f.base).await;
7081 let available = state.worktree_root().join("review-2");
7082 git::worktree_add_detached(&f.repo, &available, &f.base)
7083 .await
7084 .unwrap();
7085 let output = OutputLease::take(&busy, true).unwrap();
7086 let mut report = Report::default();
7087 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7088 assert_eq!(report.evicted_copies, 1, "{report:?}");
7089 assert!(failed_seat.exists());
7090 assert!(busy.exists());
7091 assert!(!available.exists());
7092 let census = source_census(&f.home);
7093 assert!(!census["current"]["completed_at"].is_null());
7094 assert_eq!(
7095 census["current"]["runs"][&failed.id]["seats"]["review-1"]["outcome"],
7096 "error"
7097 );
7098 assert_eq!(
7099 census["current"]["runs"][&state.id]["seats"]["review-1"]["outcome"],
7100 "protected"
7101 );
7102 drop(output);
7103 let mut retry = Report::default();
7104 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
7105 assert_eq!(retry.evicted_copies, 1, "{retry:?}");
7106 assert!(!busy.exists());
7107 assert!(failed_seat.exists());
7108 }
7109
7110 #[tokio::test]
7111 async fn source_eviction_scan_checkpoint_failure_stops_before_later_seats() {
7112 fn block_checkpoint(path: &Path) -> Result<()> {
7113 let home = path
7114 .parent()
7115 .unwrap()
7116 .parent()
7117 .unwrap()
7118 .parent()
7119 .unwrap()
7120 .join("home");
7121 let cursor = home.join("source-eviction-cursor");
7122 std::fs::rename(&cursor, home.join("saved-cursor"))?;
7123 std::fs::create_dir(&cursor)?;
7124 Ok(())
7125 }
7126 let f = Fx::new().await;
7127 let state = inactive_source(&f, "ef32", true).await;
7128 let first = reviewer(&f, &state, &f.base).await;
7129 let second = state.worktree_root().join("review-2");
7130 git::worktree_add_detached(&f.repo, &second, &f.base)
7131 .await
7132 .unwrap();
7133 let mut report = Report::default();
7134 copies::evict_with(
7135 &f.home,
7136 &Fx::disk(),
7137 Fx::now(),
7138 &mut report,
7139 block_checkpoint,
7140 )
7141 .await;
7142 assert_eq!(report.evicted_copies, 1, "{report:?}");
7143 assert!(!first.exists());
7144 assert!(second.exists());
7145 assert!(
7146 report
7147 .failures
7148 .iter()
7149 .any(|e| e.contains("persist source scan census"))
7150 );
7151 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "error");
7152 std::fs::remove_dir(f.home.join("source-eviction-cursor")).unwrap();
7153 std::fs::rename(
7154 f.home.join("saved-cursor"),
7155 f.home.join("source-eviction-cursor"),
7156 )
7157 .unwrap();
7158 let mut retry = Report::default();
7159 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
7160 assert_eq!(retry.evicted_copies, 1, "{retry:?}");
7161 assert!(!second.exists());
7162 }
7163
7164 #[tokio::test]
7165 async fn source_eviction_scan_reports_invalid_cursor_and_global_ownership() {
7166 let f = Fx::new().await;
7167 let state = inactive_source(&f, "ef21", true).await;
7168 let review = reviewer(&f, &state, &f.base).await;
7169 let cursor = f.home.join("source-eviction-cursor");
7170 std::fs::write(&cursor, "broken cursor").unwrap();
7171 let mut report = Report::default();
7172 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7173 assert_eq!(std::fs::read_to_string(&cursor).unwrap(), "broken cursor");
7174 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "error");
7175 assert!(review.exists());
7176 std::fs::write(&cursor, "20260101-000000-abcd").unwrap();
7178 let mut disabled = Fx::disk();
7179 disabled.auto_fold = false;
7180 let mut report = Report::default();
7181 copies::evict(&f.home, &disabled, Fx::now(), &mut report).await;
7182 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "disabled");
7183 let record = f.home.join("runs").join(&state.id).join("run.json");
7184 let original = std::fs::read(&record).unwrap();
7185 std::fs::write(&record, "invalid ownership").unwrap();
7186 let mut report = Report::default();
7187 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7188 assert!(
7189 report
7190 .failures
7191 .iter()
7192 .any(|e| e.contains("global run ownership validation"))
7193 );
7194 assert!(review.exists());
7195 std::fs::write(record, original).unwrap();
7196 let mut report = Report::default();
7197 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7198 assert_eq!(report.evicted_copies, 1, "{report:?}");
7199 assert_eq!(source_census(&f.home)["version"], 1);
7200 }
7201
7202 #[tokio::test]
7203 async fn source_eviction_batches_reproducible_seats_without_dropping_task_history() {
7204 let f = Fx::new().await;
7205 let mut runs = Vec::new();
7206 let mut paths = Vec::new();
7207 for tag in ["ec10", "ec11", "ec12", "ec13"] {
7208 let state = inactive_source(&f, tag, true).await;
7209 for name in ["judge-1", "advisor-1"] {
7210 let path = state.worktree_root().join(name);
7211 git::worktree_add_detached(&f.repo, &path, &f.base)
7212 .await
7213 .unwrap();
7214 paths.push(path);
7215 }
7216 runs.push(state);
7217 }
7218 let task = f.task(&runs);
7219 let mut report = Report::default();
7220 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7221 assert_eq!(report.evicted_copies, 5, "{report:?}");
7222 let mut next = Report::default();
7223 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
7224 assert_eq!(next.evicted_copies, 3, "{next:?}");
7225 assert!(paths.iter().all(|p| !p.exists()));
7226 assert!(runs.iter().all(|r| r.candidates[0].worktree.exists()));
7227 assert_eq!(
7228 crate::queue::Queue::at(f.home.join("queue"))
7229 .get(&task.id)
7230 .unwrap()
7231 .runs,
7232 task.runs
7233 );
7234 let limited = Fx::new().await;
7235 let state = inactive_source(&limited, "ec15", true).await;
7236 let copy = reviewer(&limited, &state, &limited.base).await;
7237 let evidence = limited
7238 .home
7239 .join("runs")
7240 .join(&state.id)
7241 .join("artifacts/source-evidence");
7242 std::fs::create_dir_all(&evidence).unwrap();
7243 let retained = evidence.join("reserved-evidence");
7244 std::fs::File::create(&retained)
7245 .unwrap()
7246 .set_len(64 * 1024 * 1024)
7247 .unwrap();
7248 let mut blocked = Report::default();
7249 copies::evict(&limited.home, &Fx::disk(), Fx::now(), &mut blocked).await;
7250 assert_eq!(blocked.evicted_copies, 0, "{blocked:?}");
7251 assert!(copy.exists());
7252 assert_eq!(
7253 std::fs::metadata(&retained).unwrap().len(),
7254 64 * 1024 * 1024
7255 );
7256 assert!(
7257 blocked
7258 .retained
7259 .iter()
7260 .any(|r| r.contains("per-run storage limit")),
7261 "{blocked:?}"
7262 );
7263 }
7264 #[tokio::test]
7265 async fn source_eviction_migrates_historical_records_without_rewriting_them() {
7266 let f = Fx::new().await;
7267 let mut state = inactive_source(&f, "ec14", true).await;
7268 historical_review(&mut state, &f.base);
7269 historical_review(&mut state, &f.base);
7270 state.status = RunStatus::Superseded;
7271 state.save_under(&f.home).unwrap();
7272 let review = reviewer(&f, &state, &f.base).await;
7273 let task = f.task(std::slice::from_ref(&state));
7274 let path = f.home.join("runs").join(&state.id).join("run.json");
7275 let mut raw: serde_json::Value =
7276 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
7277 raw["schema"] = 10.into();
7278 raw.as_object_mut().unwrap().remove("output_activity_at");
7279 raw["resource_wait"] = serde_json::Value::Null;
7282 raw["resource_block"] = serde_json::Value::Null;
7283 raw["driver_claimed"] = serde_json::Value::Null;
7284 raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
7285 let graph = raw["config"]["graph"].as_object_mut().unwrap();
7287 let count = graph.remove("implementers").unwrap();
7288 graph.insert("candidates".into(), count);
7289 let queue = crate::queue::Queue::at(f.home.join("queue"));
7290 let task_path = queue.path_of(&task.id);
7291 let mut task_raw: serde_json::Value =
7292 serde_json::from_slice(&std::fs::read(&task_path).unwrap()).unwrap();
7293 task_raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
7294 let task_historical = serde_json::to_vec_pretty(&task_raw).unwrap();
7295 std::fs::write(&task_path, &task_historical).unwrap();
7296 let historical = serde_json::to_vec_pretty(&raw).unwrap();
7297 std::fs::write(&path, &historical).unwrap();
7298 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
7299 assert_eq!(report.evicted_copies, 1, "{report:?}");
7300 assert!(!review.exists());
7301 assert_eq!(std::fs::read(&path).unwrap(), historical);
7302 assert_eq!(std::fs::read(&task_path).unwrap(), task_historical);
7303 assert!(state.candidates[0].worktree.exists());
7304 assert_eq!(
7305 crate::queue::Queue::at(f.home.join("queue"))
7306 .get(&task.id)
7307 .unwrap()
7308 .runs,
7309 task.runs
7310 );
7311 }
7312 #[tokio::test]
7313 async fn source_eviction_archives_old_only_reflog_oid_after_unrewritten_deletion() {
7314 let f = Fx::new().await;
7315 let state = inactive_source(&f, "ed01", true).await;
7316 let review = reviewer(&f, &state, &f.base).await;
7317 std::fs::write(review.join("old-only"), b"old-only\0binary\xff").unwrap();
7318 git::git(&review, &["add", "."]).await.unwrap();
7319 let tree = git::git(&review, &["write-tree"]).await.unwrap();
7320 let tip = git::git(
7321 &review,
7322 &[
7323 "commit-tree",
7324 &tree,
7325 "-p",
7326 &f.base,
7327 "-m",
7328 "temporary detached source",
7329 ],
7330 )
7331 .await
7332 .unwrap();
7333 git::git(&review, &["checkout", "--detach", &tip])
7334 .await
7335 .unwrap();
7336 git::git(&review, &["checkout", "--detach", &f.base])
7337 .await
7338 .unwrap();
7339 git::git(&review, &["reflog", "delete", "HEAD@{1}"])
7340 .await
7341 .unwrap();
7342 git::git(&review, &["update-ref", "-d", "ORIG_HEAD"])
7343 .await
7344 .unwrap();
7345 let shown = git::git(&review, &["reflog", "show", "--format=%H", "HEAD"])
7346 .await
7347 .unwrap();
7348 assert!(!shown.lines().any(|oid| oid == tip));
7349 let admin = PathBuf::from(
7350 git::git(&review, &["rev-parse", "--absolute-git-dir"])
7351 .await
7352 .unwrap(),
7353 );
7354 let raw = std::fs::read_to_string(admin.join("logs/HEAD")).unwrap();
7355 assert!(raw.lines().any(|line| line.starts_with(&format!("{tip} "))));
7356 fn interrupt(_: &Path) -> Result<()> {
7357 anyhow::bail!("fixture interruption before unlink")
7358 }
7359 let mut pending = Report::default();
7360 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut pending, interrupt).await;
7361 assert!(review.exists());
7362 let ledger = f
7363 .home
7364 .join("runs")
7365 .join(&state.id)
7366 .join("artifacts/eviction-review-1.json");
7367 let complete = std::fs::read(&ledger).unwrap();
7368 let mut incomplete: serde_json::Value = serde_json::from_slice(&complete).unwrap();
7369 incomplete["history"]
7370 .as_object_mut()
7371 .unwrap()
7372 .remove(&tip)
7373 .unwrap();
7374 std::fs::write(&ledger, serde_json::to_vec(&incomplete).unwrap()).unwrap();
7375 let mut blocked = Report::default();
7376 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
7377 assert!(review.exists());
7378 assert!(
7379 blocked
7380 .retained
7381 .iter()
7382 .any(|reason| reason.contains("incomplete reflog checkpoint")),
7383 "{blocked:?}"
7384 );
7385 std::fs::write(&ledger, complete).unwrap();
7386 let mut report = Report::default();
7387 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7388 assert_eq!(report.evicted_copies, 1, "{report:?}");
7389 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
7390 .await
7391 .unwrap();
7392 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
7393 assert!(
7394 !git::git_raw(&f.repo, &["cat-file", "-e", &tip])
7395 .await
7396 .unwrap()
7397 .ok()
7398 );
7399 std::fs::remove_dir_all(&f.repo).unwrap();
7400 let destination = f._temp.path().join("restore-old-only");
7401 restore_seat(&f.home, &state.id, &tip, &destination)
7402 .await
7403 .unwrap();
7404 assert_eq!(
7405 std::fs::read(destination.join("old-only")).unwrap(),
7406 b"old-only\0binary\xff"
7407 );
7408 }
7409
7410 #[tokio::test]
7411 async fn source_eviction_crashed_driver_lease_does_not_block_cleanup_or_retry() {
7412 fn interrupt(_: &Path) -> Result<()> {
7413 anyhow::bail!("fixture interruption before unlink")
7414 }
7415 fn vanish(path: &Path) -> Result<()> {
7416 std::fs::remove_dir_all(path)?;
7417 anyhow::bail!("fixture interruption before unregistration")
7418 }
7419 for stage in ["ordinary", "intent", "missing-registration", "live-driver"] {
7420 let f = Fx::new().await;
7421 let mut state = inactive_source(&f, "ed02", true).await;
7422 state.driver_exited = false;
7423 state.driver_claimed = Some(true);
7424 state.driver_pid = Some(std::process::id());
7425 state.driver_started_at = crate::proc::process_started_at(std::process::id());
7426 state.begin_pass();
7427 state.status = RunStatus::Superseded;
7428 state.save_under(&f.home).unwrap();
7429 let review = reviewer(&f, &state, &f.base).await;
7430 let identity = serde_json::json!({"pid":state.driver_pid,"started_at":state.driver_started_at,
7433 "pass":state.passes.last().unwrap().started_at,"pass_index":state.passes.last().unwrap().index});
7434 std::fs::write(
7435 f.home.join("runs").join(&state.id).join("driver.json"),
7436 serde_json::to_vec(&identity).unwrap(),
7437 )
7438 .unwrap();
7439 let live = if stage == "live-driver" {
7440 Some(crate::driver::maintenance(&f.home, &state.id).unwrap())
7441 } else {
7442 None
7443 };
7444 assert_eq!(
7445 state.liveness_under(&f.home, false),
7446 if live.is_some() {
7447 Liveness::Live
7448 } else {
7449 Liveness::Dead
7450 }
7451 );
7452 let mut report = Report::default();
7453 if stage == "intent" || stage == "missing-registration" {
7454 copies::evict_with(
7455 &f.home,
7456 &Fx::disk(),
7457 Fx::now(),
7458 &mut report,
7459 if stage == "intent" { interrupt } else { vanish },
7460 )
7461 .await;
7462 assert_eq!(report.evicted_copies, 0, "{stage}: {report:?}");
7463 let mut retry = Report::default();
7464 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
7465 assert!(!review.exists(), "{stage}: {retry:?}");
7466 let ledger = f
7467 .home
7468 .join("runs")
7469 .join(&state.id)
7470 .join("artifacts/eviction-review-1.json");
7471 let receipt: serde_json::Value =
7472 serde_json::from_slice(&std::fs::read(ledger).unwrap()).unwrap();
7473 assert_eq!(receipt["phase"], "evicted", "{stage}: {retry:?}");
7474 } else {
7475 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7476 assert_eq!(
7477 report.evicted_copies,
7478 usize::from(live.is_none()),
7479 "{stage}: {report:?}"
7480 );
7481 assert_eq!(review.exists(), live.is_some());
7482 }
7483 drop(live);
7484 assert!(
7485 !RunState::load_under(&state.id, &f.home)
7486 .unwrap()
7487 .driver_exited
7488 );
7489 }
7490 }
7491}