1mod copies;
4pub mod reconcile;
5
6pub(crate) fn has_archived_seats(home: &Path, state: &RunState) -> Result<bool> {
7 let artifacts = home.join("runs").join(&state.id).join("artifacts");
8 if !artifacts.try_exists()? {
9 return Ok(false);
10 }
11 ensure!(
12 plain_dir(&artifacts) && artifacts.canonicalize()? == artifacts,
13 "noncanonical artifacts"
14 );
15 for (i, entry) in std::fs::read_dir(artifacts)?.enumerate() {
16 ensure!(i < 4096, "eviction record inspection limit exceeded");
17 let name = entry?.file_name();
18 let name = name.to_string_lossy();
19 if name.starts_with("eviction-") && name.ends_with(".json") {
20 return Ok(true);
21 }
22 }
23 Ok(false)
24}
25
26pub(crate) async fn restore_execution_seats(home: &Path, state: &RunState) -> Result<()> {
29 copies::restore_execution_seats(home, state).await
30}
31
32pub async fn source_diff(evidence: &Path) -> Result<Vec<u8>> {
34 crate::fold_evidence::validate_snapshot(evidence).await?;
35 Ok(std::fs::read(evidence.join("change.patch"))?)
36}
37
38pub async fn restore_source(evidence: &Path, destination: &Path) -> Result<()> {
40 crate::fold_evidence::restore_snapshot(evidence, destination).await
41}
42
43pub fn seat_inspections(home: &Path, id: &str) -> Result<Vec<String>> {
45 ensure!(crate::run::is_run_id(id), "invalid run id");
46 copies::seat_inspections(home, id)
47}
48
49pub async fn seat_evidence(home: &Path, id: &str) -> Result<Vec<String>> {
51 copies::seat_evidence(home, id).await
52}
53
54pub async fn seat_heads(
56 home: &Path,
57 id: &str,
58) -> Result<std::collections::BTreeMap<String, String>> {
59 copies::seat_heads(home, id).await
60}
61
62pub async fn seat_diff(home: &Path, id: &str, head: &str) -> Result<Vec<u8>> {
64 copies::seat_diff(home, id, head).await
65}
66
67pub async fn restore_seat(home: &Path, id: &str, head: &str, destination: &Path) -> Result<()> {
69 copies::restore_seat(home, id, head, destination).await
70}
71
72use crate::{
73 git,
74 proc::Quiet as _,
75 run::{Liveness, RunState},
76};
77use anyhow::{Context, Result, ensure};
78use jiff::Timestamp;
79use serde::{Deserialize, Serialize};
80use std::{
81 fs::{File, OpenOptions},
82 path::{Path, PathBuf},
83};
84
85#[derive(Debug)]
89pub(crate) struct OutputLease {
90 _file: std::sync::Arc<OutputFile>,
91 _activity: bool,
92}
93
94#[derive(Debug)]
95struct OutputFile(File);
96
97impl std::ops::Deref for OutputFile {
98 type Target = File;
99 fn deref(&self) -> &File {
100 &self.0
101 }
102}
103
104#[cfg(unix)]
105impl Drop for OutputFile {
106 fn drop(&mut self) {
107 use std::os::fd::AsRawFd;
108 unsafe {
110 libc::flock(self.0.as_raw_fd(), libc::LOCK_UN);
111 }
112 }
113}
114
115impl OutputLease {
116 #[cfg(unix)]
117 pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
118 use std::os::fd::AsRawFd;
119 self._file.as_raw_fd()
120 }
121
122 pub(crate) fn pre_clean(dir: &Path) -> Result<Self> {
124 let file = OpenOptions::new()
125 .create(true)
126 .truncate(false)
127 .read(true)
128 .write(true)
129 .open(dir.join("magi-pre-clean.lock"))?;
130 crate::pr_head::try_lock_file(&file)
131 .context("pre-clean already running; retry after descendants terminate")?;
132 Ok(Self {
133 _file: std::sync::Arc::new(OutputFile(file)),
134 _activity: false,
135 })
136 }
137
138 pub(crate) fn duplicate(&self) -> Self {
139 Self {
140 _file: std::sync::Arc::clone(&self._file),
141 _activity: false,
142 }
143 }
144
145 pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
146 if !worktree.join(".git").is_file() {
147 return Ok(None);
148 }
149 Self::take(worktree, false).map(Some)
150 }
151 pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
152 use std::hash::{Hash, Hasher};
153 let mut hash = std::collections::hash_map::DefaultHasher::new();
154 let mut normalized = cache.to_owned();
155 let mut missing = Vec::new();
156 while !normalized.exists() {
157 let Some(name) = normalized.file_name() else {
158 break;
159 };
160 missing.push(name.to_owned());
161 if !normalized.pop() {
162 break;
163 }
164 }
165 normalized = normalized.canonicalize().unwrap_or(normalized);
166 for part in missing.into_iter().rev() {
167 normalized.push(part);
168 }
169 normalized.hash(&mut hash);
170 let dir = home.join("cache-leases");
171 std::fs::create_dir_all(&dir)?;
172 Self::lock(
173 &dir.join(format!("outputs-{:x}.lock", hash.finish())),
174 exclusive,
175 )
176 }
177 #[cfg(unix)]
181 pub(crate) fn admit(home: &Path, paths: &[PathBuf], min_free: u64) -> Result<Vec<Self>> {
182 Self::admit_with(home, paths, min_free, crate::disk::free_bytes)
183 }
184
185 #[cfg(unix)]
186 pub(crate) fn admit_with(
187 home: &Path,
188 paths: &[PathBuf],
189 min_free: u64,
190 free_bytes: impl Fn(&Path) -> Result<u64>,
191 ) -> Result<Vec<Self>> {
192 use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
193 if min_free == 0 {
194 return Ok(Vec::new());
195 }
196 const RESERVE: u64 = 1024 * 1024 * 1024;
197 std::fs::create_dir_all(home)?;
198 let dir = home.canonicalize()?.join("disk-admission");
199 std::fs::create_dir_all(&dir)?;
200 ensure!(
201 plain_dir(&dir) && dir.canonicalize()? == dir,
202 "disk admission directory is not canonical"
203 );
204 let mut devices = std::collections::BTreeSet::new();
205 let mut reservations = Vec::new();
206 for path in paths {
207 let probe = path
208 .ancestors()
209 .find(|p| p.exists())
210 .context("disk admission path missing")?;
211 let device = std::fs::metadata(probe)?.dev();
212 if !devices.insert(device) {
213 continue;
214 }
215 let open = |path: &Path| -> Result<File> {
216 Ok(OpenOptions::new()
217 .read(true)
218 .write(true)
219 .create(true)
220 .truncate(false)
221 .custom_flags(libc::O_NOFOLLOW)
222 .open(path)?)
223 };
224 let admission = Self {
228 _file: std::sync::Arc::new(OutputFile(open(&dir.join(format!("{device}.lock")))?)),
229 _activity: false,
230 };
231 let started = std::time::Instant::now();
232 loop {
233 match crate::pr_head::try_lock_file(&admission._file) {
234 Ok(()) => break,
235 Err(error)
236 if matches!(
237 error.kind(),
238 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
239 ) && started.elapsed() < std::time::Duration::from_secs(1) =>
240 {
241 std::thread::sleep(std::time::Duration::from_millis(5));
242 }
243 Err(error) => {
244 return Err(error)
245 .context("disk admission busy or unavailable; retry without dispatch");
246 }
247 }
248 }
249 let mut active = 0u64;
250 let mut available = None;
251 for slot in 0..1024 {
252 let file = dir.join(format!("{device}-{slot}.reserve"));
253 if !file.exists() {
254 if available.is_none() {
255 available = Some(file);
256 }
257 continue;
258 }
259 let file = open(&file)?;
260 match crate::pr_head::try_lock_file(&file) {
261 Ok(()) if available.is_none() => {
262 available = Some(dir.join(format!("{device}-{slot}.reserve")))
263 }
264 Ok(()) => {}
265 Err(_) => active = active.saturating_add(RESERVE),
266 }
267 }
268 let free = free_bytes(probe)?;
269 let required = min_free.saturating_add(active).saturating_add(RESERVE);
270 ensure!(
271 free >= required,
272 "disk admission blocked: {free} bytes free; floor {min_free}, active writer reservations {active}, next writer reservation {RESERVE} bytes (estimates, not quotas)"
273 );
274 let file = open(&available.context("disk admission reservation slots exhausted")?)?;
275 crate::pr_head::try_lock_file(&file)?;
276 reservations.push(Self {
277 _file: std::sync::Arc::new(OutputFile(file)),
278 _activity: false,
279 });
280 }
281 Ok(reservations)
282 }
283
284 #[cfg(not(unix))]
285 pub(crate) fn admit(_home: &Path, _paths: &[PathBuf], _min_free: u64) -> Result<Vec<Self>> {
286 Ok(Vec::new())
288 }
289
290 pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
291 let out = std::process::Command::new("git")
292 .args(["rev-parse", "--absolute-git-dir"])
293 .current_dir(worktree)
294 .quiet()
295 .output()?;
296 ensure!(
297 out.status.success(),
298 "cannot identify output lease directory"
299 );
300 let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
301 Self::lock(&dir.join("magi-output.lock"), exclusive)
302 }
303 fn lock(path: &Path, exclusive: bool) -> Result<Self> {
304 let file = OpenOptions::new()
305 .create(true)
306 .truncate(false)
307 .read(true)
308 .write(true)
309 .open(path)
310 .with_context(|| format!("output lease: open {}", path.display()))?;
311 #[cfg(unix)]
312 {
313 use std::os::fd::AsRawFd;
314 let mode = if exclusive {
315 libc::LOCK_EX
316 } else {
317 libc::LOCK_SH
318 };
319 ensure!(
321 unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
322 "output lease: outputs are in use or being reclaimed: {}",
323 path.display()
324 );
325 }
326 #[cfg(not(unix))]
327 let _ = exclusive;
328 if !exclusive {
329 file.set_len(1)?;
331 file.set_modified(std::time::SystemTime::now())
332 .with_context(|| format!("record output activity {}", path.display()))?;
333 }
334 Ok(Self {
335 _file: std::sync::Arc::new(OutputFile(file)),
336 _activity: !exclusive,
337 })
338 }
339}
340#[cfg(unix)]
341impl Drop for OutputLease {
342 fn drop(&mut self) {
343 if self._activity
344 && let Err(error) = self._file.set_modified(std::time::SystemTime::now())
345 {
346 let _ = self._file.set_len(2);
349 tracing::warn!("could not record output lease release activity: {error}");
350 }
351 }
352}
353
354#[derive(Default, Debug, Serialize, Deserialize)]
355#[serde(default)]
356pub(crate) struct Report {
357 pub checked_at: Option<Timestamp>,
358 pub limit_bytes: u64,
359 pub reclaim_budget_bytes: u64,
360 pub measured_bytes: u64,
362 pub freed_bytes: u64,
364 pub measured_allocated_file_bytes: Option<u64>,
365 pub proven_reclaimable_bytes: u64,
367 pub proven_reclaimable_allocated_bytes: Option<u64>,
370 pub protected_or_unproven_bytes: u64,
371 pub estimated_released_allocated_bytes: Option<u64>,
374 pub observed_home_free_change_bytes: Option<i64>,
377 pub retired: usize,
378 pub evicted_copies: usize,
379 pub source_scan: Option<serde_json::Value>,
380 pub removed_empty_roots: usize,
381 pub source_unlinked_logical_bytes: u64,
382 pub source_unlinked_allocated_bytes: Option<u64>,
383 pub source_evidence_added_logical_bytes: u64,
384 pub source_evidence_added_allocated_bytes: Option<u64>,
385 pub retained: Vec<String>,
386 pub failures: Vec<String>,
387}
388
389#[cfg(unix)]
390fn allocated_inventory(
391 root: &Path,
392 seen: &mut std::collections::BTreeSet<(u64, u64)>,
393) -> Result<u64> {
394 use std::os::unix::fs::MetadataExt;
395 ensure!(
396 plain_dir(root) && root.canonicalize()? == root,
397 "allocated inventory root is not canonical"
398 );
399 let mut dirs = vec![root.to_owned()];
400 let mut bytes = 0u64;
401 while let Some(dir) = dirs.pop() {
402 for entry in std::fs::read_dir(dir)? {
403 let path = entry?.path();
404 let metadata = std::fs::symlink_metadata(&path)?;
405 if metadata.is_dir() {
406 dirs.push(path);
407 } else if metadata.is_file() && seen.insert((metadata.dev(), metadata.ino())) {
408 bytes = bytes.saturating_add(metadata.blocks().saturating_mul(512));
409 }
410 }
411 }
412 Ok(bytes)
413}
414
415#[derive(Default)]
416struct ReleasedAllocation {
417 #[cfg(unix)]
418 files: std::collections::BTreeMap<PathBuf, (u64, u64, u64, u64)>,
419 #[cfg(unix)]
420 removed: std::collections::BTreeMap<(u64, u64), u64>,
421}
422
423impl ReleasedAllocation {
424 fn of(paths: &[PathBuf]) -> Result<Self> {
425 #[cfg(unix)]
426 {
427 use std::os::unix::fs::MetadataExt;
428 let mut files = std::collections::BTreeMap::new();
429 for path in paths {
430 let m = std::fs::symlink_metadata(path)?;
431 ensure!(m.is_file(), "allocation candidate is not regular");
432 files.insert(
433 path.clone(),
434 (m.dev(), m.ino(), m.nlink(), m.blocks().saturating_mul(512)),
435 );
436 }
437 Ok(Self {
438 files,
439 removed: Default::default(),
440 })
441 }
442 #[cfg(not(unix))]
443 {
444 let _ = paths;
445 Ok(Self::default())
446 }
447 }
448 fn reclaimable_allocated_bytes(&self) -> Option<u64> {
449 #[cfg(unix)]
450 {
451 let mut inodes = std::collections::BTreeMap::new();
452 for &(device, inode, links, blocks) in self.files.values() {
453 let entry = inodes
454 .entry((device, inode))
455 .or_insert((0u64, links, blocks));
456 entry.0 += 1;
457 }
458 Some(
459 inodes
460 .values()
461 .filter(|(count, links, _)| count == links)
462 .fold(0u64, |sum, (_, _, blocks)| sum.saturating_add(*blocks)),
463 )
464 }
465 #[cfg(not(unix))]
466 {
467 None
468 }
469 }
470
471 fn unlinked(&mut self, path: &Path) -> Option<u64> {
472 #[cfg(unix)]
473 {
474 let (device, inode, links, blocks) = self.files[path];
475 let removed = self.removed.entry((device, inode)).or_default();
476 *removed += 1;
477 Some(if *removed == links { blocks } else { 0 })
478 }
479 #[cfg(not(unix))]
480 {
481 let _ = path;
482 None
483 }
484 }
485}
486
487fn plain_dir(path: &Path) -> bool {
488 std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
489}
490
491fn run_paths_missing(state: &RunState) -> Result<bool> {
494 for path in [state.worktree_root(), state.repo.clone()] {
495 match std::fs::metadata(&path) {
496 Ok(_) => {}
497 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
498 Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
499 }
500 }
501 Ok(false)
502}
503
504pub(crate) async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
506 let root = state.worktree_root();
507 if !plain_dir(&root) {
508 return Ok(Vec::new());
509 }
510 let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
511 registered_worktrees(state, &list)
512}
513
514fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
515 let root = state.worktree_root();
516 if !plain_dir(&root) {
517 return Ok(Vec::new());
518 }
519 let root = root.canonicalize()?;
520 let mut paths = Vec::new();
521 for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
522 let path = PathBuf::from(path);
523 #[cfg(windows)]
524 let path = if plain_dir(&path) {
525 git::canonical_output_path(path.to_str().context("non UTF-8 registration")?)?
526 } else {
527 continue;
528 };
529 if plain_dir(&path)
530 && path.parent() == Some(root.as_path())
531 && path.canonicalize()? == path
532 && path.join(".git").is_file()
533 {
534 paths.push(path);
535 }
536 }
537 Ok(paths)
538}
539
540async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
543 validate_cargo_target(worktree, false).await
544}
545
546async fn validate_cargo_target(worktree: &Path, allow_missing_tag: bool) -> Result<PathBuf> {
547 let target = worktree.join("target");
548 ensure!(plain_dir(&target), "target absent or not a plain directory");
549 ensure!(target.canonicalize()? == target, "target path changed");
550 ensure!(
551 git::git(worktree, &["ls-files", "--", "target"])
552 .await?
553 .is_empty(),
554 "tracked target contents"
555 );
556 ensure!(
557 git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
558 .await?
559 .ok(),
560 "target is not ignored"
561 );
562 cache_tag(&target, allow_missing_tag)?;
563 let path = target.join(".rustc_info.json");
564 plain_file(&path, "rustc identity")?;
565 let info: serde_json::Value = serde_json::from_slice(
566 &std::fs::read(&path).with_context(|| format!("read rustc identity {}", path.display()))?,
567 )
568 .with_context(|| format!("parse rustc identity {}", path.display()))?;
569 ensure!(
570 info.get("rustc_fingerprint").is_some(),
571 "unknown Cargo identity {}",
572 path.display()
573 );
574 if allow_missing_tag && !target.join("CACHEDIR.TAG").exists() {
575 ensure!(
576 info.get("rustc_fingerprint")
577 .is_some_and(serde_json::Value::is_u64)
578 && info
579 .get("outputs")
580 .and_then(serde_json::Value::as_object)
581 .is_some_and(|outputs| outputs.values().any(|output| {
582 output.get("success").and_then(serde_json::Value::as_bool) == Some(true)
583 && output
584 .get("stdout")
585 .and_then(serde_json::Value::as_str)
586 .is_some_and(|stdout| stdout.starts_with("rustc "))
587 && output
588 .get("stderr")
589 .is_some_and(serde_json::Value::is_string)
590 }))
591 && info
592 .get("successes")
593 .is_some_and(serde_json::Value::is_object),
594 "missing-tag rustc identity structure invalid: {}",
595 path.display()
596 );
597 let mut profiles = 0;
598 for profile in ["debug", "release"] {
599 let dir = target.join(profile);
600 if !dir.exists() {
601 continue;
602 }
603 for subdir in [dir.clone(), dir.join("deps"), dir.join(".fingerprint")] {
604 ensure!(
605 plain_dir(&subdir) && subdir.canonicalize()? == subdir,
606 "missing-tag profile proof: {} is not a plain directory",
607 subdir.display()
608 );
609 }
610 plain_file(&dir.join(".cargo-lock"), "missing-tag Cargo lock")?;
611 profiles += 1;
612 }
613 ensure!(
614 profiles > 0,
615 "missing-tag proof: no Cargo profile in {}",
616 target.display()
617 );
618 }
619 for name in ["debug", "release"] {
620 let path = target.join(name);
621 match std::fs::symlink_metadata(&path) {
622 Ok(metadata) => ensure!(
623 metadata.is_dir() && path.canonicalize()? == path,
624 "profile validation: not a plain directory: {}",
625 path.display()
626 ),
627 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
628 Err(error) => {
629 return Err(error)
630 .with_context(|| format!("profile validation: inspect {}", path.display()));
631 }
632 }
633 }
634 Ok(target)
635}
636
637fn plain_file(path: &Path, stage: &str) -> Result<()> {
638 let meta = std::fs::symlink_metadata(path)
639 .with_context(|| format!("{stage}: inspect {}", path.display()))?;
640 ensure!(
641 meta.is_file() && path.canonicalize()? == path,
642 "{stage}: not a plain file: {}",
643 path.display()
644 );
645 Ok(())
646}
647
648fn cache_tag(target: &Path, allow_missing: bool) -> Result<()> {
649 let path = target.join("CACHEDIR.TAG");
650 match std::fs::symlink_metadata(&path) {
651 Err(error) if allow_missing && error.kind() == std::io::ErrorKind::NotFound => {
652 return Ok(());
653 }
654 result => {
655 result.with_context(|| format!("cache tag: inspect {}", path.display()))?;
656 }
657 }
658 plain_file(&path, "cache tag")?;
659 let tag = std::fs::read_to_string(&path)
660 .with_context(|| format!("cache tag: read {}", path.display()))?;
661 ensure!(
662 tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55"),
663 "unknown Cargo cache tag: {}",
664 path.display()
665 );
666 Ok(())
667}
668
669fn output_age(target: &Path, now: Timestamp, grace: u64) -> Result<()> {
673 let mut dirs = vec![target.to_owned()];
674 let mut entries = 0;
675 while let Some(dir) = dirs.pop() {
676 for entry in std::fs::read_dir(&dir)
677 .with_context(|| format!("output activity: list {}", dir.display()))?
678 {
679 let path = entry?.path();
680 entries += 1;
681 ensure!(
682 entries <= 100_000,
683 "output activity inventory too large: {}",
684 target.display()
685 );
686 let metadata = std::fs::symlink_metadata(&path)
687 .with_context(|| format!("output activity: inspect {}", path.display()))?;
688 if metadata.is_dir() {
689 if dir != target
691 || matches!(
692 path.file_name().and_then(|s| s.to_str()),
693 Some("debug" | "release")
694 )
695 {
696 dirs.push(path);
697 }
698 } else if metadata.is_file()
699 && (dir != target || path.file_name().is_some_and(|n| n == ".rustc_info.json"))
700 {
701 let at =
702 Timestamp::try_from(metadata.modified().with_context(|| {
703 format!("output activity: modified {}", path.display())
704 })?)?;
705 ensure!(
706 crate::clean::due(now, at, grace),
707 "output activity within grace period: {} modified {at} (now {now}, grace {grace}s)",
708 path.display()
709 );
710 }
711 }
712 }
713 Ok(())
714}
715
716pub(crate) fn artifact_signature(path: &Path) -> Result<bool> {
717 use std::io::Read as _;
718 let mut header = [0; 8];
719 let n = File::open(path)?.read(&mut header)?;
720 let header = &header[..n];
721 Ok(header.starts_with(b"\x7fELF")
722 || header.starts_with(b"!<arch>\n")
723 || header.starts_with(b"rust\0\0\0")
724 || header.starts_with(b"MZ\x90\0")
725 || [
726 [0xfe, 0xed, 0xfa, 0xce],
727 [0xce, 0xfa, 0xed, 0xfe],
728 [0xfe, 0xed, 0xfa, 0xcf],
729 [0xcf, 0xfa, 0xed, 0xfe],
730 [0xca, 0xfe, 0xba, 0xbe],
731 [0xca, 0xfe, 0xba, 0xbf],
732 ]
733 .iter()
734 .any(|magic| header.starts_with(magic)))
735}
736
737fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
741 let mut outputs = std::collections::BTreeSet::new();
742 for profile in ["debug", "release"] {
743 for dir in [target.join(profile), target.join(profile).join("deps")] {
744 if !plain_dir(&dir) || dir.canonicalize()? != dir {
745 continue;
746 }
747 for entry in std::fs::read_dir(&dir)? {
748 let path = entry?.path();
749 if path.extension().is_none_or(|ext| ext != "d")
750 || !std::fs::symlink_metadata(&path)?.is_file()
751 || std::fs::metadata(&path)?.len() > 1024 * 1024
752 || path.canonicalize()? != path
753 {
754 continue;
755 }
756 let body = std::fs::read_to_string(&path)?;
757 for line in body.lines() {
758 let Some((names, dependencies)) = line.split_once(": ") else {
759 continue;
760 };
761 if dependencies.trim().is_empty() {
762 continue;
763 }
764 let mut words = Vec::new();
767 let mut word = String::new();
768 let mut escaped = false;
769 for c in names.chars() {
770 if escaped {
771 word.push(c);
772 escaped = false;
773 } else if c == '\\' {
774 escaped = true;
775 } else if c.is_whitespace() {
776 if !word.is_empty() {
777 words.push(std::mem::take(&mut word));
778 }
779 } else {
780 word.push(c);
781 }
782 }
783 if escaped || word.contains(['$', '%']) {
784 continue;
785 }
786 if !word.is_empty() {
787 words.push(word);
788 }
789 for name in words {
790 let output = PathBuf::from(name);
791 let stem = path.file_stem().unwrap_or_default().to_string_lossy();
792 let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
793 let artifact_name =
794 output_stem == stem || output_stem == format!("lib{stem}");
795 let artifact_kind = output.extension().is_none_or(|ext| {
796 matches!(
797 ext.to_str(),
798 Some(
799 "rlib"
800 | "rmeta"
801 | "so"
802 | "dylib"
803 | "a"
804 | "lib"
805 | "dll"
806 | "exe"
807 | "pdb"
808 )
809 )
810 });
811 if artifact_name
812 && artifact_kind
813 && output.parent() == Some(dir.as_path())
814 && output.extension().is_none_or(|ext| ext != "d")
815 && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
816 && output.canonicalize().is_ok_and(|p| p == output)
817 && artifact_signature(&output)?
818 {
819 outputs.insert(output);
820 }
821 }
822 }
823 }
824 }
825 }
826 Ok(outputs.into_iter().collect())
827}
828
829#[derive(Serialize, Deserialize)]
830struct CargoResidue {
831 target: PathBuf,
832 files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
833}
834
835fn residue_files(
838 target: &Path,
839 outputs: &[PathBuf],
840) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
841 let mut files = std::collections::BTreeMap::new();
842 let mut dirs = vec![target.to_owned()];
843 let mut bytes = 0usize;
844 let mut entries = 0usize;
845 while let Some(dir) = dirs.pop() {
846 for entry in std::fs::read_dir(dir)? {
847 let path = entry?.path();
848 entries += 1;
849 ensure!(entries <= 4096, "residue inventory too large");
850 let metadata = std::fs::symlink_metadata(&path)?;
851 ensure!(
852 !metadata.file_type().is_symlink(),
853 "symlink in Cargo residue"
854 );
855 if metadata.is_dir() {
856 dirs.push(path);
857 } else {
858 ensure!(metadata.is_file(), "non-file in Cargo residue");
859 if outputs.contains(&path) {
860 continue;
861 }
862 ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
863 let body = std::fs::read(&path)?;
864 bytes = bytes.saturating_add(body.len());
865 ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
866 files.insert(path.strip_prefix(target)?.to_owned(), body);
867 }
868 }
869 }
870 Ok(files)
871}
872
873fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
874 Ok(home
875 .join("runs")
876 .join(id)
877 .join("build-residue")
878 .join(worktree.file_name().context("missing worktree name")?))
879}
880
881fn residue_root(target: &Path) -> Result<()> {
884 for entry in std::fs::read_dir(target)? {
885 let name = entry?.file_name();
886 ensure!(
887 matches!(
888 name.to_str(),
889 Some("debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json")
890 ),
891 "unrecognized Cargo residue root entry"
892 );
893 }
894 Ok(())
895}
896
897fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
898 cache_tag(target, false)?;
899 residue_root(target)?;
900 let files = residue_files(target, outputs)?;
901 for (relative, body) in &files {
902 let path = target.join(relative);
903 let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
904 let lock = matches!(
905 relative.to_str(),
906 Some("debug/.cargo-lock" | "release/.cargo-lock")
907 ) && body.is_empty();
908 let dep_info = path.extension().is_some_and(|ext| ext == "d")
909 && outputs.iter().any(|output| {
910 output.parent() == path.parent()
911 && (output.file_stem() == path.file_stem()
912 || output.file_stem().is_some_and(|stem| {
913 stem.to_string_lossy()
914 == format!(
915 "lib{}",
916 path.file_stem().unwrap_or_default().to_string_lossy()
917 )
918 }))
919 });
920 ensure!(identity || lock || dep_info, "unproven Cargo residue");
921 }
922 let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
923 std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
924 let tmp = proof.with_extension("tmp");
925 std::fs::write(
926 &tmp,
927 serde_json::to_vec(&CargoResidue {
928 target: target.to_owned(),
929 files,
930 })?,
931 )?;
932 std::fs::rename(tmp, proof)?;
933 Ok(())
934}
935
936pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
937 let target = cargo_target(worktree).await?;
938 residue_root(&target)?;
939 let locks = cargo_locks(&target)?;
940 let proof: CargoResidue =
941 serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
942 ensure!(
943 proof.target == target && proof.files == residue_files(&target, &[])?,
944 "Cargo residue changed or contains unproven files"
945 );
946 Ok(locks)
947}
948
949pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
951 let mut held = Vec::new();
952 for (profile, name) in ["debug", "release"].into_iter().flat_map(|profile| {
953 [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"].map(|name| (profile, name))
954 }) {
955 let lock = target.join(profile).join(name);
956 match std::fs::symlink_metadata(&lock) {
957 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
958 result => {
959 result.with_context(|| format!("Cargo lock: inspect {}", lock.display()))?;
960 }
961 }
962 {
963 plain_file(&lock, "Cargo lock")?;
964 let file = OpenOptions::new()
965 .read(true)
966 .write(true)
967 .open(&lock)
968 .with_context(|| format!("Cargo lock: open {}", lock.display()))?;
969 crate::pr_head::try_lock_file(&file)
970 .with_context(|| format!("Cargo build is active: {}", lock.display()))?;
971 held.push(file);
972 }
973 }
974 Ok(held)
975}
976
977pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
980 let home = home.to_owned();
981 let disk = disk.clone();
982 let runtime = tokio::runtime::Handle::current();
983 tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
984 .await
985 .unwrap_or_else(|e| Report {
986 failures: vec![format!("retention worker failed: {e}")],
987 ..Report::default()
988 })
989}
990
991#[derive(Default, Serialize, Deserialize)]
992struct Measurement {
993 at: Option<Timestamp>,
994 bytes: u64,
995}
996
997async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
998 let mut report = Report {
999 checked_at: Some(now),
1000 limit_bytes: disk.worktree_limit_bytes,
1001 reclaim_budget_bytes: disk.worktree_reclaim_bytes,
1002 ..Report::default()
1003 };
1004 if !cfg!(unix) {
1005 report
1006 .failures
1007 .push("Automatic output reclamation requires Unix process containment".into());
1008 return report;
1009 }
1010 let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
1011 return report;
1012 };
1013 let mut targets = Vec::new();
1014 let mut under_pressure = false;
1015 let mut registrations = std::collections::HashMap::new();
1016 let measurements_path = home.join("output-measurements.json");
1017 let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
1018 std::fs::read(&measurements_path)
1019 .ok()
1020 .and_then(|b| serde_json::from_slice(&b).ok())
1021 .unwrap_or_default();
1022 let mut observed = std::collections::BTreeSet::new();
1023 let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
1024 .into_iter()
1025 .flatten()
1026 .flatten()
1027 .map(|e| e.file_name().to_string_lossy().into_owned())
1028 .collect();
1029 ids.sort();
1030 for id in ids {
1031 if !crate::run::is_run_id(&id) {
1032 continue;
1033 }
1034 let state = match RunState::load_under(&id, home) {
1035 Ok(state) => state,
1036 Err(e) => {
1037 report
1038 .retained
1039 .push(format!("{id}: unreadable run ownership: {e:#}"));
1040 continue;
1041 }
1042 };
1043 if disk.worktree_limit_bytes == 0 {
1044 continue;
1045 }
1046 match run_paths_missing(&state) {
1047 Ok(true) => continue,
1048 Ok(false) => {}
1049 Err(e) => {
1050 report.failures.push(format!("{id}: {e:#}"));
1051 continue;
1052 }
1053 }
1054 let list = match registrations.entry(state.repo.clone()) {
1055 std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
1056 std::collections::hash_map::Entry::Vacant(entry) => {
1057 match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
1058 Ok(list) => entry.insert(list),
1059 Err(e) => {
1060 if matches!(run_paths_missing(&state), Ok(true)) {
1063 continue;
1064 }
1065 report.failures.push(format!("{id}: {e:#}"));
1066 continue;
1067 }
1068 }
1069 }
1070 };
1071 let paths = match registered_worktrees(&state, list) {
1072 Ok(paths) => paths,
1073 Err(e) => {
1074 report.failures.push(format!("{id}: {e:#}"));
1075 continue;
1076 }
1077 };
1078 for path in paths {
1079 if !path.join("target").exists() {
1080 continue;
1081 }
1082 let target = path.join("target");
1083 observed.insert(target.clone());
1084 let pressure =
1085 crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
1086 under_pressure |= pressure;
1087 let measurement = measurements.entry(target).or_default();
1088 if pressure
1089 || measurement
1090 .at
1091 .is_none_or(|at| crate::clean::due(now, at, 300))
1092 {
1093 measurement.bytes = crate::disk::dir_size(&path.join("target"));
1094 measurement.at = Some(now);
1095 }
1096 let size = measurement.bytes;
1097 report.measured_bytes = report.measured_bytes.saturating_add(size);
1098 targets.push((state.output_activity(), id.clone(), path, size));
1099 }
1100 }
1101 if under_pressure {
1102 targets.sort_by_key(|t| (std::cmp::Reverse(t.3), t.0));
1105 } else {
1106 targets.sort_by_key(|t| t.0);
1107 }
1108 let free_before = crate::disk::free_bytes(home).ok();
1109 #[cfg(unix)]
1110 {
1111 let mut seen = std::collections::BTreeSet::new();
1112 report.measured_allocated_file_bytes = Some(0);
1113 report.estimated_released_allocated_bytes = Some(0);
1114 report.proven_reclaimable_allocated_bytes = Some(0);
1115 for (_, _, path, _) in &targets {
1116 report.measured_allocated_file_bytes = report
1117 .measured_allocated_file_bytes
1118 .zip(allocated_inventory(&path.join("target"), &mut seen).ok())
1119 .map(|(a, b)| a.saturating_add(b));
1120 }
1121 }
1122 let mut remaining = report.measured_bytes;
1123 let pressure_paths: Vec<_> = targets.iter().map(|t| t.2.clone()).collect();
1124 let mut deletion_started = false;
1125 let mut oversized_started = false;
1126 for (updated, id, path, _) in targets {
1127 if disk.worktree_limit_bytes == 0
1128 || remaining <= disk.worktree_limit_bytes
1129 && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
1130 {
1131 report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
1132 continue;
1133 }
1134 if oversized_started || report.freed_bytes >= disk.worktree_reclaim_bytes {
1135 report
1136 .retained
1137 .push(format!("{id}: per-pass byte budget spent"));
1138 continue;
1139 }
1140 let result = async {
1141 ensure!(
1142 crate::clean::due(now, updated, disk.fold_grace_secs),
1143 "execution activity within grace period: {updated} (now {now}, grace {}s)", disk.fold_grace_secs
1144 );
1145 let _owner = crate::pr_head::owner_lock(home, &id)?;
1146 let state = RunState::load_under(&id, home)?;
1147 ensure!(
1148 state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
1149 == Liveness::Dead,
1150 "run live or ownership unknown"
1151 );
1152 ensure!(crate::clean::due(now, state.output_activity(), disk.fold_grace_secs), "execution activity after inventory: {}", state.output_activity());
1153 let _driver = crate::driver::maintenance(home, &id)?;
1154 let _outputs = OutputLease::take(&path, true)?;
1155 ensure!(
1156 worktrees(&state).await?.contains(&path),
1157 "worktree registration changed"
1158 );
1159 let target = validate_cargo_target(&path, true).await
1160 .with_context(|| format!("stage validate-target: {}", path.join("target").display()))?;
1161 let _cargo = cargo_locks(&target)
1162 .with_context(|| format!("stage cargo-locks: {}", target.display()))?;
1163 validate_cargo_target(&path, true).await
1166 .with_context(|| format!("stage validate-target (locked): {}", target.display()))?;
1167 let lease_meta = _outputs._file.metadata().context("output lease activity: metadata")?;
1168 if lease_meta.len() > 0 {
1169 ensure!(lease_meta.len() == 1, "output lease activity witness is unreadable or incomplete for {}", path.display());
1170 let lease_at = Timestamp::try_from(lease_meta.modified().context("output lease activity: modified")?)?;
1171 ensure!(crate::clean::due(now, lease_at, disk.fold_grace_secs),
1172 "output lease activity within grace period: {lease_at} (now {now}, grace {}s)", disk.fold_grace_secs);
1173 }
1174 output_age(&target, now, disk.fold_grace_secs)
1175 .with_context(|| format!("stage output-age: {}", target.display()))?;
1176 let mut outputs = declared_outputs(&target)
1177 .with_context(|| format!("stage declared-outputs: {}", target.display()))?;
1178 let incremental = match crate::incremental::candidates(&path) {
1179 Ok(candidates) => candidates,
1180 Err(error) => {
1181 report.retained.push(format!("{id}: incremental cache retained: {error:#}"));
1182 crate::incremental::Reclaim::default()
1183 }
1184 };
1185 let incremental_files = incremental.files.clone();
1186 outputs.extend(incremental_files.iter().cloned());
1187 outputs.sort();
1188 outputs.dedup();
1189 ensure!(!outputs.is_empty(), "no dep-info-proven outputs or provenance-proven incremental caches; legacy incremental/build and unknown files retained");
1190 let sizes = outputs.iter().map(|p| std::fs::metadata(p).map(|m| m.len())
1191 .with_context(|| format!("stage measure: {}", p.display())))
1192 .collect::<Result<Vec<_>>>()?;
1193 let current_size = sizes.iter().fold(0u64, |bytes, size| bytes.saturating_add(*size));
1194 report.proven_reclaimable_bytes = report.proven_reclaimable_bytes.saturating_add(current_size);
1195 let mut allocation = ReleasedAllocation::of(&outputs)?;
1196 report.proven_reclaimable_allocated_bytes = report.proven_reclaimable_allocated_bytes
1197 .zip(allocation.reclaimable_allocated_bytes()).map(|(a, b)| a.saturating_add(b));
1198 let exceeds_budget = current_size > disk.worktree_reclaim_bytes.saturating_sub(report.freed_bytes);
1199 ensure!(!deletion_started || !exceeds_budget,
1203 "declared outputs exceed remaining per-pass byte budget");
1204 let ledger = home.join("runs").join(&id).join("build-outputs.json");
1207 std::fs::write(
1208 &ledger,
1209 serde_json::to_vec(
1210 &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","identity":if target.join("CACHEDIR.TAG").exists() { "cache-tag" } else { "rustc-profile-dep-info" },"files":outputs}),
1211 )?,
1212 ).with_context(|| format!("stage ledger: {}", ledger.display()))?;
1213 if !incremental_files.is_empty() {
1216 report.retained.push(format!("{id}: incremental proof does not authorize retirement"));
1217 } else if let Err(error) = record_residue(home, &id, &target, &outputs) {
1218 report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
1219 }
1220 ensure!(worktrees(&state).await?.contains(&path),
1221 "worktree registration changed before deletion");
1222 deletion_started = true;
1223 oversized_started = exceeds_budget;
1224 measurements.remove(&target);
1225 for (output, size) in outputs.iter().zip(sizes) {
1226 std::fs::remove_file(output).with_context(|| format!("stage remove: {}", output.display()))?;
1227 report.estimated_released_allocated_bytes = report.estimated_released_allocated_bytes
1228 .zip(allocation.unlinked(output)).map(|(a, b)| a.saturating_add(b));
1229 report.freed_bytes = report.freed_bytes.saturating_add(size);
1231 remaining = remaining.saturating_sub(size);
1232 }
1233 incremental.finish().context("finish incremental reclamation")?;
1234 report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
1235 Ok::<_, anyhow::Error>(())
1236 }
1237 .await;
1238 if let Err(e) = result {
1239 report.retained.push(format!(
1240 "{id}/{}: {e:#}",
1241 path.file_name().unwrap_or_default().to_string_lossy()
1242 ));
1243 }
1244 }
1245 measurements.retain(|path, _| observed.contains(path));
1246 if let Ok(body) = serde_json::to_vec(&measurements)
1247 && let Err(e) = std::fs::write(&measurements_path, body)
1248 {
1249 report.failures.push(format!("output measurements: {e}"));
1250 }
1251 copies::evict(home, disk, now, &mut report).await;
1252 retire_empty(home, disk, now, &mut report).await;
1253 report.protected_or_unproven_bytes = report
1254 .measured_bytes
1255 .saturating_sub(report.proven_reclaimable_bytes);
1256 let pressure_remaining = pressure_paths
1257 .iter()
1258 .any(|path| crate::disk::free_bytes(path).map_or(true, |free| free < disk.min_free_bytes));
1259 if !report.failures.is_empty()
1260 || disk.worktree_limit_bytes > 0
1261 && (remaining > disk.worktree_limit_bytes || pressure_remaining)
1262 {
1263 let details = report
1264 .retained
1265 .iter()
1266 .filter(|reason| {
1267 !reason.contains("within policy")
1268 && !reason.ends_with("unmanifested files and Cargo metadata retained")
1269 })
1270 .take(4)
1271 .chain(report.failures.iter().take(2))
1272 .cloned()
1273 .collect::<Vec<_>>()
1274 .join("; ");
1275 crate::notices::raise_in(
1276 home,
1277 crate::notices::Notice::warn(
1278 "disk:retention",
1279 format!(
1280 "Retention remains blocked by the disk cap, free-space floor or inspection failures (logical inventory {}B, proven reclaimable {}B, protected or unproven {}B, unlinked {}B). {} Retained/held targets: {details}. Inspect the cleanup report for active owners, retained recovery sources and failures.",
1281 report.measured_bytes,
1282 report.proven_reclaimable_bytes,
1283 report.protected_or_unproven_bytes,
1284 report.freed_bytes,
1285 if report.proven_reclaimable_bytes == 0 {
1286 "No inspected contents have deletion authority."
1287 } else {
1288 "Authorized reclamation was insufficient; remaining protections and budgets still apply."
1289 }
1290 ),
1291 ),
1292 );
1293 }
1294 report.observed_home_free_change_bytes = free_before
1295 .zip(crate::disk::free_bytes(home).ok())
1296 .map(|(before, after)| {
1297 (i128::from(after) - i128::from(before))
1298 .clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
1299 });
1300 let store = home.join("retention.json");
1301 if let Ok(body) = serde_json::to_vec_pretty(&report) {
1302 let tmp = home.join("retention.json.tmp");
1303 if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
1304 tracing::warn!("retention report could not be saved: {e}");
1305 }
1306 }
1307 tracing::info!(
1308 measured_bytes = report.measured_bytes,
1309 freed_bytes = report.freed_bytes,
1310 retired = report.retired,
1311 retained = report.retained.len(),
1312 failures = report.failures.len(),
1313 "retained worktree cleanup"
1314 );
1315 for reason in report.retained.iter().chain(&report.failures) {
1316 tracing::debug!(%reason, "worktree retained");
1317 }
1318 report
1319}
1320
1321async fn retire_empty(
1322 home: &Path,
1323 disk: &crate::config::Disk,
1324 now: Timestamp,
1325 report: &mut Report,
1326) {
1327 if !disk.auto_fold {
1328 return;
1329 }
1330 let queue = crate::queue::Queue::at(home.join("queue"));
1331 let questions = crate::ask::Questions::at(home.join("questions"));
1332 let open = questions.list();
1333 for listed in queue.list() {
1334 let Ok(_claim) = queue.claim(&listed.id) else {
1335 continue;
1336 };
1337 let Ok(task) = queue.get(&listed.id) else {
1338 continue;
1339 };
1340 if open
1341 .iter()
1342 .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
1343 {
1344 report.retained.push(format!(
1345 "{}: task has an unanswered owner decision",
1346 task.id
1347 ));
1348 continue;
1349 }
1350 let sources: Vec<_> = task
1351 .runs
1352 .iter()
1353 .collect::<std::collections::BTreeSet<_>>()
1354 .into_iter()
1355 .cloned()
1356 .collect();
1357 let mut useful = Vec::new();
1358 let mut selected = Vec::new();
1359 let mut public = Vec::new();
1360 let mut unreadable = false;
1361 for id in &sources {
1362 let Ok(state) = RunState::load_under(id, home) else {
1363 unreadable = true;
1364 continue;
1365 };
1366 if state.released() || state.retired_to.is_some() {
1367 continue;
1368 }
1369 if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
1370 selected.push(state.clone());
1371 }
1372 if state.pr.is_some() && state.pr_taken_over_by.is_none() {
1373 public.push(state.clone());
1374 }
1375 if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
1376 useful.push(state);
1377 }
1378 }
1379 let pool = if !selected.is_empty() {
1383 selected
1384 } else if !public.is_empty() {
1385 public
1386 } else {
1387 useful
1388 };
1389 if unreadable || pool.len() != 1 {
1390 report.retained.push(format!(
1391 "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
1392 task.id,
1393 sources.len()
1394 ));
1395 continue;
1396 }
1397 let authority = &pool[0];
1398 if authority.candidates.is_empty() {
1399 continue;
1400 }
1401 let latest = &authority.id;
1402 for id in &sources {
1403 if id == latest || report.retired >= 5 {
1404 continue;
1405 }
1406 if let Ok(state) = RunState::load_under(id, home)
1407 && state.retired_to.as_ref() == Some(latest)
1408 && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
1409 {
1410 continue;
1411 }
1412 let result = async {
1413 ensure!(
1414 !task
1415 .resume_override
1416 .as_ref()
1417 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
1418 "operator-pinned recovery run"
1419 );
1420 ensure!(
1421 !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
1422 "selected replacement recovery source"
1423 );
1424 let _owner = crate::pr_head::owner_lock(home, id)?;
1425 let mut state = RunState::load_under(id, home)?;
1426 ensure!(
1427 state.repo == authority.repo,
1428 "repository differs from recovery authority"
1429 );
1430 ensure!(
1431 !crate::pr_recovery::explicit_sources_retained(home, id),
1432 "explicitly retained recovery source"
1433 );
1434 ensure!(
1435 state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
1436 == Liveness::Dead,
1437 "run live or ownership unknown"
1438 );
1439 let _driver = crate::driver::maintenance(home, id)?;
1440 ensure!(
1441 crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
1442 "within grace period"
1443 );
1444 ensure!(
1445 !open.iter().any(|q| q.run == *id && q.status.open()),
1446 "unanswered owner decision"
1447 );
1448 ensure!(
1449 state.pr.is_none()
1450 && state.pr_head.is_none()
1451 && state.pr_recovery.is_none()
1452 && state.land_approval.is_none()
1453 && state.merge_decision.is_none()
1454 && state.reviews.is_empty()
1455 && state.recovery.is_none()
1456 && state.candidate_preservation.is_empty()
1457 && state.interrupted_implementations.is_empty()
1458 && !state.released(),
1459 "PR ownership or recovery checkpoint"
1460 );
1461 let proof = prove_empty(home, &state, latest).await?;
1462 commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
1463 for path in &proof.paths {
1466 ensure!(
1467 git::worktree_remove_clean(&state.repo, path).await?,
1468 "Git refused clean worktree removal"
1469 );
1470 }
1471 Ok::<_, anyhow::Error>(())
1472 }
1473 .await;
1474 match result {
1475 Ok(()) => report.retired += 1,
1476 Err(e) => report.retained.push(format!("{id}: {e:#}")),
1477 }
1478 }
1479 }
1480}
1481
1482struct EmptyProof {
1483 paths: Vec<PathBuf>,
1484 _outputs: Vec<OutputLease>,
1485 _cargo: Vec<File>,
1486}
1487
1488pub(crate) async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
1491 let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
1492 let mut files = Vec::new();
1493 let mut blobs = Vec::new();
1494 for record in tree.split('\0').filter(|s| !s.is_empty()) {
1495 let (header, name) = record.split_once('\t').context("unknown tree record")?;
1496 let parts: Vec<_> = header.split_whitespace().collect();
1497 ensure!(
1498 parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
1499 "unsupported tracked source kind"
1500 );
1501 let path = worktree.join(name);
1502 let meta = std::fs::symlink_metadata(&path)?;
1503 ensure!(
1504 meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
1505 "noncanonical tracked source"
1506 );
1507 #[cfg(unix)]
1508 {
1509 use std::os::unix::fs::PermissionsExt as _;
1510 ensure!(
1511 (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
1512 "tracked executable mode differs from original base"
1513 );
1514 }
1515 path.to_str().context("non UTF-8 source")?;
1516 files.push(git::path_arg(&path));
1517 blobs.push(parts[2].to_owned());
1518 }
1519 for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
1520 let mut args = vec!["hash-object", "--no-filters", "--"];
1521 args.extend(paths.iter().map(String::as_str));
1522 let actual = git::git(worktree, &args).await?;
1523 ensure!(
1524 actual.lines().eq(expected.iter().map(String::as_str)),
1525 "tracked bytes differ from original base"
1526 );
1527 }
1528 Ok(())
1529}
1530
1531async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
1532 let id = &state.id;
1533 let paths = worktrees(state).await?;
1534 let mut guards = Vec::new();
1535 let mut cargo_guards = Vec::new();
1536 for path in &paths {
1537 guards.push(OutputLease::take(path, true)?);
1538 ensure!(
1539 git::rev_parse(path, "HEAD").await? == state.base_commit,
1540 "unique or unrecorded commits"
1541 );
1542 ensure!(
1543 git::git(
1544 path,
1545 &[
1546 "--no-optional-locks",
1547 "status",
1548 "--porcelain",
1549 "--untracked-files=all"
1550 ]
1551 )
1552 .await?
1553 .is_empty(),
1554 "tracked or untracked source changes"
1555 );
1556 ensure!(
1557 git::git(
1558 path,
1559 &[
1560 "--no-optional-locks",
1561 "status",
1562 "--porcelain",
1563 "--untracked-files=all",
1564 "--ignored",
1565 "--",
1566 ".",
1567 ":(exclude)target"
1568 ]
1569 )
1570 .await?
1571 .is_empty(),
1572 "source changes, ignored files or build outputs"
1573 );
1574 tracked_base(path, &state.base_commit).await?;
1575 match std::fs::symlink_metadata(path.join("target")) {
1576 Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
1577 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1578 Err(error) => return Err(error.into()),
1579 }
1580 for checkpoint in [
1581 "rebase-merge",
1582 "rebase-apply",
1583 "sequencer",
1584 "BISECT_LOG",
1585 "REVERT_HEAD",
1586 "AUTO_MERGE",
1587 "MERGE_MSG",
1588 "CHERRY_PICK_HEAD",
1589 "MERGE_HEAD",
1590 "REBASE_HEAD",
1591 ] {
1592 let location = git::git(
1593 path,
1594 &[
1595 "rev-parse",
1596 "--path-format=absolute",
1597 "--git-path",
1598 checkpoint,
1599 ],
1600 )
1601 .await?;
1602 ensure!(
1603 matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
1604 "retained Git recovery metadata: {checkpoint}"
1605 );
1606 }
1607 let origin_path = git::git(
1610 path,
1611 &[
1612 "rev-parse",
1613 "--path-format=absolute",
1614 "--git-path",
1615 "ORIG_HEAD",
1616 ],
1617 )
1618 .await?;
1619 match std::fs::symlink_metadata(origin_path.trim()) {
1620 Ok(meta) => {
1621 ensure!(
1622 meta.is_file() && !meta.file_type().is_symlink(),
1623 "unknown ORIG_HEAD artifact"
1624 );
1625 git::rev_parse(path, "ORIG_HEAD")
1626 .await
1627 .context("unreadable ORIG_HEAD")?;
1628 }
1629 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1630 Err(e) => return Err(e.into()),
1631 }
1632 if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1633 ensure!(
1634 git::git_raw(
1635 path,
1636 &["merge-base", "--is-ancestor", &original, &state.base_commit]
1637 )
1638 .await?
1639 .ok(),
1640 "ORIG_HEAD retains commits outside the original base"
1641 );
1642 }
1643 let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1644 for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1645 ensure!(
1646 git::git_raw(
1647 path,
1648 &["merge-base", "--is-ancestor", head, &state.base_commit]
1649 )
1650 .await?
1651 .ok(),
1652 "worktree reflog retains commits outside the original base"
1653 );
1654 }
1655 for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1656 ensure!(
1657 !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1658 .await?
1659 .ok(),
1660 "unfinished Git operation"
1661 );
1662 }
1663 }
1664 if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1667 return Ok(EmptyProof {
1668 paths,
1669 _outputs: guards,
1670 _cargo: cargo_guards,
1671 });
1672 }
1673 ensure!(!paths.is_empty(), "no worktrees to prove empty");
1674 for candidate in &state.candidates {
1675 ensure!(
1676 (paths.contains(&candidate.worktree)
1677 || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1678 && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1679 "candidate recovery source"
1680 );
1681 }
1682 let pattern = format!("refs/heads/magi/{}/", state.short());
1683 let branches = git::git(
1684 &state.repo,
1685 &["for-each-ref", "--format=%(objectname)", &pattern],
1686 )
1687 .await?;
1688 ensure!(
1689 branches.lines().all(|head| head == state.base_commit),
1690 "unrecorded candidate branch changes"
1691 );
1692 Ok(EmptyProof {
1693 paths,
1694 _outputs: guards,
1695 _cargo: cargo_guards,
1696 })
1697}
1698
1699fn commit_retirement(
1704 home: &Path,
1705 queue: &crate::queue::Queue,
1706 expected: &crate::queue::Task,
1707 state: &mut RunState,
1708 authority: &str,
1709 sources: &[String],
1710 now: Timestamp,
1711) -> Result<()> {
1712 let _write = queue.lock_task(&expected.id)?;
1713 let current = queue.get(&expected.id)?;
1714 ensure!(
1715 serde_json::to_value(¤t)? == serde_json::to_value(expected)?,
1716 "task changed during retirement proof; retaining worktrees for the current owner decision"
1717 );
1718 ensure!(
1719 !current
1720 .resume_override
1721 .as_ref()
1722 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1723 && !current
1724 .replacement
1725 .as_ref()
1726 .is_some_and(|pin| pin.run == state.id),
1727 "current owner selected this recovery source"
1728 );
1729 ensure!(
1730 !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1731 "explicitly retained recovery source"
1732 );
1733 ensure!(
1734 !crate::ask::Questions::at(home.join("questions"))
1735 .list()
1736 .iter()
1737 .any(|q| q.status.open()
1738 && (q.run == state.id
1739 || q.run == current.id
1740 || current.blocked_by.contains(&q.id))),
1741 "owner decision arrived during retirement proof"
1742 );
1743 let dir = home.join("recovery-lineages");
1744 std::fs::create_dir_all(&dir)?;
1745 let tmp = dir.join(format!("{}.tmp", current.id));
1746 let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1747 "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1748 std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1749 std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1750 state.retired_to = Some(authority.to_owned());
1751 state.event(
1752 "retention",
1753 format!(
1754 "empty attempt retired to {authority}; recovery sources: {}",
1755 sources.join(", ")
1756 ),
1757 );
1758 state.save_under(home)?;
1759 Ok(())
1761}
1762
1763pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1767 if min_free == 0 {
1768 return std::future::pending().await;
1769 }
1770 loop {
1771 for path in &paths {
1772 let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1773 match crate::disk::free_bytes(existing) {
1774 Ok(free) if free >= min_free / 2 => {}
1775 Ok(free) => {
1776 return format!(
1777 "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1778 );
1779 }
1780 Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1781 }
1782 }
1783 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1784 }
1785}
1786
1787#[cfg(all(test, unix))]
1788mod tests {
1789 use super::*;
1790 use crate::{
1791 config::{Config, Disk},
1792 queue::{Source, Task},
1793 run::{Candidate, RunStatus},
1794 };
1795
1796 struct Fx {
1797 _temp: tempfile::TempDir,
1798 home: PathBuf,
1799 repo: PathBuf,
1800 bay: PathBuf,
1801 base: String,
1802 }
1803 impl Fx {
1804 async fn new() -> Self {
1805 let temp = tempfile::tempdir().unwrap();
1806 let root = temp.path().canonicalize().unwrap();
1807 let home = root.join("home");
1808 let repo = root.join("repo");
1809 let bay = root.join("worktrees");
1810 std::fs::create_dir_all(&repo).unwrap();
1811 std::fs::create_dir_all(home.join("runs")).unwrap();
1812 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1813 git::git(&repo, &["config", "user.name", "Fixture"])
1814 .await
1815 .unwrap();
1816 git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1817 .await
1818 .unwrap();
1819 git::git(&repo, &["config", "commit.gpgsign", "false"])
1820 .await
1821 .unwrap();
1822 std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1823 std::fs::write(repo.join("source"), "base\n").unwrap();
1824 git::git(&repo, &["add", "."]).await.unwrap();
1825 git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1826 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1827 Self {
1828 _temp: temp,
1829 home,
1830 repo,
1831 bay,
1832 base,
1833 }
1834 }
1835 async fn run(&self, tag: &str, useful: bool) -> RunState {
1836 let mut cfg = Config::default();
1837 cfg.graph.worktree_root = Some(self.bay.clone());
1838 let mut state = RunState::new(
1839 self.repo.clone(),
1840 "main".into(),
1841 self.base.clone(),
1842 "task".into(),
1843 cfg,
1844 );
1845 state.id = format!("20260101-000000-{tag}");
1846 state.status = RunStatus::Blocked;
1847 state.driver_exited = true;
1848 let wt = state.worktree_root().join("cand-A");
1849 let branch = format!("magi/{tag}/A");
1850 git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1851 .await
1852 .unwrap();
1853 if useful {
1854 std::fs::write(wt.join("source"), tag).unwrap();
1855 git::git(&wt, &["add", "."]).await.unwrap();
1856 git::git(&wt, &["commit", "-m", "candidate change"])
1857 .await
1858 .unwrap();
1859 }
1860 state.candidates.push(Candidate {
1861 index: 0,
1862 label: 'A',
1863 agent: "fixture".into(),
1864 branch,
1865 worktree: wt,
1866 summary: String::new(),
1867 stat: String::new(),
1868 files: usize::from(useful),
1869 commits: usize::from(useful),
1870 empty: !useful,
1871 failed: None,
1872 verified_noop: None,
1873 duration_ms: 0,
1874 folded: false,
1875 });
1876 state.save_under(&self.home).unwrap();
1877 state
1878 }
1879 fn disk() -> Disk {
1880 Disk {
1881 worktree_limit_bytes: 1,
1882 worktree_reclaim_bytes: 1024 * 1024,
1883 fold_grace_secs: 0,
1884 min_free_bytes: 0,
1885 ..Disk::default()
1886 }
1887 }
1888 fn target(wt: &Path) {
1889 let target = wt.join("target");
1890 std::fs::create_dir_all(target.join("debug")).unwrap();
1891 let mut artifact = [1; 200];
1892 artifact[..4].copy_from_slice(b"\x7fELF");
1893 std::fs::write(target.join("debug/output"), artifact).unwrap();
1894 let output = target
1895 .join("debug/output")
1896 .display()
1897 .to_string()
1898 .replace('\\', "\\\\")
1899 .replace(' ', "\\ ");
1900 std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1901 std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1902 std::fs::write(
1903 target.join("CACHEDIR.TAG"),
1904 "Signature: 8a477f597d28d172789f06886806bc55\n",
1905 )
1906 .unwrap();
1907 std::fs::write(
1908 target.join(".rustc_info.json"),
1909 "{\"rustc_fingerprint\":123}",
1910 )
1911 .unwrap();
1912 }
1913 fn now() -> Timestamp {
1914 Timestamp::now() + jiff::SignedDuration::from_secs(2)
1915 }
1916 fn task(&self, runs: &[RunState]) -> Task {
1917 let mut task = Task::new(
1918 "fixture".into(),
1919 "task".into(),
1920 self.repo.clone(),
1921 Source::Human,
1922 );
1923 task.runs = runs.iter().map(|s| s.id.clone()).collect();
1924 crate::queue::Queue::at(self.home.join("queue"))
1925 .put(&mut task)
1926 .unwrap();
1927 task
1928 }
1929 }
1930
1931 #[cfg(unix)]
1932 #[tokio::test]
1933 async fn incremental_only_reclamation_preserves_legacy_and_unknown_data() {
1934 let f = Fx::new().await;
1935 let state = f.run("ca01", true).await;
1936 let wt = &state.candidates[0].worktree;
1937 let generation = crate::incremental::tests::generate(wt);
1938 for dir in [wt.join("target/debug"), wt.join("target/debug/deps")] {
1940 for entry in std::fs::read_dir(dir).unwrap() {
1941 let path = entry.unwrap().path();
1942 if path.extension().is_some_and(|x| x == "d") {
1943 std::fs::remove_file(path).unwrap();
1944 }
1945 }
1946 }
1947 assert!(declared_outputs(&wt.join("target")).unwrap().is_empty());
1948 assert_eq!(
1949 sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes,
1950 0,
1951 "live generation is protected"
1952 );
1953 drop(generation);
1954 let candidates = crate::incremental::candidates(wt).unwrap();
1955 let logical: u64 = candidates
1956 .files
1957 .iter()
1958 .map(|p| std::fs::metadata(p).unwrap().len())
1959 .sum();
1960 let unknown = candidates.files[0].parent().unwrap().join("operator-note");
1961 std::fs::write(&unknown, "retain unknown data").unwrap();
1962 drop(candidates);
1963 let refused = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1964 assert_eq!(refused.freed_bytes, 0, "{refused:?}");
1965 assert!(
1966 refused
1967 .retained
1968 .iter()
1969 .any(|s| s.contains("provenance mismatch"))
1970 );
1971 std::fs::remove_file(&unknown).unwrap();
1972 let legacy = wt.join("target/debug/incremental/legacy-session");
1973 std::fs::create_dir(&legacy).unwrap();
1974 let candidates = crate::incremental::candidates(wt).unwrap();
1976 let bin = candidates
1977 .files
1978 .iter()
1979 .find(|p| p.extension().is_some_and(|x| x == "bin"))
1980 .unwrap();
1981 std::fs::copy(bin, legacy.join("query-cache.bin")).unwrap();
1982 drop(candidates);
1983 let cargo = cargo_locks(&wt.join("target")).unwrap();
1984 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1985 drop(cargo);
1986 let outputs = OutputLease::borrow(wt).unwrap().unwrap();
1987 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1988 drop(outputs);
1989 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1990 assert_eq!(report.freed_bytes, logical, "{report:?}");
1991 assert_eq!(report.proven_reclaimable_bytes, logical);
1992 assert!(report.measured_bytes > logical);
1993 assert_eq!(
1994 report.protected_or_unproven_bytes,
1995 report.measured_bytes - logical
1996 );
1997 #[cfg(unix)]
1998 {
1999 assert!(report.estimated_released_allocated_bytes.unwrap() > 0);
2000 assert!(
2001 report.measured_allocated_file_bytes.unwrap()
2002 >= report.estimated_released_allocated_bytes.unwrap()
2003 );
2004 }
2005 assert!(report.observed_home_free_change_bytes.is_some());
2006 assert!(legacy.join("query-cache.bin").is_file());
2007 assert!(wt.join("src/lib.rs").is_file());
2008 assert_eq!(report.retired, 0);
2009 }
2010
2011 #[cfg(unix)]
2012 #[tokio::test]
2013 async fn incremental_and_deps_links_account_for_retained_third_link() {
2014 use std::os::unix::fs::MetadataExt;
2015 let f = Fx::new().await;
2016 let state = f.run("ca02", true).await;
2017 let wt = &state.candidates[0].worktree;
2018 let generation = crate::incremental::tests::generate(wt);
2019 drop(generation);
2020 let candidates = crate::incremental::candidates(wt).unwrap();
2021 let objects = candidates
2022 .files
2023 .iter()
2024 .filter(|p| p.extension().is_some_and(|x| x == "o"))
2025 .take(2)
2026 .cloned()
2027 .collect::<Vec<_>>();
2028 assert_eq!(
2029 objects.len(),
2030 2,
2031 "fixture must produce real incremental objects"
2032 );
2033 let mut outputs = candidates.files.clone();
2034 drop(candidates);
2035 let target = wt.join("target");
2036 for dir in [target.join("debug"), target.join("debug/deps")] {
2037 for entry in std::fs::read_dir(dir).unwrap() {
2038 let path = entry.unwrap().path();
2039 if path.extension().is_some_and(|x| x == "d") {
2040 std::fs::remove_file(path).unwrap();
2041 }
2042 }
2043 }
2044 for entry in std::fs::read_dir(target.join("debug/deps")).unwrap() {
2047 let path = entry.unwrap().path();
2048 if path.is_file() {
2049 std::fs::remove_file(path).unwrap();
2050 }
2051 }
2052 for (index, object) in objects.iter().enumerate() {
2053 assert_eq!(std::fs::metadata(object).unwrap().nlink(), 1);
2054 let dep = target.join(format!("debug/deps/link{index}"));
2055 std::fs::hard_link(object, &dep).unwrap();
2056 std::fs::write(
2057 dep.with_extension("d"),
2058 format!("{}: src/lib.rs\n", dep.display()),
2059 )
2060 .unwrap();
2061 outputs.push(dep);
2062 }
2063 let retained = wt.join("retained-input");
2064 std::fs::hard_link(&objects[1], &retained).unwrap();
2065 let sentinel = std::fs::read(&retained).unwrap();
2066 let logical: u64 = outputs
2067 .iter()
2068 .map(|p| std::fs::metadata(p).unwrap().len())
2069 .sum();
2070 let mut seen = std::collections::BTreeSet::new();
2071 let retained_meta = std::fs::metadata(&retained).unwrap();
2072 let retained_inode = (retained_meta.dev(), retained_meta.ino());
2073 let mut expected = 0u64;
2074 for output in &outputs {
2075 let m = std::fs::metadata(output).unwrap();
2076 let inode = (m.dev(), m.ino());
2077 if inode != retained_inode && seen.insert(inode) {
2078 expected += m.blocks() * 512;
2079 }
2080 }
2081 assert!(expected > 0);
2082 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2083 assert_eq!(report.proven_reclaimable_bytes, logical, "{report:?}");
2084 assert_eq!(report.freed_bytes, logical, "{report:?}");
2085 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(expected));
2086 assert_eq!(report.estimated_released_allocated_bytes, Some(expected));
2087 assert!(outputs.iter().all(|p| !p.exists()));
2088 assert_eq!(std::fs::read(retained).unwrap(), sentinel);
2089 assert!(wt.join("src/lib.rs").is_file());
2090 }
2091
2092 #[tokio::test]
2093 async fn retention_notice_explains_unproven_inventory_and_live_owner() {
2094 let f = Fx::new().await;
2095 let state = f.run("ca03", true).await;
2096 let wt = &state.candidates[0].worktree;
2097 Fx::target(wt);
2098 std::fs::remove_file(wt.join("target/debug/output")).unwrap();
2099 let unknown = wt.join("target/debug/operator-note");
2100 std::fs::write(&unknown, vec![1; 4096]).unwrap();
2101 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2102 assert_eq!(report.proven_reclaimable_bytes, 0);
2103 #[cfg(unix)]
2104 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(0));
2105 assert!(report.protected_or_unproven_bytes >= 4096);
2106 let message = || {
2107 crate::notices::Notices::at(f.home.join("notifications"))
2108 .list()
2109 .into_iter()
2110 .find(|n| n.key == "disk:retention")
2111 .unwrap()
2112 .message
2113 };
2114 let notice = message();
2115 assert!(notice.contains("proven reclaimable 0B"), "{notice}");
2116 assert!(notice.contains("protected or unproven"), "{notice}");
2117 assert!(
2118 notice.contains("legacy incremental/build and unknown files retained"),
2119 "{notice}"
2120 );
2121 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2122 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2123 assert_eq!(report.freed_bytes, 0);
2124 let notice = message();
2125 assert!(notice.contains(&state.id), "{notice}");
2126 assert!(
2127 notice.contains("lock") || notice.contains("owner"),
2128 "{notice}"
2129 );
2130 drop(owner);
2131 assert_eq!(std::fs::read(unknown).unwrap(), vec![1; 4096]);
2132 }
2133
2134 #[tokio::test]
2135 async fn real_cargo_build_holds_the_target_against_reclamation() {
2136 let f = Fx::new().await;
2137 let state = f.run("ca02", true).await;
2138 let wt = &state.candidates[0].worktree;
2139 Fx::target(wt);
2140 std::fs::create_dir(wt.join("src")).unwrap();
2141 std::fs::write(
2142 wt.join("Cargo.toml"),
2143 "[package]\nname='live-cache-fixture'\nversion='0.1.0'\nedition='2021'\n[workspace]\n",
2144 )
2145 .unwrap();
2146 std::fs::write(wt.join("src/lib.rs"), "pub fn answer() -> usize { 42 }\n").unwrap();
2147 std::fs::write(
2148 wt.join("build.rs"),
2149 r#"fn main() {
2150 println!("cargo:rerun-if-env-changed=FIXTURE_READY");
2151 let Ok(ready) = std::env::var("FIXTURE_READY") else { return };
2152 let release = std::env::var("FIXTURE_RELEASE").unwrap();
2153 std::fs::write(ready, "ready").unwrap();
2154 let start = std::time::Instant::now();
2155 while !std::path::Path::new(&release).exists() {
2156 assert!(start.elapsed().as_secs() < 120, "fixture was not released");
2157 std::thread::sleep(std::time::Duration::from_millis(10));
2158 }
2159 }"#,
2160 )
2161 .unwrap();
2162 let ready = wt.join("ready");
2163 let release = wt.join("release");
2164 let mut cargo = tokio::process::Command::new("cargo");
2165 cargo.env_clear();
2166 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2167 if let Some(value) = std::env::var_os(key) {
2168 cargo.env(key, value);
2169 }
2170 }
2171 cargo
2172 .current_dir(wt)
2173 .env("RUSTUP_TOOLCHAIN", "stable")
2174 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2175 .env("CARGO_INCREMENTAL", "1")
2176 .args(["build", "--offline", "--jobs", "1", "--target-dir"])
2177 .arg(wt.join("target"));
2178 let warm = cargo.output().await.unwrap();
2182 assert!(
2183 warm.status.success(),
2184 "fixture compilation failed: {}",
2185 String::from_utf8_lossy(&warm.stderr)
2186 );
2187 let log_path = f._temp.path().join("live-cargo.stderr");
2188 let mut child = cargo
2189 .env("FIXTURE_READY", &ready)
2190 .env("FIXTURE_RELEASE", &release)
2191 .stdout(std::process::Stdio::null())
2192 .stderr(std::fs::File::create(&log_path).unwrap())
2193 .kill_on_drop(true)
2194 .spawn()
2195 .unwrap();
2196 let readiness = tokio::time::timeout(std::time::Duration::from_secs(60), async {
2197 while !ready.exists() {
2198 if let Some(status) = child.try_wait().unwrap() {
2199 return Err(format!("fixture exited before readiness: {status}"));
2200 }
2201 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2202 }
2203 Ok(())
2204 })
2205 .await;
2206 let arrived = matches!(readiness, Ok(Ok(())));
2207 let report = if arrived {
2208 Some(sweep(&f.home, &Fx::disk(), Fx::now()).await)
2209 } else {
2210 None
2211 };
2212 std::fs::write(release, "release").unwrap();
2214 let status = tokio::time::timeout(std::time::Duration::from_secs(60), child.wait()).await;
2215 if status.is_err() {
2216 child.kill().await.unwrap();
2217 child.wait().await.unwrap();
2218 }
2219 assert!(
2220 arrived,
2221 "fixture build never reached its active build script: {readiness:?}\n{}",
2222 std::fs::read_to_string(&log_path).unwrap()
2223 );
2224 assert!(
2225 status.unwrap().unwrap().success(),
2226 "fixture build failed: {}",
2227 std::fs::read_to_string(&log_path).unwrap()
2228 );
2229 let report = report.unwrap();
2230 assert_eq!(report.freed_bytes, 0, "{report:?}");
2231 assert!(
2232 report
2233 .retained
2234 .iter()
2235 .any(|s| s.contains("Cargo build is active")),
2236 "{report:?}"
2237 );
2238 assert!(wt.join("target/debug/output").is_file());
2239 assert!(crate::disk::dir_size(&wt.join("target/debug/incremental")) > 0);
2240 }
2241
2242 #[cfg(unix)]
2243 #[test]
2244 fn allocated_accounting_deduplicates_links_and_requires_the_last_link() {
2245 let temp = tempfile::tempdir().unwrap();
2246 let a = temp.path().join("a");
2247 let b = temp.path().join("b");
2248 let outside = temp.path().join("retained-input");
2249 std::fs::write(&a, vec![1; 65536]).unwrap();
2250 std::fs::hard_link(&a, &b).unwrap();
2251 std::fs::hard_link(&a, &outside).unwrap();
2252 let mut seen = Default::default();
2253 let blocks = allocated_inventory(&temp.path().canonicalize().unwrap(), &mut seen).unwrap();
2254 assert!(blocks > 0);
2255 let mut accounting = ReleasedAllocation::of(&[a.clone(), b.clone()]).unwrap();
2256 std::fs::remove_file(&a).unwrap();
2257 assert_eq!(accounting.unlinked(&a), Some(0));
2258 std::fs::remove_file(&b).unwrap();
2259 assert_eq!(accounting.unlinked(&b), Some(0));
2260 assert_eq!(std::fs::read(&outside).unwrap().len(), 65536);
2261 let mut accounting = ReleasedAllocation::of(std::slice::from_ref(&outside)).unwrap();
2262 std::fs::remove_file(&outside).unwrap();
2263 assert_eq!(accounting.unlinked(&outside), Some(blocks));
2264 }
2265
2266 #[cfg(unix)]
2267 #[test]
2268 fn relative_admission_home_matches_absolute_home_and_retains_symlink_checks() {
2269 let temp = tempfile::tempdir_in(".").unwrap();
2270 let home = PathBuf::from(temp.path().file_name().unwrap()).join("state");
2271 assert!(home.is_relative());
2272 let paths = vec![std::env::current_dir().unwrap()];
2273 let gib = 1024 * 1024 * 1024;
2274 let reservation = OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).unwrap();
2275 assert!(
2276 OutputLease::admit_with(&home.canonicalize().unwrap(), &paths, 8 * gib, |_| Ok(
2277 9 * gib
2278 ))
2279 .is_err()
2280 );
2281 drop(reservation);
2282 assert!(OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).is_ok());
2283 let bad = temp.path().join("symlink-state");
2284 std::fs::create_dir(&bad).unwrap();
2285 std::os::unix::fs::symlink(
2286 home.canonicalize().unwrap().join("disk-admission"),
2287 bad.join("disk-admission"),
2288 )
2289 .unwrap();
2290 assert!(OutputLease::admit_with(&bad, &paths, 8 * gib, |_| Ok(9 * gib)).is_err());
2291 }
2292
2293 #[cfg(unix)]
2294 #[test]
2295 fn concurrent_disk_admission_reserves_once_per_volume_until_release() {
2296 let temp = tempfile::tempdir().unwrap();
2297 let home = temp.path().canonicalize().unwrap();
2298 let paths = vec![home.clone(), home.join("not-created-yet")];
2299 let gib = 1024 * 1024 * 1024;
2300 let floor = 8 * gib;
2301 let first = OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap();
2302 assert_eq!(first.len(), 1);
2303 let blocked =
2304 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap_err();
2305 assert!(blocked.to_string().contains("active writer reservations"));
2306 let second =
2307 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + 2 * gib)).unwrap();
2308 drop(first);
2309 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_err());
2310 drop(second);
2311 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_ok());
2312 assert!(
2313 OutputLease::admit_with(&home, &paths, floor, |_| anyhow::bail!("unmeasured")).is_err()
2314 );
2315 }
2316
2317 #[cfg(unix)]
2318 #[test]
2319 fn simultaneous_writers_cannot_spend_the_same_volume_headroom() {
2320 let temp = tempfile::tempdir().unwrap();
2321 let home = temp.path().canonicalize().unwrap();
2322 let barrier = std::sync::Barrier::new(2);
2323 let gib = 1024 * 1024 * 1024;
2324 let results = std::thread::scope(|scope| {
2325 let threads: Vec<_> = (0..2)
2326 .map(|_| {
2327 scope.spawn(|| {
2328 barrier.wait();
2329 OutputLease::admit_with(&home, std::slice::from_ref(&home), 100, |_| {
2330 Ok(100 + gib)
2331 })
2332 })
2333 })
2334 .collect();
2335 threads
2336 .into_iter()
2337 .map(|thread| thread.join().unwrap())
2338 .collect::<Vec<_>>()
2339 });
2340 assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
2341 assert!(
2342 results
2343 .iter()
2344 .filter_map(|r| r.as_ref().err())
2345 .all(|e| e.to_string().contains("active writer reservations"))
2346 );
2347 }
2348
2349 #[tokio::test]
2350 async fn sweep_reclaims_real_cargo_outputs_and_preserves_source_and_unknown_files() {
2351 real_cargo_fixture(false).await;
2352 }
2353
2354 #[tokio::test]
2355 async fn missing_tag_from_precreated_target_reclaims_only_real_cargo_artifacts() {
2356 real_cargo_fixture(true).await;
2357 }
2358
2359 async fn real_cargo_fixture(precreate: bool) {
2360 let f = Fx::new().await;
2361 let state = f.run("ca00", true).await;
2362 let wt = &state.candidates[0].worktree;
2363 let source = "fn main() { println!(\"retained fixture\"); }\n";
2364 std::fs::create_dir(wt.join("src")).unwrap();
2365 std::fs::write(wt.join("src/main.rs"), source).unwrap();
2366 std::fs::write(
2367 wt.join("Cargo.toml"),
2368 "[package]\nname = \"retained-fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n",
2369 )
2370 .unwrap();
2371 if precreate {
2372 std::fs::create_dir_all(wt.join("target/magi-tools")).unwrap();
2373 }
2374 let mut cargo = std::process::Command::new("cargo");
2375 cargo.env_clear();
2376 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2377 if let Some(value) = std::env::var_os(key) {
2378 cargo.env(key, value);
2379 }
2380 }
2381 let build = cargo
2382 .current_dir(wt)
2383 .env("RUSTUP_TOOLCHAIN", "stable")
2384 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2385 .env("CARGO_INCREMENTAL", "0")
2386 .args([
2387 "build",
2388 "--offline",
2389 "--release",
2390 "--jobs",
2391 "1",
2392 "--target-dir",
2393 ])
2394 .arg(wt.join("target"))
2395 .output()
2396 .unwrap();
2397 assert!(
2398 build.status.success(),
2399 "{}",
2400 String::from_utf8_lossy(&build.stderr)
2401 );
2402 assert_eq!(wt.join("target/CACHEDIR.TAG").exists(), !precreate);
2403 if precreate {
2404 let again = cargo.output().unwrap();
2406 assert!(
2407 again.status.success(),
2408 "{}",
2409 String::from_utf8_lossy(&again.stderr)
2410 );
2411 assert!(!wt.join("target/CACHEDIR.TAG").exists());
2412 assert!(cargo_target(wt).await.is_err());
2413 }
2414 git::git(wt, &["add", "Cargo.toml", "Cargo.lock", "src"])
2415 .await
2416 .unwrap();
2417 git::git(wt, &["commit", "-m", "real Cargo fixture"])
2418 .await
2419 .unwrap();
2420 let artifact = wt.join("target/release/retained-fixture");
2421 assert!(artifact.is_file());
2422 let unknown = wt.join("target/release/operator-note");
2423 std::fs::write(&unknown, "keep this note\n").unwrap();
2424
2425 let target = wt.join("target");
2426 let mut preserved = std::collections::BTreeMap::new();
2427 for relative in [
2428 "tmp/sentinel",
2429 "magi-tools/sentinel",
2430 "lifecycle-client-check.log",
2431 ] {
2432 let path = target.join(relative);
2433 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
2434 std::fs::write(&path, format!("retain {relative}\n")).unwrap();
2435 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2436 }
2437 let decoy = target.join("magi-tools/retained-fixture");
2440 std::fs::copy(&artifact, &decoy).unwrap();
2441 let dep_info = decoy.with_extension("d");
2442 std::fs::write(&dep_info, format!("{}: src/main.rs\n", decoy.display())).unwrap();
2443 for path in [
2444 decoy,
2445 dep_info,
2446 wt.join("Cargo.toml"),
2447 wt.join("Cargo.lock"),
2448 ] {
2449 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2450 }
2451 let proven = declared_outputs(&target).unwrap();
2452
2453 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2454 assert!(out.freed_bytes > 0, "{out:?}");
2455 for (path, bytes) in preserved {
2456 assert_eq!(std::fs::read(&path).unwrap(), bytes, "{}", path.display());
2457 }
2458 assert!(!artifact.exists(), "{out:?}");
2459 assert_eq!(
2460 std::fs::read_to_string(wt.join("src/main.rs")).unwrap(),
2461 source
2462 );
2463 assert_eq!(
2464 std::fs::read_to_string(&unknown).unwrap(),
2465 "keep this note\n"
2466 );
2467 assert_eq!(std::fs::read_to_string(wt.join("source")).unwrap(), "ca00");
2468 let ledger: serde_json::Value = serde_json::from_slice(
2469 &std::fs::read(
2470 f.home
2471 .join("runs")
2472 .join(&state.id)
2473 .join("build-outputs.json"),
2474 )
2475 .unwrap(),
2476 )
2477 .unwrap();
2478 assert_eq!(
2479 ledger["identity"],
2480 if precreate {
2481 "rustc-profile-dep-info"
2482 } else {
2483 "cache-tag"
2484 }
2485 );
2486 if precreate {
2487 assert!(!target.join("CACHEDIR.TAG").exists());
2488 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2489 }
2490 let deleted: Vec<PathBuf> = serde_json::from_value(ledger["files"].clone()).unwrap();
2491 assert!(deleted.contains(&artifact));
2492 assert_eq!(deleted, proven);
2493 for path in deleted {
2494 assert!(!path.exists());
2495 assert!(
2496 [target.join("release"), target.join("release/deps")]
2497 .contains(&path.parent().unwrap().to_owned())
2498 );
2499 }
2500 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2501 }
2502
2503 fn missing_tag_target(wt: &Path) {
2504 Fx::target(wt);
2505 let target = wt.join("target");
2506 std::fs::remove_file(target.join("CACHEDIR.TAG")).unwrap();
2507 for subdir in ["debug/deps", "debug/.fingerprint"] {
2508 std::fs::create_dir_all(target.join(subdir)).unwrap();
2509 }
2510 std::fs::write(target.join(".rustc_info.json"),
2511 r#"{"rustc_fingerprint":123,"outputs":{"1":{"success":true,"stdout":"rustc fixture","stderr":""}},"successes":{}}"#).unwrap();
2512 }
2513
2514 #[tokio::test]
2515 async fn bookkeeping_flushes_do_not_extend_artifact_grace_including_legacy_records() {
2516 for legacy in [false, true] {
2517 let f = Fx::new().await;
2518 let mut state = f.run("ac00", true).await;
2519 Fx::target(&state.candidates[0].worktree);
2520 let original = state.updated_at;
2521 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2522 if legacy {
2523 let path = f.home.join("runs").join(&state.id).join("run.json");
2524 let mut value = serde_json::to_value(&state).unwrap();
2525 value.as_object_mut().unwrap().remove("output_activity_at");
2526 std::fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
2527 state = RunState::load_under(&state.id, &f.home).unwrap();
2528 assert_eq!(state.output_activity(), original);
2529 }
2530 let age = state.output_activity();
2531 for offset in [30, 20, 10] {
2532 state
2533 .save_under_at(&f.home, now - jiff::SignedDuration::from_secs(offset))
2534 .unwrap();
2535 state = RunState::load_under(&state.id, &f.home).unwrap();
2536 assert_eq!(state.output_activity(), age);
2537 }
2538 let mut disk = Fx::disk();
2539 disk.fold_grace_secs = 3600;
2540 let report = sweep(&f.home, &disk, now).await;
2541 assert_eq!(report.freed_bytes, 200, "legacy={legacy}: {report:?}");
2542 }
2543 }
2544
2545 #[tokio::test]
2546 async fn recent_execution_outputs_and_released_output_leases_retain_artifacts() {
2547 for evidence in [
2548 "execution",
2549 "artifact",
2550 "dep-info",
2551 "fingerprint",
2552 "lease",
2553 "future",
2554 ] {
2555 let f = Fx::new().await;
2556 let mut state = f.run("ac01", true).await;
2557 let wt = state.candidates[0].worktree.clone();
2558 missing_tag_target(&wt);
2559 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2560 let recent = now - jiff::SignedDuration::from_secs(10);
2561 match evidence {
2562 "execution" => {
2563 state.note_output_activity(recent);
2564 state.save_under_at(&f.home, recent).unwrap();
2565 }
2566 "lease" => {
2567 let lease = OutputLease::borrow(&wt).unwrap().unwrap();
2568 let file = lease._file.try_clone().unwrap();
2569 drop(lease);
2570 file.set_modified(std::time::SystemTime::from(recent))
2571 .unwrap();
2572 }
2573 _ => {
2574 let relative = match evidence {
2575 "dep-info" => "debug/output.d",
2576 "fingerprint" => "debug/.fingerprint/invoked.timestamp",
2577 _ => "debug/output",
2578 };
2579 let path = wt.join("target").join(relative);
2580 let file = OpenOptions::new()
2581 .write(true)
2582 .create(true)
2583 .truncate(false)
2584 .open(path)
2585 .unwrap();
2586 let at = if evidence == "future" {
2587 now + jiff::SignedDuration::from_secs(10)
2588 } else {
2589 recent
2590 };
2591 file.set_modified(std::time::SystemTime::from(at)).unwrap();
2592 }
2593 }
2594 let mut disk = Fx::disk();
2595 disk.fold_grace_secs = 3600;
2596 let report = sweep(&f.home, &disk, now).await;
2597 assert_eq!(report.freed_bytes, 0, "{evidence}: {report:?}");
2598 assert!(
2599 report.retained.iter().any(|s| s.contains("grace period")
2600 || s.contains("execution activity after inventory")),
2601 "{evidence}: {report:?}"
2602 );
2603 let aged = now + jiff::SignedDuration::from_secs(7200);
2604 let report = sweep(&f.home, &disk, aged).await;
2605 assert_eq!(report.freed_bytes, 200, "{evidence}: {report:?}");
2606 }
2607 }
2608
2609 #[tokio::test]
2610 async fn missing_tag_proof_preserves_locks_unknown_files_and_retirement_boundary() {
2611 let f = Fx::new().await;
2612 let state = f.run("ac02", true).await;
2613 let wt = &state.candidates[0].worktree;
2614 missing_tag_target(wt);
2615 let target = wt.join("target");
2616 let unknown = target.join("debug/notes");
2617 std::fs::write(&unknown, "operator data").unwrap();
2618 for lock in [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"] {
2619 let path = target.join("debug").join(lock);
2620 let file = OpenOptions::new()
2621 .create(true)
2622 .truncate(false)
2623 .read(true)
2624 .write(true)
2625 .open(&path)
2626 .unwrap();
2627 crate::pr_head::try_lock_file(&file).unwrap();
2628 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2629 assert_eq!(report.freed_bytes, 0, "{lock}: {report:?}");
2630 assert!(
2631 report
2632 .retained
2633 .iter()
2634 .any(|s| s.contains(path.to_str().unwrap())),
2635 "{report:?}"
2636 );
2637 }
2638 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2639 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2640 drop(owner);
2641 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2642 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2643 drop(driver);
2644 let output = OutputLease::borrow(wt).unwrap().unwrap();
2645 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2646 drop(output);
2647 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2648 assert_eq!(report.freed_bytes, 200, "{report:?}");
2649 assert_eq!(std::fs::read_to_string(unknown).unwrap(), "operator data");
2650 assert!(!target.join("CACHEDIR.TAG").exists());
2651 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2652 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2653 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2654 }
2655
2656 #[tokio::test]
2657 async fn missing_tag_partial_reclamation_can_resume_without_adopting_residue() {
2658 let f = Fx::new().await;
2659 let state = f.run("ac04", true).await;
2660 let wt = &state.candidates[0].worktree;
2661 missing_tag_target(wt);
2662 let target = wt.join("target");
2663 let first = target.join("debug/output");
2664 let second = target.join("debug/second");
2665 std::fs::copy(&first, &second).unwrap();
2666 std::fs::write(
2667 second.with_extension("d"),
2668 format!("{}: source\n", second.display()),
2669 )
2670 .unwrap();
2671 let outputs = declared_outputs(&target).unwrap();
2672 assert_eq!(outputs.len(), 2);
2673 std::fs::write(f.home.join("runs").join(&state.id).join("build-outputs.json"),
2675 serde_json::to_vec(&serde_json::json!({"target":target,"files":outputs,"action":"reclaim-declared-outputs"})).unwrap()).unwrap();
2676 std::fs::remove_file(first).unwrap();
2677 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2678 assert_eq!(report.freed_bytes, 200, "{report:?}");
2679 assert!(!second.exists());
2680 assert!(second.with_extension("d").exists());
2681 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2682 }
2683
2684 #[tokio::test]
2685 async fn missing_tag_proof_rejects_invalid_identity_structure_and_symlinks() {
2686 for invalid in [
2687 "identity",
2688 "deps",
2689 "fingerprint",
2690 "lock",
2691 "dep-info",
2692 "tag-symlink",
2693 "bad-tag",
2694 ] {
2695 let f = Fx::new().await;
2696 let state = f.run("ac03", true).await;
2697 let wt = &state.candidates[0].worktree;
2698 missing_tag_target(wt);
2699 let target = wt.join("target");
2700 match invalid {
2701 "identity" => std::fs::write(
2702 target.join(".rustc_info.json"),
2703 r#"{"rustc_fingerprint":123}"#,
2704 )
2705 .unwrap(),
2706 "deps" | "fingerprint" => {
2707 let path = target.join("debug").join(if invalid == "deps" {
2708 "deps"
2709 } else {
2710 ".fingerprint"
2711 });
2712 std::fs::remove_dir(&path).unwrap();
2713 std::os::unix::fs::symlink(&f.repo, path).unwrap();
2714 }
2715 "lock" => {
2716 std::fs::remove_file(target.join("debug/.cargo-lock")).unwrap();
2717 }
2718 "dep-info" => std::fs::write(target.join("debug/output.d"), "").unwrap(),
2719 "tag-symlink" => {
2720 std::os::unix::fs::symlink(target.join("absent"), target.join("CACHEDIR.TAG"))
2721 .unwrap()
2722 }
2723 "bad-tag" => std::fs::write(target.join("CACHEDIR.TAG"), "unknown").unwrap(),
2724 _ => unreachable!(),
2725 }
2726 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2727 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
2728 assert!(target.join("debug/output").is_file());
2729 }
2730 }
2731
2732 #[tokio::test]
2733 async fn sweep_budget_oversized_target_makes_progress_without_a_second_deletion() {
2734 let f = Fx::new().await;
2735 let first = f.run("ba00", true).await;
2736 let second = f.run("ba01", true).await;
2737 for state in [&first, &second] {
2738 Fx::target(&state.candidates[0].worktree);
2739 }
2740 let mut disk = Fx::disk();
2741 disk.worktree_reclaim_bytes = 100;
2742 let out = sweep(&f.home, &disk, Fx::now()).await;
2743 assert_eq!(out.freed_bytes, 200, "{out:?}");
2744 assert!(
2745 out.retained
2746 .contains(&format!("{}: per-pass byte budget spent", second.id)),
2747 "{out:?}"
2748 );
2749 assert!(
2750 !first.candidates[0]
2751 .worktree
2752 .join("target/debug/output")
2753 .exists()
2754 );
2755 assert!(
2756 second.candidates[0]
2757 .worktree
2758 .join("target/debug/output")
2759 .exists()
2760 );
2761 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2762 }
2763
2764 #[tokio::test]
2765 async fn sweep_budget_pressure_prioritizes_largest_over_oldest() {
2766 let f = Fx::new().await;
2767 let small = f.run("bb00", true).await;
2768 let large = f.run("bb01", true).await;
2769 for state in [&small, &large] {
2770 Fx::target(&state.candidates[0].worktree);
2771 }
2772 let output = large.candidates[0].worktree.join("target/debug/output");
2773 std::fs::OpenOptions::new()
2774 .write(true)
2775 .open(&output)
2776 .unwrap()
2777 .set_len(1000)
2778 .unwrap();
2779 let mut disk = Fx::disk();
2780 disk.worktree_reclaim_bytes = 300;
2781 disk.min_free_bytes = u64::MAX;
2782 disk.worktree_limit_bytes = u64::MAX;
2784 let out = sweep(&f.home, &disk, Fx::now()).await;
2785 assert_eq!(out.freed_bytes, 1000, "{out:?}");
2786 assert!(!output.exists());
2787 assert!(
2788 small.candidates[0]
2789 .worktree
2790 .join("target/debug/output")
2791 .exists()
2792 );
2793 }
2794
2795 #[tokio::test]
2796 async fn sweep_budget_zero_remaining_and_inventory_are_distinct() {
2797 let f = Fx::new().await;
2798 let first = f.run("bc00", true).await;
2799 let second = f.run("bc01", true).await;
2800 for state in [&first, &second] {
2801 Fx::target(&state.candidates[0].worktree);
2802 }
2803 std::fs::create_dir_all(
2805 first.candidates[0]
2806 .worktree
2807 .join("target/debug/incremental"),
2808 )
2809 .unwrap();
2810 std::fs::write(
2811 first.candidates[0]
2812 .worktree
2813 .join("target/debug/incremental/cache"),
2814 [0; 1000],
2815 )
2816 .unwrap();
2817 let mut disk = Fx::disk();
2818 disk.worktree_reclaim_bytes = 0;
2819 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2820 disk.worktree_reclaim_bytes = 300;
2821 let out = sweep(&f.home, &disk, Fx::now()).await;
2822 assert_eq!(out.freed_bytes, 200, "{out:?}");
2823 assert!(
2824 !first.candidates[0]
2825 .worktree
2826 .join("target/debug/output")
2827 .exists()
2828 );
2829 assert!(
2830 second.candidates[0]
2831 .worktree
2832 .join("target/debug/output")
2833 .exists()
2834 );
2835 assert!(
2836 out.retained
2837 .iter()
2838 .any(|r| r.contains("declared outputs exceed"))
2839 );
2840 }
2841
2842 #[tokio::test]
2843 async fn sweep_budget_exception_preserves_grace_liveness_and_locks() {
2844 let f = Fx::new().await;
2845 let mut protected = f.run("bd00", true).await;
2846 let later = f.run("bd01", true).await;
2847 for state in [&protected, &later] {
2848 Fx::target(&state.candidates[0].worktree);
2849 }
2850 let mut disk = Fx::disk();
2851 disk.worktree_reclaim_bytes = 100;
2852 disk.fold_grace_secs = 3600;
2853 let out = sweep(&f.home, &disk, Fx::now()).await;
2854 assert_eq!(out.freed_bytes, 0);
2855 assert!(
2856 out.retained
2857 .iter()
2858 .any(|r| r.contains("within grace period"))
2859 );
2860 disk.fold_grace_secs = 0;
2861 protected.driver_exited = false;
2862 protected.driver_pid = Some(std::process::id());
2863 protected.driver_started_at = crate::proc::process_started_at(std::process::id());
2864 protected
2865 .save_under_at(&f.home, protected.updated_at)
2866 .unwrap();
2867 let out = sweep(&f.home, &disk, Fx::now()).await;
2868 assert!(
2869 out.retained
2870 .iter()
2871 .any(|r| r.contains("run live or ownership unknown"))
2872 );
2873 assert_eq!(
2874 out.freed_bytes, 200,
2875 "live target must not consume exception: {out:?}"
2876 );
2877 let output = protected.candidates[0].worktree.join("target/debug/output");
2878 assert!(output.exists());
2879 protected.driver_exited = true;
2880 protected.save_under(&f.home).unwrap();
2881 let owner = crate::pr_head::owner_lock(&f.home, &protected.id).unwrap();
2882 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2883 assert!(output.exists());
2884 drop(owner);
2885 let cargo = cargo_locks(&protected.candidates[0].worktree.join("target")).unwrap();
2886 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2887 assert!(output.exists());
2888 drop(cargo);
2889 let lease = OutputLease::take(&protected.candidates[0].worktree, true).unwrap();
2890 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2891 assert!(output.exists());
2892 drop(lease);
2893 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2894 assert!(!output.exists());
2895 }
2896
2897 #[tokio::test]
2898 async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
2899 for remove_root in [true, false] {
2900 let f = Fx::new().await;
2901 let stale = Fx::new().await;
2902 let mut missing = stale.run("aa00", false).await;
2903 Fx::target(&missing.candidates[0].worktree);
2904 missing.save_under(&f.home).unwrap();
2905 if remove_root {
2906 std::fs::remove_dir_all(missing.worktree_root()).unwrap();
2907 std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
2910 } else {
2911 std::fs::remove_dir_all(&stale.repo).unwrap();
2912 }
2913 let healthy = f.run("aa01", false).await;
2914 Fx::target(&healthy.candidates[0].worktree);
2915
2916 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2917 assert!(report.failures.is_empty(), "{report:?}");
2918 assert_eq!(report.freed_bytes, 200, "{report:?}");
2919 assert!(
2920 !healthy.candidates[0]
2921 .worktree
2922 .join("target/debug/output")
2923 .exists()
2924 );
2925 if !remove_root {
2926 assert!(
2928 missing.candidates[0]
2929 .worktree
2930 .join("target/debug/output")
2931 .exists()
2932 );
2933 }
2934 }
2935 }
2936
2937 #[tokio::test]
2938 async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
2939 let f = Fx::new().await;
2940 let mut missing = f.run("aa00", false).await;
2941 Fx::target(&missing.candidates[0].worktree);
2942 missing.repo = missing.candidates[0].worktree.clone();
2943 missing.save_under(&f.home).unwrap();
2944 std::fs::remove_dir_all(&missing.repo).unwrap();
2945 assert!(missing.worktree_root().is_dir());
2946 assert!(f.repo.is_dir());
2947
2948 let healthy = f.run("aa01", false).await;
2949 Fx::target(&healthy.candidates[0].worktree);
2950
2951 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2952 assert!(report.failures.is_empty(), "{report:?}");
2953 assert_eq!(report.freed_bytes, 200, "{report:?}");
2954 assert!(
2955 !healthy.candidates[0]
2956 .worktree
2957 .join("target/debug/output")
2958 .exists()
2959 );
2960 }
2961
2962 #[tokio::test]
2963 async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
2964 let f = Fx::new().await;
2965 let broken = Fx::new().await;
2966 let mut invalid = broken.run("aa00", false).await;
2967 Fx::target(&invalid.candidates[0].worktree);
2968 invalid.save_under(&f.home).unwrap();
2969 std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
2970 let healthy = f.run("aa01", false).await;
2971 Fx::target(&healthy.candidates[0].worktree);
2972
2973 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2974 assert_eq!(report.failures.len(), 1, "{report:?}");
2975 assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
2976 assert_eq!(report.freed_bytes, 200, "{report:?}");
2977 assert!(
2978 !healthy.candidates[0]
2979 .worktree
2980 .join("target/debug/output")
2981 .exists()
2982 );
2983 assert!(
2984 invalid.candidates[0]
2985 .worktree
2986 .join("target/debug/output")
2987 .exists()
2988 );
2989 }
2990
2991 async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
2992 let path = reconcile::run(&f.home, &task.id, &source.id, None)
2993 .await
2994 .unwrap();
2995 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
2996 }
2997
2998 #[tokio::test]
2999 async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
3000 use crate::{
3001 chat_control::{Operation, dispatch},
3002 talk::{self, Talks},
3003 };
3004 let f = Fx::new().await;
3005 let source = f.run("ae00", true).await;
3006 let clean = f.run("ae01", false).await;
3007 let queue = crate::queue::Queue::at(f.home.join("queue"));
3008 let talks = Talks::at(f.home.join("talks"));
3009 let open_talk = |id: &str| -> talk::Talk {
3010 let mut conversation = serde_json::from_value(serde_json::json!({
3011 "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
3012 "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
3013 "seat":crate::agent::SeatState::new("talk","worker",1)
3014 }))
3015 .unwrap();
3016 talks.put(&mut conversation).unwrap();
3017 conversation
3018 };
3019 let conversation = open_talk("talk-owner");
3020 let other = open_talk("talk-other");
3021 let task = f.task(&[source.clone(), clean.clone()]);
3023 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3024 let operation = |apply: Option<&str>| Operation::ReconcileRetention {
3025 task: task.id.clone(),
3026 authority: source.id.clone(),
3027 apply: apply.map(str::to_owned),
3028 };
3029 let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
3031 .await
3032 .unwrap();
3033 let legacy: serde_json::Value =
3034 serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
3035 let legacy_id = legacy["id"].as_str().unwrap();
3036 assert!(
3037 dispatch(
3038 &queue,
3039 &f.home,
3040 &talks,
3041 &conversation.id,
3042 operation(Some(legacy_id))
3043 )
3044 .await
3045 .is_err()
3046 );
3047 let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
3048 .await
3049 .unwrap();
3050 assert!(summary.contains("Complete review summary"));
3051 assert!(summary.contains(&source.id) && summary.contains(&clean.id));
3052 let root = f.home.join("retention-reconciliations");
3053 let displayed: serde_json::Value =
3054 serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
3055 let id = displayed["plan"].as_str().unwrap();
3056 let dir = root.join(id);
3057 let plan: serde_json::Value =
3058 serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
3059 assert_eq!(plan["version"], 2);
3060 assert!(clean.candidates[0].worktree.exists());
3061 assert!(!dir.join("approved.json").exists());
3062 assert!(
3064 dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
3065 .await
3066 .is_err()
3067 );
3068 assert!(!dir.join("approved.json").exists());
3069 let error = reconcile::interrupt_chat_after_fence(
3072 &f.home,
3073 &task,
3074 id,
3075 reconcile::ChatRequest {
3076 talks: &talks,
3077 conversation: &conversation.id,
3078 },
3079 )
3080 .await
3081 .unwrap_err();
3082 assert!(
3083 error.to_string().contains("simulated interruption"),
3084 "{error:#}"
3085 );
3086 assert!(clean.candidates[0].worktree.exists());
3087 assert!(dir.join("approved.json").exists());
3088 dispatch(
3089 &queue,
3090 &f.home,
3091 &talks,
3092 &conversation.id,
3093 operation(Some(id)),
3094 )
3095 .await
3096 .unwrap();
3097 assert!(!clean.candidates[0].worktree.exists());
3098 assert!(source.candidates[0].worktree.exists());
3099 let audit: serde_json::Value =
3100 serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
3101 assert_eq!(audit["version"], 2);
3102 assert_eq!(
3103 audit["owner"],
3104 serde_json::json!({"conversation": conversation.id})
3105 );
3106 dispatch(
3107 &queue,
3108 &f.home,
3109 &talks,
3110 &conversation.id,
3111 operation(Some(id)),
3112 )
3113 .await
3114 .unwrap();
3115 }
3116
3117 #[tokio::test]
3118 async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
3119 let f = Fx::new().await;
3120 let source = f.run("af00", true).await;
3121 let mut clean = f.run("af01", false).await;
3122 let mut dirty = f.run("af02", false).await;
3123 for state in [&mut clean, &mut dirty] {
3124 state.status = RunStatus::Prep;
3125 state.prep_complete = Some(false);
3126 state.candidates[0].empty = false;
3127 state.save_under(&f.home).unwrap();
3128 }
3129 std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
3130 let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
3131 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3132 let plan = reconciliation_plan(&f, &task, &source).await;
3133 let selected: Vec<_> = plan["candidates"]
3134 .as_array()
3135 .unwrap()
3136 .iter()
3137 .filter(|e| e["selected"] == true)
3138 .collect();
3139 assert_eq!(selected.len(), 1, "{plan:#}");
3140 assert_eq!(selected[0]["run"], clean.id);
3141 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3142 .await
3143 .unwrap();
3144 assert!(!clean.candidates[0].worktree.exists());
3145 assert!(dirty.candidates[0].worktree.exists());
3146 assert!(
3147 RunState::load_under(&dirty.id, &f.home)
3148 .unwrap()
3149 .retired_to
3150 .is_none()
3151 );
3152 }
3153
3154 #[tokio::test]
3155 async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
3156 let f = Fx::new().await;
3157 let mut source = f.run("aa00", true).await;
3158 let alternative = f.run("aa01", true).await;
3159 let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
3160 .await
3161 .unwrap();
3162 source.reviewed_commits = Some(vec![head.clone()]);
3163 source.save_under(&f.home).unwrap();
3164 let mut runs = vec![source.clone(), alternative.clone()];
3165 for i in 2..14 {
3166 runs.push(f.run(&format!("aa{i:02}"), false).await);
3167 }
3168 let mut task = f.task(&runs);
3169 task.pr_recovery = Some(crate::pr_recovery::Selection {
3170 url: "https://example.invalid/pull/1".into(),
3171 branch: "selected".into(),
3172 head: head.clone(),
3173 source_head: Some(head.clone()),
3174 source_run: Some(source.id.clone()),
3175 source_proof: None,
3176 disposition: Some(crate::pr_recovery::Disposition::Land),
3177 instruction: task.instruction.clone(),
3178 approved_at: Timestamp::now(),
3179 });
3180 source.pr_recovery = task.pr_recovery.clone();
3181 source.save_under(&f.home).unwrap();
3182 runs[0] = source.clone();
3183 crate::queue::Queue::at(f.home.join("queue"))
3184 .put(&mut task)
3185 .unwrap();
3186 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3187 let task_before = serde_json::to_value(&task).unwrap();
3188 let mut report = Report::default();
3189 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3190 assert_eq!(
3191 report.retired, 0,
3192 "automatic cleanup must still refuse blanket markers"
3193 );
3194 assert!(
3195 report
3196 .retained
3197 .iter()
3198 .any(|reason| reason.contains("explicitly retained"))
3199 );
3200 let mut total = 0;
3201 for expected in [5, 5, 2, 0] {
3202 let plan = reconciliation_plan(&f, &task, &source).await;
3203 let selected: Vec<_> = plan["candidates"]
3204 .as_array()
3205 .unwrap()
3206 .iter()
3207 .filter(|e| e["selected"] == true)
3208 .collect();
3209 assert_eq!(selected.len(), expected, "{plan:#}");
3210 for e in &selected {
3211 assert!(
3212 RunState::load_under(e["run"].as_str().unwrap(), &f.home)
3213 .unwrap()
3214 .retired_to
3215 .is_none()
3216 );
3217 }
3218 let operation = plan["id"].as_str().unwrap();
3219 for _ in 0..2 {
3220 reconcile::run(&f.home, &task.id, &source.id, Some(operation))
3221 .await
3222 .unwrap();
3223 }
3224 total += expected;
3225 }
3226 assert_eq!(total, 12);
3227 for run in &runs {
3228 let dir = f.home.join("runs").join(&run.id);
3229 assert!(dir.join("run.json").exists());
3230 assert!(dir.join("retain-recovery-sources").exists());
3231 assert!(
3232 git::branch_exists(&f.repo, &run.candidates[0].branch)
3233 .await
3234 .unwrap()
3235 );
3236 }
3237 assert!(source.candidates[0].worktree.exists());
3238 assert!(alternative.candidates[0].worktree.exists());
3239 assert_eq!(
3240 git::rev_parse(&f.repo, &source.candidates[0].branch)
3241 .await
3242 .unwrap(),
3243 head
3244 );
3245 assert_eq!(
3246 serde_json::to_value(
3247 crate::queue::Queue::at(f.home.join("queue"))
3248 .get(&task.id)
3249 .unwrap()
3250 )
3251 .unwrap(),
3252 task_before
3253 );
3254 assert_eq!(
3255 runs.iter()
3256 .filter(|r| RunState::load_under(&r.id, &f.home)
3257 .unwrap()
3258 .retired_to
3259 .is_none())
3260 .count(),
3261 2
3262 );
3263 }
3264
3265 #[tokio::test]
3266 async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
3267 let f = Fx::new().await;
3268 let source = f.run("ab00", true).await;
3269 let mut runs = vec![source.clone()];
3270 for i in 1..9 {
3271 runs.push(f.run(&format!("ab{i:02}"), false).await);
3272 }
3273 std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
3274 std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
3275 std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
3276 std::fs::write(
3277 runs[3].candidates[0].worktree.join("scratch/source"),
3278 "ignored source",
3279 )
3280 .unwrap();
3281 std::fs::write(
3282 runs[4].worktree_root().join("unknown"),
3283 "unregistered source",
3284 )
3285 .unwrap();
3286 let checkpoint = git::git(
3287 &runs[5].candidates[0].worktree,
3288 &[
3289 "rev-parse",
3290 "--path-format=absolute",
3291 "--git-path",
3292 "sequencer",
3293 ],
3294 )
3295 .await
3296 .unwrap();
3297 std::fs::create_dir(checkpoint.trim()).unwrap();
3298 runs[6].driver_exited = false;
3299 runs[6].driver_pid = Some(std::process::id());
3300 runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
3301 runs[6].save_under(&f.home).unwrap();
3302 runs[7].driver_exited = false;
3303 runs[7].driver_pid = None;
3304 runs[7].driver_claimed = None; runs[7].save_under(&f.home).unwrap();
3306 assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
3307
3308 let hidden = &runs[8].candidates[0].worktree;
3309 git::git(hidden, &["update-index", "--assume-unchanged", "source"])
3310 .await
3311 .unwrap();
3312 std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
3313 assert!(
3314 git::git(hidden, &["status", "--porcelain"])
3315 .await
3316 .unwrap()
3317 .is_empty()
3318 );
3319 let task = f.task(&runs);
3320 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3321 let plan = reconciliation_plan(&f, &task, &source).await;
3322 assert!(
3323 plan["candidates"]
3324 .as_array()
3325 .unwrap()
3326 .iter()
3327 .all(|e| e["selected"] == false),
3328 "{plan:#}"
3329 );
3330 for run in &runs {
3331 assert!(run.candidates[0].worktree.exists());
3332 }
3333 }
3334
3335 #[tokio::test]
3336 async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
3337 let f = Fx::new().await;
3338 let source = f.run("ac00", true).await;
3339 let empty = f.run("ac01", false).await;
3340 let task = f.task(&[source.clone(), empty.clone()]);
3341 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3342 let plan = reconciliation_plan(&f, &task, &source).await;
3343 std::fs::write(
3344 f.home.join("runs").join(&source.id).join("new-report"),
3345 "new review checkpoint",
3346 )
3347 .unwrap();
3348 assert!(
3349 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3350 .await
3351 .is_err()
3352 );
3353 assert!(empty.candidates[0].worktree.exists());
3354 assert!(
3355 RunState::load_under(&empty.id, &f.home)
3356 .unwrap()
3357 .retired_to
3358 .is_none()
3359 );
3360 let marker = f
3361 .home
3362 .join("runs")
3363 .join(&empty.id)
3364 .join("retain-recovery-sources");
3365 std::fs::write(&marker, "different owner").unwrap();
3366 let plan = reconciliation_plan(&f, &task, &source).await;
3367 assert!(
3368 plan["candidates"]
3369 .as_array()
3370 .unwrap()
3371 .iter()
3372 .all(|e| e["selected"] == false)
3373 );
3374 }
3375
3376 #[tokio::test]
3377 async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
3378 for partial in [false, true] {
3379 let f = Fx::new().await;
3380 let source = f.run("ab10", true).await;
3381 let first = f.run("ab11", false).await;
3382 let second = f.run("ab12", false).await;
3383 let task = f.task(&[source.clone(), first.clone(), second.clone()]);
3384 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3385 let plan = reconciliation_plan(&f, &task, &source).await;
3386 let id = plan["id"].as_str().unwrap();
3387 let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
3388 std::fs::write(&staging, b"").unwrap();
3390 assert!(
3391 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3392 .await
3393 .is_err()
3394 );
3395 assert!(staging.exists());
3396 std::fs::remove_file(&staging).unwrap();
3397 let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
3398 .await
3399 .unwrap_err();
3400 assert!(
3401 error.to_string().contains("before run state rename"),
3402 "{error:#}"
3403 );
3404 assert!(
3405 RunState::load_under(&first.id, &f.home)
3406 .unwrap()
3407 .retired_to
3408 .is_some()
3409 );
3410 assert!(
3411 RunState::load_under(&second.id, &f.home)
3412 .unwrap()
3413 .retired_to
3414 .is_none()
3415 );
3416 assert!(first.candidates[0].worktree.exists());
3417 assert!(second.candidates[0].worktree.exists());
3418 let interrupted = std::fs::read(&staging).unwrap();
3419 std::fs::write(&staging, b"unrelated recovery source").unwrap();
3421 assert!(
3422 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3423 .await
3424 .is_err()
3425 );
3426 assert_eq!(
3427 std::fs::read(&staging).unwrap(),
3428 b"unrelated recovery source"
3429 );
3430 std::fs::write(&staging, interrupted).unwrap();
3431 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3432 .await
3433 .unwrap();
3434 assert!(!staging.exists());
3435 assert!(!first.candidates[0].worktree.exists());
3436 assert!(!second.candidates[0].worktree.exists());
3437 assert_eq!(
3438 RunState::load_under(&second.id, &f.home)
3439 .unwrap()
3440 .retired_to,
3441 Some(source.id)
3442 );
3443 }
3444 }
3445
3446 #[tokio::test]
3447 async fn reconciliation_resumes_only_its_exact_durable_fence() {
3448 let f = Fx::new().await;
3449 let source = f.run("ad00", true).await;
3450 let empty = f.run("ad01", false).await;
3451 let task = f.task(&[source.clone(), empty.clone()]);
3452 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3453 let plan = reconciliation_plan(&f, &task, &source).await;
3454 let id = plan["id"].as_str().unwrap();
3455 assert!(
3456 reconcile::interrupt_after_fence(&f.home, &task, id)
3457 .await
3458 .is_err()
3459 );
3460 assert!(empty.candidates[0].worktree.exists());
3461 assert_eq!(
3462 RunState::load_under(&empty.id, &f.home)
3463 .unwrap()
3464 .retired_to
3465 .as_ref(),
3466 Some(&source.id)
3467 );
3468 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3469 .await
3470 .unwrap();
3471 assert!(!empty.candidates[0].worktree.exists());
3472 git::worktree_add_branch(
3474 &f.repo,
3475 &empty.candidates[0].worktree,
3476 "restore-empty",
3477 &f.base,
3478 )
3479 .await
3480 .unwrap();
3481 assert!(
3483 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3484 .await
3485 .is_err()
3486 );
3487 assert!(empty.candidates[0].worktree.exists());
3488 let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
3489 changed.prep_failure = Some("new owner edit".into());
3490 changed.save_under(&f.home).unwrap();
3491 assert!(
3492 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3493 .await
3494 .is_err()
3495 );
3496 }
3497
3498 #[tokio::test]
3499 async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
3500 let f = Fx::new().await;
3501 let source = f.run("ae00", true).await;
3502 let mut prep = f.run("ae01", false).await;
3503 let candidate = prep.candidates.pop().unwrap();
3504 prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
3505 prep.save_under(&f.home).unwrap();
3506 let reset = f.run("ae02", false).await;
3507 let wt = &reset.candidates[0].worktree;
3508 std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
3509 git::git(wt, &["add", "."]).await.unwrap();
3510 git::git(wt, &["commit", "-m", "unique implementation"])
3511 .await
3512 .unwrap();
3513 git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
3514 let unknown = f.run("ae03", false).await;
3515 let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
3516 let mut raw: serde_json::Value =
3517 serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
3518 raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
3519 std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
3520 let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
3521 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3522 let plan = reconciliation_plan(&f, &task, &source).await;
3523 let selected: Vec<_> = plan["candidates"]
3524 .as_array()
3525 .unwrap()
3526 .iter()
3527 .filter(|e| e["selected"] == true)
3528 .collect();
3529 assert_eq!(selected.len(), 1, "{plan:#}");
3530 assert_eq!(selected[0]["run"], prep.id);
3531 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3532 .await
3533 .unwrap();
3534 assert!(!candidate.worktree.exists());
3535 assert!(reset.candidates[0].worktree.exists());
3536 assert!(unknown.candidates[0].worktree.exists());
3537 }
3538
3539 #[tokio::test]
3540 async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
3541 let f = Fx::new().await;
3542 let mut state = f.run("aaaa", true).await;
3543 let candidate = state.candidates[0].worktree.clone();
3544 std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
3545 Fx::target(&candidate);
3546 for seat in ["review-1", "review-2"] {
3547 let wt = state.worktree_root().join(seat);
3548 git::worktree_add_detached(&f.repo, &wt, &f.base)
3549 .await
3550 .unwrap();
3551 Fx::target(&wt);
3552 }
3553 state.status = RunStatus::Landing;
3554 state.save_under(&f.home).unwrap();
3555 let second = Fx::new().await;
3556 let other = second.run("bbbb", true).await;
3557 let mut imported = other.clone();
3559 imported.save_under(&f.home).unwrap();
3560 Fx::target(&other.candidates[0].worktree);
3561 let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
3562 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3563 assert!(
3564 candidate.join("target/debug/output").exists(),
3565 "active writer protected"
3566 );
3567 assert!(
3568 !state
3569 .worktree_root()
3570 .join("review-1/target/debug/output")
3571 .exists()
3572 );
3573 assert!(
3574 !state
3575 .worktree_root()
3576 .join("review-2/target/debug/output")
3577 .exists()
3578 );
3579 assert!(
3580 !other.candidates[0]
3581 .worktree
3582 .join("target/debug/output")
3583 .exists()
3584 );
3585 assert!(out.freed_bytes >= 600);
3586 drop(guard);
3587 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3588 assert!(out.freed_bytes >= 200);
3589 assert_eq!(
3590 std::fs::read_to_string(candidate.join("source")).unwrap(),
3591 "uncommitted work"
3592 );
3593 assert!(
3594 git::branch_exists(&f.repo, &state.candidates[0].branch)
3595 .await
3596 .unwrap()
3597 );
3598 }
3599
3600 #[tokio::test]
3601 async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
3602 let f = Fx::new().await;
3603 let state = f.run("cc01", false).await;
3604 let wt = &state.candidates[0].worktree;
3605 Fx::target(wt);
3606 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3607 let dir = wt.join("target").join(dir);
3608 std::fs::create_dir_all(&dir).unwrap();
3609 std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
3610 }
3611 let manifest = wt.join("target/debug/output.d");
3613 let output = wt.join("target/debug/output");
3614 std::fs::write(
3615 &manifest,
3616 format!(
3617 "{}: {}\n",
3618 output.display(),
3619 wt.join("target/debug/notes.txt").display()
3620 ),
3621 )
3622 .unwrap();
3623 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3624 assert_eq!(report.freed_bytes, 200);
3625 assert!(!output.exists());
3626 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3627 assert_eq!(
3628 std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
3629 "user evidence"
3630 );
3631 }
3632 assert!(manifest.exists());
3634 assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
3635 std::fs::write(&output, "user note replacing an old binary").unwrap();
3636 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3637 assert_eq!(report.freed_bytes, 0);
3638 assert_eq!(
3639 std::fs::read_to_string(&output).unwrap(),
3640 "user note replacing an old binary"
3641 );
3642 }
3643
3644 #[tokio::test]
3645 async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
3646 let f = Fx::new().await;
3647 let state = f.run("cc02", false).await;
3648 let wt = &state.candidates[0].worktree;
3649 Fx::target(wt);
3650 let mut disk = Fx::disk();
3651 disk.worktree_limit_bytes = u64::MAX;
3652 let now = Fx::now();
3653 let first = sweep(&f.home, &disk, now).await;
3654 std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
3655 assert_eq!(
3656 sweep(&f.home, &disk, now).await.measured_bytes,
3657 first.measured_bytes
3658 );
3659 assert!(
3660 sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
3661 .await
3662 .measured_bytes
3663 > first.measured_bytes
3664 );
3665 disk.worktree_limit_bytes = 0;
3666 assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
3667 }
3668
3669 #[tokio::test]
3670 async fn unfinished_source_fold_reports_its_retention_reason() {
3671 let f = Fx::new().await;
3672 let mut source = f.run("cc03", true).await;
3673 let task = f.task(&[source.clone()]);
3674 let error = crate::graph::fold_run(&mut source, true, &f.home)
3675 .await
3676 .unwrap_err();
3677 assert!(error.to_string().contains(&task.id));
3678 assert!(source.candidates[0].worktree.exists());
3679 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3680 let error = crate::graph::fold_run(&mut source, true, &f.home)
3681 .await
3682 .unwrap_err();
3683 assert!(error.to_string().contains("explicitly retained"));
3684 }
3685
3686 #[tokio::test]
3687 async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
3688 let f = Fx::new().await;
3689 let mut source = f.run("cc04", true).await;
3690 let mut task = f.task(&[source.clone()]);
3691 task.hold_manual(Some("queue settlement pending".into()));
3692 let queue = crate::queue::Queue::at(f.home.join("queue"));
3693 queue.put(&mut task).unwrap();
3694 source.merge = Some(crate::run::MergeOutcome {
3695 mode: crate::config::MergeMode::Pr,
3696 ok: true,
3697 detail: "confirmed merged".into(),
3698 empty: false,
3699 });
3700 for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
3702 source.status = status;
3703 assert!(
3704 crate::graph::fold_run(&mut source, true, &f.home)
3705 .await
3706 .is_err()
3707 );
3708 assert!(source.candidates[0].worktree.exists());
3709 }
3710 git::git(
3711 &f.repo,
3712 &["merge", "--ff-only", &source.candidates[0].branch],
3713 )
3714 .await
3715 .unwrap();
3716 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3717 .await
3718 .unwrap();
3719 source.status = RunStatus::Merged;
3720 source.save_under(&f.home).unwrap();
3721 assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
3722 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3723 assert!(
3724 crate::graph::fold_run(&mut source, true, &f.home)
3725 .await
3726 .unwrap_err()
3727 .to_string()
3728 .contains("explicitly retained")
3729 );
3730 std::fs::remove_file(
3733 f.home
3734 .join("runs")
3735 .join(&source.id)
3736 .join("retain-recovery-sources"),
3737 )
3738 .unwrap();
3739 let (folded, unreadable) = crate::clean::fold_due(
3740 &f.home.join("runs"),
3741 &f.home,
3742 &f.bay,
3743 &Fx::disk(),
3744 Fx::now(),
3745 )
3746 .await
3747 .unwrap();
3748 assert_eq!((folded, unreadable), (1, 0));
3749 assert!(!source.candidates[0].worktree.exists());
3750 let evidence = f
3751 .home
3752 .join("runs")
3753 .join(&source.id)
3754 .join("artifacts/fold-evidence");
3755 let names: Vec<_> = std::fs::read_dir(evidence)
3756 .unwrap()
3757 .map(|entry| entry.unwrap().file_name())
3758 .collect();
3759 assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
3760 assert_eq!(
3761 queue.get(&task.id).unwrap().status,
3762 crate::queue::TaskStatus::Held
3763 );
3764 }
3765
3766 #[tokio::test]
3767 async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
3768 for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
3769 let f = Fx::new().await;
3770 let mut state = f.run("cc06", true).await;
3771 let candidate = state.candidates[0].clone();
3772 let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3773 std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
3774 git::rescue_commit(&candidate.worktree, "second candidate change")
3775 .await
3776 .unwrap();
3777 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3778 state.tally = Some(crate::run::Tally {
3779 first_choice: Default::default(),
3780 borda: Default::default(),
3781 winner: 'A',
3782 rankings: 0,
3783 unanimous_initial: true,
3784 deliberated: false,
3785 changed_votes: 0,
3786 unanimous_final: true,
3787 tie_break: None,
3788 judges: 0,
3789 present: 0,
3790 quorum: 0,
3791 met_quorum: true,
3792 uncontested: Some("fixture".into()),
3793 });
3794 let reviewer = state.worktree_root().join("review-1");
3797 git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
3798 .await
3799 .unwrap();
3800 std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
3803 git::rescue_commit(&f.repo, "advance before squash")
3804 .await
3805 .unwrap();
3806 let published = if variant == "partial" {
3807 &intermediate
3808 } else {
3809 &head
3810 };
3811 git::git(&f.repo, &["merge", "--squash", published])
3812 .await
3813 .unwrap();
3814 git::git(&f.repo, &["commit", "-m", "squash candidate"])
3815 .await
3816 .unwrap();
3817 let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
3818 assert!(
3819 !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
3820 .await
3821 .unwrap()
3822 .ok()
3823 );
3824 std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
3825 git::rescue_commit(&f.repo, "advance after squash")
3826 .await
3827 .unwrap();
3828 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3829 .await
3830 .unwrap();
3831 if matches!(variant, "checkpoint" | "reviewer") {
3832 std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
3834 git::rescue_commit(&reviewer, "unpublished checkpoint")
3835 .await
3836 .unwrap();
3837 if variant == "checkpoint" {
3838 git::git(&reviewer, &["reset", "--hard", &intermediate])
3839 .await
3840 .unwrap();
3841 }
3842 }
3843 if variant == "loser" {
3844 let mut loser = candidate.clone();
3845 loser.label = 'B';
3846 loser.branch = "magi/cc06/B".into();
3847 loser.worktree = state.worktree_root().join("cand-B");
3848 git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
3849 .await
3850 .unwrap();
3851 std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
3852 git::rescue_commit(&loser.worktree, "independent loser")
3853 .await
3854 .unwrap();
3855 state.candidates.push(loser);
3856 }
3857 state.status = RunStatus::Merged;
3858 state.merge = Some(crate::run::MergeOutcome {
3859 mode: crate::config::MergeMode::Pr,
3860 ok: true,
3861 detail: "candidate A squash merged".into(),
3862 empty: false,
3863 });
3864 state.save_under(&f.home).unwrap();
3865 let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
3866 let result = crate::graph::fold_run(&mut state, true, &f.home).await;
3867 result.unwrap();
3868 assert!(
3869 !candidate.worktree.exists() && !reviewer.exists(),
3870 "{variant}"
3871 );
3872 assert!(
3873 !git::branch_exists(&f.repo, &candidate.branch)
3874 .await
3875 .unwrap()
3876 );
3877 let artifacts = f
3878 .home
3879 .join("runs")
3880 .join(&state.id)
3881 .join("artifacts/fold-evidence");
3882 assert_eq!(
3883 artifacts.join(format!("{head}.patch")).exists(),
3884 variant == "partial",
3885 "only a partially published winner needs evidence: {variant}"
3886 );
3887 assert_eq!(
3888 artifacts.join(format!("{intermediate}.patch")).exists(),
3889 variant == "partial",
3890 "squash proof also covers intermediate reviewer tips: {variant}"
3891 );
3892 let patches: Vec<_> = std::fs::read_dir(&artifacts)
3893 .unwrap()
3894 .map(|entry| entry.unwrap().path())
3895 .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
3896 .collect();
3897 let expected = match variant {
3898 "partial" => 2,
3899 "loser" | "reviewer" => 1,
3900 _ => 0,
3901 };
3902 assert_eq!(patches.len(), expected, "{variant}");
3903 if variant == "loser" {
3904 assert!(
3905 std::fs::read_to_string(&patches[0])
3906 .unwrap()
3907 .contains("losing candidate")
3908 );
3909 }
3910 assert!(artifacts.join("manifest.json").exists());
3911 if variant == "reviewer" {
3912 assert!(
3913 std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
3914 .unwrap()
3915 .contains("reviewer checkpoint")
3916 );
3917 }
3918 crate::graph::fold_run(&mut state, true, &f.home)
3919 .await
3920 .unwrap();
3921 }
3922 }
3923
3924 #[tokio::test]
3925 async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
3926 for missing in [true, false] {
3927 let f = Fx::new().await;
3928 let mut state = f.run("cc05", true).await;
3929 git::git(
3932 &f.repo,
3933 &["update-ref", "refs/remotes/origin/main", &f.base],
3934 )
3935 .await
3936 .unwrap();
3937 let candidate = state.candidates[0].clone();
3938 if !missing {
3939 std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
3940 git::rescue_commit(&f.repo, "advance base before rebase")
3941 .await
3942 .unwrap();
3943 git::git(&candidate.worktree, &["rebase", "main"])
3944 .await
3945 .unwrap();
3946 git::sync_to_head(&candidate.worktree).await.unwrap();
3949 }
3950 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3951 state.reviewed_commits = Some(Vec::new());
3953 state.candidates[0].worktree = state.worktree_root().join("under-review");
3954 git::git(
3955 &f.repo,
3956 &[
3957 "worktree",
3958 "move",
3959 candidate.worktree.to_str().unwrap(),
3960 state.candidates[0].worktree.to_str().unwrap(),
3961 ],
3962 )
3963 .await
3964 .unwrap();
3965 let reviewer = state.worktree_root().join("review-1");
3966 git::worktree_add_detached(&f.repo, &reviewer, &head)
3967 .await
3968 .unwrap();
3969 std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
3972 git::rescue_commit(&reviewer, "old review history")
3973 .await
3974 .unwrap();
3975 git::git(&reviewer, &["reset", "--hard", &head])
3976 .await
3977 .unwrap();
3978 if missing {
3979 std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
3980 }
3981 let run_dir = f.home.join("runs").join(&state.id);
3982 let artifacts = run_dir.join("artifacts");
3983 std::fs::create_dir_all(&artifacts).unwrap();
3984 std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
3985 if missing {
3986 std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
3987 }
3988 state.status = RunStatus::Merged;
3989 state.merge = Some(crate::run::MergeOutcome {
3990 mode: crate::config::MergeMode::Pr,
3991 ok: true,
3992 detail: "confirmed squash merge".into(),
3993 empty: false,
3994 });
3995 state.save_under(&f.home).unwrap();
3996 let owned = state.candidates[0].worktree.clone();
3998 state.candidates[0].worktree = f.home.join("missing-external");
3999 assert!(
4000 crate::graph::fold_run(&mut state, true, &f.home)
4001 .await
4002 .unwrap_err()
4003 .to_string()
4004 .contains("outside the inventoried")
4005 );
4006 state.candidates[0].worktree = owned;
4007 let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
4009 assert!(
4010 crate::graph::fold_run(&mut state, true, &f.home)
4011 .await
4012 .is_err()
4013 );
4014 assert!(reviewer.exists());
4015 drop(guard);
4016 let evidence = artifacts.join("fold-evidence");
4018 std::fs::write(&evidence, "blocked").unwrap();
4019 assert!(
4020 crate::graph::fold_run(&mut state, true, &f.home)
4021 .await
4022 .is_err()
4023 );
4024 assert!(reviewer.exists());
4025 std::fs::remove_file(&evidence).unwrap();
4026 std::fs::create_dir_all(&evidence).unwrap();
4027 let oversized = evidence.join("capacity-fixture");
4028 std::fs::File::create(&oversized)
4029 .unwrap()
4030 .set_len(64 * 1024 * 1024)
4031 .unwrap();
4032 assert!(
4033 crate::graph::fold_run(&mut state, true, &f.home)
4034 .await
4035 .is_err()
4036 );
4037 assert!(reviewer.exists());
4038 std::fs::remove_file(oversized).unwrap();
4039 std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
4042 std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
4043 assert!(
4044 crate::graph::fold_run(&mut state, true, &f.home)
4045 .await
4046 .is_err()
4047 );
4048 std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
4049 let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
4050 .await
4051 .unwrap();
4052 std::fs::write(&checkpoint, &head).unwrap();
4053 assert!(
4054 crate::graph::fold_run(&mut state, true, &f.home)
4055 .await
4056 .is_err()
4057 );
4058 std::fs::remove_file(checkpoint).unwrap();
4059 let (folded, unreadable) = crate::clean::fold_due(
4060 &f.home.join("runs"),
4061 &f.home,
4062 &f.bay,
4063 &Fx::disk(),
4064 Fx::now(),
4065 )
4066 .await
4067 .unwrap();
4068 if folded != 1 {
4069 panic!(
4070 "normal cleanup failed: {:#}",
4071 crate::graph::fold_run(&mut state, true, &f.home)
4072 .await
4073 .unwrap_err()
4074 );
4075 }
4076 assert_eq!((folded, unreadable), (1, 0));
4077 assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
4078 state = RunState::load_under(&state.id, &f.home).unwrap();
4079 assert!(state.candidates[0].folded);
4080 let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
4081 crate::graph::fold_run(&mut state, true, &f.home)
4082 .await
4083 .unwrap();
4084 assert_eq!(
4085 std::fs::read(evidence.join("manifest.json")).unwrap(),
4086 manifest
4087 );
4088 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
4089 .await
4090 .unwrap();
4091 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
4092 assert!(
4093 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
4094 .await
4095 .unwrap()
4096 .ok()
4097 );
4098 let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
4101 let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
4102 .as_str()
4103 .unwrap();
4104 let recovered = f.home.join("recovered");
4105 std::fs::create_dir_all(&recovered).unwrap();
4106 git::git(&recovered, &["init"]).await.unwrap();
4107 std::fs::write(
4108 recovered.join("source"),
4109 std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
4110 )
4111 .unwrap();
4112 git::git(
4113 &recovered,
4114 &[
4115 "apply",
4116 evidence.join(format!("{head}.patch")).to_str().unwrap(),
4117 ],
4118 )
4119 .await
4120 .unwrap();
4121 assert_eq!(
4122 std::fs::read_to_string(recovered.join("source")).unwrap(),
4123 "cc05"
4124 );
4125 assert!(
4126 !std::fs::read(evidence.join(format!("{head}.patch")))
4127 .unwrap()
4128 .is_empty()
4129 );
4130 assert_eq!(
4131 std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
4132 "review findings"
4133 );
4134 if missing {
4135 assert_eq!(
4136 std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
4137 "original snapshot"
4138 );
4139 } else {
4140 assert!(!artifacts.join("cand-A.patch").exists());
4141 }
4142 assert!(run_dir.join("run.json").exists());
4143 }
4144 }
4145
4146 #[tokio::test]
4147 async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
4148 use std::os::unix::fs::PermissionsExt as _;
4149 let f = Fx::new().await;
4150 let source = f.run("dd03", true).await;
4151 let empty = f.run("dd04", false).await;
4152 let task = f.task(&[source, empty.clone()]);
4153 let entered = f.repo.join("proof-entered");
4154 let release = f.repo.join("proof-release");
4155 let hook = f.repo.join("proof-monitor");
4156 std::fs::write(&hook, format!(
4157 "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
4158 entered.display(), release.display())).unwrap();
4159 std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
4160 git::git(
4161 &f.repo,
4162 &["config", "core.fsmonitor", hook.to_str().unwrap()],
4163 )
4164 .await
4165 .unwrap();
4166 struct ReleaseProof(PathBuf);
4167 impl Drop for ReleaseProof {
4168 fn drop(&mut self) {
4169 let _ = std::fs::write(&self.0, "continue");
4170 }
4171 }
4172 let _release_on_panic = ReleaseProof(release.clone());
4173 let home = f.home.clone();
4174 let sweep = tokio::spawn(async move {
4175 let mut report = Report::default();
4176 retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
4177 report
4178 });
4179 let watchdog = std::time::Duration::from_secs(60);
4182 tokio::time::timeout(watchdog, async {
4183 while !entered.exists() {
4184 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4185 }
4186 })
4187 .await
4188 .expect("Git retirement proof did not enter the blocking hook");
4189 let queue = crate::queue::Queue::at(f.home.join("queue"));
4190 let mut selected = queue.get(&task.id).unwrap();
4191 selected.resume_override = Some(crate::queue::OperatorResume {
4192 question_id: "during-slow-proof".into(),
4193 at: Fx::now(),
4194 conductor_rehold: None,
4195 forced: false,
4196 pinned_run: Some(empty.id.clone()),
4197 });
4198 let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
4199 let updated_while_blocked = match tokio::time::timeout(watchdog, &mut update).await {
4203 Ok(result) => {
4204 result.unwrap().unwrap();
4205 true
4206 }
4207 Err(_) => {
4208 std::fs::write(&release, "continue").unwrap();
4209 update.await.unwrap().unwrap();
4210 false
4211 }
4212 };
4213 std::fs::write(release, "continue").unwrap();
4214 let report = tokio::time::timeout(watchdog, sweep)
4215 .await
4216 .expect("Git retirement proof did not finish after release")
4217 .unwrap();
4218 assert!(
4219 updated_while_blocked,
4220 "task writers must not wait behind Git retirement proof"
4221 );
4222 assert_eq!(report.retired, 0);
4223 assert!(report.retained.iter().any(|r| r.contains("task changed")));
4224 assert!(empty.candidates[0].worktree.exists());
4225 assert!(
4226 RunState::load_under(&empty.id, &f.home)
4227 .unwrap()
4228 .retired_to
4229 .is_none()
4230 );
4231 }
4232
4233 #[tokio::test]
4234 async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
4235 let f = Fx::new().await;
4236 let source = f.run("dd01", true).await;
4237 let mut empty = f.run("dd02", false).await;
4238 let expected = f.task(&[source.clone(), empty.clone()]);
4239 let queue = crate::queue::Queue::at(f.home.join("queue"));
4240 let _claim = queue.claim(&expected.id).unwrap();
4241 let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
4243 let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
4244 let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
4245 let mut selected = queue.get(&expected.id).unwrap();
4246 selected.resume_override = Some(crate::queue::OperatorResume {
4247 question_id: "new-owner-decision".into(),
4248 at: Fx::now(),
4249 conductor_rehold: None,
4250 forced: false,
4251 pinned_run: Some(empty.id.clone()),
4252 });
4253 queue.put(&mut selected).unwrap();
4254 let error = commit_retirement(
4255 &f.home,
4256 &queue,
4257 &expected,
4258 &mut empty,
4259 &source.id,
4260 &expected.runs,
4261 Fx::now(),
4262 )
4263 .unwrap_err();
4264 assert!(error.to_string().contains("task changed"));
4265 assert!(
4266 RunState::load_under(&empty.id, &f.home)
4267 .unwrap()
4268 .retired_to
4269 .is_none()
4270 );
4271 assert!(empty.candidates[0].worktree.exists());
4272 assert_eq!(
4273 queue
4274 .get(&expected.id)
4275 .unwrap()
4276 .resume_override
4277 .unwrap()
4278 .pinned_run,
4279 Some(empty.id.clone())
4280 );
4281 assert!(
4282 !f.home
4283 .join("recovery-lineages")
4284 .join(format!("{}.json", expected.id))
4285 .exists()
4286 );
4287 assert!(
4289 commit_retirement(
4290 &f.home,
4291 &queue,
4292 &selected,
4293 &mut empty,
4294 &source.id,
4295 &selected.runs,
4296 Fx::now()
4297 )
4298 .unwrap_err()
4299 .to_string()
4300 .contains("selected this recovery source")
4301 );
4302 }
4303
4304 #[tokio::test]
4305 async fn cargo_owners_and_driver_resumes_are_protected_with_retained_root_data() {
4306 let f = Fx::new().await;
4307 let state = f.run("cccc", false).await;
4308 let wt = &state.candidates[0].worktree;
4309 Fx::target(wt);
4310 std::fs::write(wt.join("target/notes"), "user file").unwrap();
4311 std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
4312 assert!(
4313 !declared_outputs(&wt.join("target"))
4314 .unwrap()
4315 .contains(&wt.join("target/debug/link"))
4316 );
4317 std::fs::remove_file(wt.join("target/debug/link")).unwrap();
4318 let cargo = cargo_locks(&wt.join("target")).unwrap();
4319 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4320 assert_eq!(out.freed_bytes, 0);
4321 assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
4322 drop(cargo);
4323 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
4324 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4325 assert_eq!(out.freed_bytes, 0);
4326 drop(driver);
4327 let exclusive = OutputLease::take(wt, true).unwrap();
4328 assert!(
4329 OutputLease::borrow(wt).is_err(),
4330 "resume cannot start a writer during cleanup"
4331 );
4332 drop(exclusive);
4333 assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
4334 assert_eq!(
4335 std::fs::read(wt.join("target/notes")).unwrap(),
4336 b"user file"
4337 );
4338 }
4339
4340 #[tokio::test]
4341 async fn root_leftovers_allow_eviction_but_keep_empty_attempts_registered() {
4342 for leftover in ["tmp", "magi-tools", "lifecycle-client-check.log"] {
4343 let f = Fx::new().await;
4344 let source = f.run("aa10", true).await;
4345 let empty = f.run("aa11", false).await;
4346 f.task(&[source, empty.clone()]);
4347 let wt = &empty.candidates[0].worktree;
4348 Fx::target(wt);
4349 let path = wt.join("target").join(leftover);
4350 if leftover.ends_with(".log") {
4351 std::fs::write(&path, "keep log\n").unwrap();
4352 } else {
4353 std::fs::create_dir(&path).unwrap();
4355 }
4356 let mut disk = Fx::disk();
4357 disk.auto_fold = true;
4358 let report = sweep(&f.home, &disk, Fx::now()).await;
4359 assert_eq!(report.freed_bytes, 200, "{report:?}");
4360 assert_eq!(report.retired, 0, "{report:?}");
4361 assert!(!wt.join("target/debug/output").exists());
4362 assert!(!residue_proof(&f.home, &empty.id, wt).unwrap().exists());
4363 assert!(path.exists());
4364 if leftover.ends_with(".log") {
4365 assert_eq!(std::fs::read(&path).unwrap(), b"keep log\n");
4366 }
4367 assert_eq!(std::fs::read(wt.join("source")).unwrap(), b"base\n");
4368 assert!(worktrees(&empty).await.unwrap().contains(wt));
4369 assert!(
4370 git::branch_exists(&f.repo, &empty.candidates[0].branch)
4371 .await
4372 .unwrap()
4373 );
4374 assert!(
4375 RunState::load_under(&empty.id, &f.home)
4376 .unwrap()
4377 .retired_to
4378 .is_none()
4379 );
4380 }
4381 }
4382
4383 #[tokio::test]
4384 async fn residue_proof_rejects_new_unknown_root_entries_including_empty_directories() {
4385 let f = Fx::new().await;
4386 let state = f.run("aa12", false).await;
4387 let wt = &state.candidates[0].worktree;
4388 Fx::target(wt);
4389 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4390 assert_eq!(report.freed_bytes, 200, "{report:?}");
4391 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4392 for name in ["tmp", "magi-tools", "log"] {
4393 let path = wt.join("target").join(name);
4394 std::fs::create_dir(&path).unwrap();
4395 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
4396 std::fs::remove_dir(path).unwrap();
4397 }
4398 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4399 }
4400
4401 #[tokio::test]
4402 async fn target_identity_and_known_profiles_still_fail_closed_with_root_leftovers() {
4403 for invalid in [
4404 "tag",
4405 "identity",
4406 "tracked",
4407 "profile-file",
4408 "profile-symlink",
4409 "identity-symlink",
4410 ] {
4411 let f = Fx::new().await;
4412 let state = f.run("aa13", false).await;
4413 let wt = &state.candidates[0].worktree;
4414 Fx::target(wt);
4415 std::fs::create_dir(wt.join("target/tmp")).unwrap();
4416 let target = wt.join("target");
4417 match invalid {
4418 "tag" => std::fs::write(target.join("CACHEDIR.TAG"), "not Cargo").unwrap(),
4419 "identity" => std::fs::write(target.join(".rustc_info.json"), "{}").unwrap(),
4420 "tracked" => {
4421 git::git(wt, &["add", "-f", "target/debug/output"])
4422 .await
4423 .unwrap();
4424 }
4425 "profile-file" => {
4426 std::fs::write(target.join("release"), "keep").unwrap();
4427 }
4428 "profile-symlink" => {
4429 std::os::unix::fs::symlink(target.join("debug"), target.join("release"))
4430 .unwrap();
4431 }
4432 "identity-symlink" => {
4433 std::fs::rename(target.join(".rustc_info.json"), target.join("saved-info"))
4434 .unwrap();
4435 std::os::unix::fs::symlink(
4436 target.join("saved-info"),
4437 target.join(".rustc_info.json"),
4438 )
4439 .unwrap();
4440 }
4441 _ => unreachable!(),
4442 }
4443 let before = std::fs::read(target.join("debug/output")).unwrap();
4444 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4445 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
4446 assert_eq!(std::fs::read(target.join("debug/output")).unwrap(), before);
4447 assert!(target.join("tmp").is_dir());
4448 }
4449 }
4450
4451 #[tokio::test]
4452 async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
4453 let f = Fx::new().await;
4454 let source = f.run("0000", true).await;
4455 let empty = f.run("0001", false).await;
4456 let changed = f.run("0002", false).await;
4457 let unknown = f.run("0003", false).await;
4458 let unproven = f.run("0004", false).await;
4459 f.task(&[
4460 source.clone(),
4461 empty.clone(),
4462 changed.clone(),
4463 unknown.clone(),
4464 unproven.clone(),
4465 ]);
4466 for run in [&empty, &changed, &unknown] {
4467 Fx::target(&run.candidates[0].worktree);
4468 }
4469 let reviewer = empty.worktree_root().join("review-1");
4470 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4471 .await
4472 .unwrap();
4473 Fx::target(&reviewer);
4474 let mut disk = Fx::disk();
4475 disk.auto_fold = false;
4476 let report = sweep(&f.home, &disk, Fx::now()).await;
4477 assert_eq!(report.freed_bytes, 800, "{report:?}");
4478 let wt = &empty.candidates[0].worktree;
4479 assert!(wt.join("target/debug/output.d").exists());
4480 assert!(!wt.join("target/debug/output").exists());
4481 let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
4482 let proof: CargoResidue =
4483 serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
4484 assert_eq!(
4485 proof.files[Path::new("debug/output.d")],
4486 std::fs::read(wt.join("target/debug/output.d")).unwrap()
4487 );
4488 std::fs::write(
4489 changed.candidates[0].worktree.join("target/debug/output.d"),
4490 "owner notes",
4491 )
4492 .unwrap();
4493 std::fs::write(
4494 unknown.candidates[0].worktree.join("target/debug/notes"),
4495 "keep",
4496 )
4497 .unwrap();
4498 Fx::target(&unproven.candidates[0].worktree);
4499 std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
4500 disk.auto_fold = true;
4501 let locks = cargo_locks(&wt.join("target")).unwrap();
4502 let mut report = Report::default();
4503 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4504 assert_eq!(report.retired, 0, "{report:?}");
4505 assert!(
4506 report
4507 .retained
4508 .iter()
4509 .any(|reason| reason.contains("Cargo build"))
4510 );
4511 drop(locks);
4512 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4513 assert_eq!(report.retired, 1, "{report:?}");
4514 assert!(!wt.exists());
4515 assert!(!reviewer.exists());
4516 assert!(
4517 residue_proof(&f.home, &empty.id, &reviewer)
4518 .unwrap()
4519 .exists()
4520 );
4521 assert!(
4522 proof_path.exists(),
4523 "metadata evidence outlives the worktree"
4524 );
4525 assert_eq!(
4526 RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
4527 Some(source.id)
4528 );
4529 for run in [&changed, &unknown, &unproven] {
4530 assert!(run.candidates[0].worktree.exists());
4531 assert!(
4532 RunState::load_under(&run.id, &f.home)
4533 .unwrap()
4534 .retired_to
4535 .is_none()
4536 );
4537 }
4538 }
4539
4540 #[tokio::test]
4541 async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
4542 let f = Fx::new().await;
4543 let source = f.run("0000", true).await;
4544 let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
4545 .await
4546 .unwrap();
4547 let mut runs = vec![source.clone()];
4548 for n in 1..=12 {
4549 runs.push(f.run(&format!("{n:04}"), false).await);
4550 }
4551 let dirty = runs[1].candidates[0].worktree.clone();
4552 std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
4553 let ignored = runs[2].candidates[0].worktree.clone();
4554 std::fs::create_dir(ignored.join("scratch")).unwrap();
4555 std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
4556 runs[1].candidates.clear();
4559 runs[1].save_under(&f.home).unwrap();
4560 let task = f.task(&runs);
4561 let mut total = 0;
4562 for _ in 0..4 {
4563 let mut report = Report::default();
4564 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4565 assert!(report.retired <= 5);
4566 total += report.retired;
4567 }
4568 assert_eq!(total, 10);
4569 assert!(dirty.exists());
4570 assert!(ignored.exists());
4571 assert_eq!(
4572 git::rev_parse(&f.repo, &source.candidates[0].branch)
4573 .await
4574 .unwrap(),
4575 source_head
4576 );
4577 let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
4578 assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
4579 assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
4580 let mut released = task.clone();
4581 released.release();
4582 assert_eq!(
4583 crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
4584 RunState::load_under(id, &f.home)
4585 }),
4586 Some(source.id.clone())
4587 );
4588
4589 assert!(
4590 f.home
4591 .join("runs")
4592 .join(&retired.id)
4593 .join("run.json")
4594 .exists()
4595 );
4596 assert!(
4597 f.home
4598 .join("recovery-lineages")
4599 .join(format!("{}.json", task.id))
4600 .exists()
4601 );
4602 }
4603
4604 #[tokio::test]
4605 async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
4606 let f = Fx::new().await;
4607 let a = f.run("dddd", true).await;
4608 let b = f.run("eeee", true).await;
4609 let empty = f.run("ffff", false).await;
4610 let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
4611 assert!(
4612 crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
4613 "unfinished queue history protects older source branches from ordinary folding"
4614 );
4615 let mut prior = a.clone();
4616 assert!(
4617 crate::graph::fold_run(&mut prior, true, &f.home)
4618 .await
4619 .unwrap_err()
4620 .to_string()
4621 .contains(&task.id)
4622 );
4623 let mut report = Report::default();
4624 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4625 assert_eq!(report.retired, 0);
4626 assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
4627 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4628 Fx::target(&a.candidates[0].worktree);
4629 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4630 assert!(
4631 report.freed_bytes > 0,
4632 "source retention does not retain regenerable outputs"
4633 );
4634 assert!(a.candidates[0].worktree.exists());
4635 assert!(b.candidates[0].worktree.exists());
4636 assert!(empty.candidates[0].worktree.exists());
4637 }
4638
4639 #[tokio::test]
4640 async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
4641 let f = Fx::new().await;
4642 let authority = f.run("p000", true).await;
4643 let question_run = f.run("p001", false).await;
4644 let mut checkpoint = f.run("p002", false).await;
4645 checkpoint.land_approval = Some(crate::run::LandApproval {
4646 question: "pending".into(),
4647 head: f.base.clone(),
4648 });
4649 checkpoint.save_under(&f.home).unwrap();
4650 let retained = f.run("p003", false).await;
4651 std::fs::write(
4652 f.home
4653 .join("runs")
4654 .join(&retained.id)
4655 .join("retain-recovery-sources"),
4656 "owner",
4657 )
4658 .unwrap();
4659 let mut partial = f.run("p004", false).await;
4660 let reviewer = partial.worktree_root().join("review-1");
4661 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4662 .await
4663 .unwrap();
4664 partial.retired_to = Some(authority.id.clone());
4666 partial.save_under(&f.home).unwrap();
4667 assert!(
4668 git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
4669 .await
4670 .unwrap()
4671 );
4672 let git_checkpoint = f.run("p005", false).await;
4673 let location = git::git(
4674 &git_checkpoint.candidates[0].worktree,
4675 &[
4676 "rev-parse",
4677 "--path-format=absolute",
4678 "--git-path",
4679 "sequencer",
4680 ],
4681 )
4682 .await
4683 .unwrap();
4684 std::fs::create_dir(location.trim()).unwrap();
4685 let mut preservation = f.run("p006", false).await;
4686 preservation
4687 .candidate_preservation
4688 .insert('A', "pending preservation".into());
4689 preservation.save_under(&f.home).unwrap();
4690 let task = f.task(&[
4691 preservation.clone(),
4692 git_checkpoint.clone(),
4693 authority.clone(),
4694 question_run.clone(),
4695 checkpoint.clone(),
4696 retained.clone(),
4697 partial.clone(),
4698 ]);
4699 let questions = crate::ask::Questions::at(f.home.join("questions"));
4700 let mut question = crate::ask::Question::new(
4701 question_run.id.clone(),
4702 "implement".into(),
4703 "fixture".into(),
4704 "owner decision".into(),
4705 String::new(),
4706 vec!["keep".into()],
4707 );
4708 questions.put(&mut question).unwrap();
4709 let mut report = Report::default();
4710 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4711 assert_eq!(report.retired, 1, "{report:?}");
4712 assert!(!reviewer.exists());
4713 for state in [
4714 &question_run,
4715 &checkpoint,
4716 &retained,
4717 &git_checkpoint,
4718 &preservation,
4719 ] {
4720 assert!(state.candidates[0].worktree.exists());
4721 assert!(
4722 RunState::load_under(&state.id, &f.home)
4723 .unwrap()
4724 .retired_to
4725 .is_none()
4726 );
4727 }
4728 assert!(questions.get(&question.id).unwrap().status.open());
4729 assert_eq!(
4730 crate::queue::Queue::at(f.home.join("queue"))
4731 .get(&task.id)
4732 .unwrap()
4733 .status,
4734 task.status
4735 );
4736 }
4737
4738 #[tokio::test]
4739 async fn preservation_schema_migration_remains_distinct_from_retirement() {
4740 let f = Fx::new().await;
4741 let mut state = f.run("p007", false).await;
4742 state.schema = 15;
4743 state.save_under(&f.home).unwrap();
4744 let native = RunState::load_under(&state.id, &f.home).unwrap();
4745 assert_eq!(native.schema, crate::run::SCHEMA);
4746 assert!(native.candidate_preservation.is_empty());
4747 assert!(native.retired_to.is_none());
4748 state.schema = 14;
4749 state.save_under(&f.home).unwrap();
4750 let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
4751 assert!(legacy.candidate_preservation.contains_key(&'A'));
4752 legacy.save_under(&f.home).unwrap();
4753 let restored = RunState::load_under(&state.id, &f.home).unwrap();
4754 assert_eq!(
4755 restored.candidate_preservation,
4756 legacy.candidate_preservation
4757 );
4758 }
4759
4760 #[tokio::test]
4761 async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
4762 let f = Fx::new().await;
4763 let cache = f.repo.join("cache");
4764 let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
4765 std::fs::create_dir_all(&cache).unwrap();
4766 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4767 assert!(
4768 crate::cache::maintenance_prune(&f.home, &cache, 1)
4769 .unwrap()
4770 .is_none()
4771 );
4772 drop(guard);
4773 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4774 }
4775
4776 #[tokio::test]
4777 async fn launcher_death_does_not_release_live_tool_output_reservations() {
4778 let f = Fx::new().await;
4779 let run = f.run("abcd", false).await;
4780 let wt = &run.candidates[0].worktree;
4781 let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
4782 helper
4783 .quiet()
4784 .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
4785 .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
4786 .env("MAGI_TEST_WORKTREE", wt)
4787 .stdin(std::process::Stdio::null())
4788 .stdout(std::process::Stdio::null())
4789 .stderr(std::process::Stdio::null())
4790 .kill_on_drop(true);
4791 assert!(
4792 tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
4793 .await
4794 .unwrap()
4795 .unwrap()
4796 .success()
4797 );
4798 let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
4799 .unwrap()
4800 .trim()
4801 .parse()
4802 .unwrap();
4803 struct KillTool(i32);
4804 impl Drop for KillTool {
4805 fn drop(&mut self) {
4806 unsafe { libc::kill(self.0, libc::SIGKILL) };
4808 }
4809 }
4810 let tool = KillTool(pid);
4811 assert!(OutputLease::take(wt, true).is_err());
4812 assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
4813 drop(tool);
4814 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4815 loop {
4816 if OutputLease::take(wt, true).is_ok()
4817 && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
4818 {
4819 break;
4820 }
4821 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4822 }
4823 })
4824 .await
4825 .unwrap();
4826 }
4827
4828 #[tokio::test]
4829 async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
4830 let f = Fx::new().await;
4831 let run = f.run("abcd", false).await;
4832 let wt = &run.candidates[0].worktree;
4833 let cache = f.repo.join("cache");
4834 std::fs::create_dir(&cache).unwrap();
4835 for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
4836 let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
4837 let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
4838 cmd.quiet()
4839 .args(["--exact", "seat_process::tests::orphan_tool_helper"])
4840 .env("MAGI_TEST_ORPHAN_PID", &pid_file)
4841 .stdin(std::process::Stdio::null())
4842 .stdout(std::process::Stdio::null())
4843 .stderr(std::process::Stdio::null())
4844 .kill_on_drop(true);
4845 if clear_owner {
4846 cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
4847 } else {
4848 cmd.env("MAGI_TEST_CLOSE_FDS", "1");
4849 }
4850 let gib = 1024 * 1024 * 1024;
4851 let mut leases = vec![
4852 OutputLease::borrow(wt).unwrap().unwrap(),
4853 OutputLease::cache(&f.home, &cache, false).unwrap(),
4854 ];
4855 leases.extend(
4856 OutputLease::admit_with(
4857 &f.home,
4858 std::slice::from_ref(wt),
4859 8 * gib,
4860 |_| Ok(9 * gib),
4861 )
4862 .unwrap(),
4863 );
4864 let mut running = crate::seat_process::Running::spawn(cmd, None, leases).unwrap();
4865 assert!(
4866 tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
4867 .await
4868 .unwrap()
4869 .unwrap()
4870 .success()
4871 );
4872 let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
4873 struct KillTool(Option<i32>);
4874 impl Drop for KillTool {
4875 fn drop(&mut self) {
4876 if let Some(pid) = self.0 {
4878 unsafe { libc::kill(pid, libc::SIGKILL) };
4879 }
4880 }
4881 }
4882 let mut tool = KillTool(Some(pid));
4883 assert!(OutputLease::take(wt, true).is_err());
4887 let finish = if cancel {
4888 drop(running);
4889 None
4890 } else {
4891 Some(tokio::spawn(async move { running.finish().await }))
4892 };
4893 if clear_owner {
4894 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
4895 assert!(crate::proc::pid_alive(pid as u32));
4896 assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
4897 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4898 assert!(
4899 OutputLease::admit_with(&f.home, std::slice::from_ref(wt), 8 * gib, |_| Ok(
4900 9 * gib
4901 ))
4902 .is_err(),
4903 "orphan still owns its disk reservation"
4904 );
4905 assert!(
4906 crate::cache::maintenance_prune(&f.home, &cache, 0)
4907 .unwrap()
4908 .is_none()
4909 );
4910 if let Some(finish) = &finish {
4911 assert!(!finish.is_finished());
4912 }
4913 unsafe { libc::kill(pid, libc::SIGKILL) };
4915 }
4916 if let Some(finish) = finish {
4917 tokio::time::timeout(std::time::Duration::from_secs(5), finish)
4918 .await
4919 .unwrap()
4920 .unwrap();
4921 }
4922 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4923 loop {
4924 if OutputLease::take(wt, true).is_ok()
4925 && OutputLease::cache(&f.home, &cache, true).is_ok()
4926 && OutputLease::admit_with(
4927 &f.home,
4928 std::slice::from_ref(wt),
4929 8 * gib,
4930 |_| Ok(9 * gib),
4931 )
4932 .is_ok()
4933 {
4934 if !clear_owner {
4936 assert!(!crate::proc::pid_alive(pid as u32));
4937 }
4938 break;
4939 }
4940 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4941 }
4942 })
4943 .await
4944 .unwrap();
4945 tool.0 = None;
4946 }
4947 }
4948
4949 #[tokio::test]
4950 async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
4951 let f = Fx::new().await;
4952 let run = f.run("abcd", false).await;
4953 let wt = &run.candidates[0].worktree;
4954 let cache = f.repo.join("cache");
4955 std::fs::create_dir(&cache).unwrap();
4956 let pid_file = f.repo.join("child-pid");
4957 let mut cmd = tokio::process::Command::new("sh");
4958 cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
4959 .quiet()
4960 .env("CHILD_PID_FILE", &pid_file)
4961 .process_group(0)
4962 .kill_on_drop(true);
4963 let running = crate::seat_process::Running::spawn(
4964 cmd,
4965 None,
4966 vec![
4967 OutputLease::borrow(wt).unwrap().unwrap(),
4968 OutputLease::cache(&f.home, &cache, false).unwrap(),
4969 ],
4970 )
4971 .unwrap();
4972 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4973 while !pid_file.exists() {
4974 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4975 }
4976 })
4977 .await
4978 .unwrap();
4979 assert!(OutputLease::take(wt, true).is_err());
4980 assert!(
4981 crate::cache::maintenance_prune(&f.home, &cache, 0)
4982 .unwrap()
4983 .is_none()
4984 );
4985 let pid: u32 = std::fs::read_to_string(pid_file)
4986 .unwrap()
4987 .trim()
4988 .parse()
4989 .unwrap();
4990 drop(running);
4991 tokio::time::timeout(std::time::Duration::from_secs(10), async {
4992 loop {
4993 if let Ok(guard) = OutputLease::take(wt, true) {
4994 assert!(
4995 !crate::proc::pid_alive(pid),
4996 "output lock released before tool death"
4997 );
4998 drop(guard);
4999 break;
5000 }
5001 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5002 }
5003 })
5004 .await
5005 .unwrap();
5006 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
5007 }
5008
5009 #[tokio::test]
5010 async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
5011 let f = Fx::new().await;
5012 let reason = tokio::time::timeout(
5013 std::time::Duration::from_secs(2),
5014 emergency(vec![f.repo.clone()], u64::MAX),
5015 )
5016 .await
5017 .unwrap();
5018 assert!(reason.contains("disk pressure"));
5019 assert!(
5020 tokio::time::timeout(
5021 std::time::Duration::from_millis(10),
5022 emergency(vec![f.repo.clone()], 0)
5023 )
5024 .await
5025 .is_err()
5026 );
5027 assert_eq!(
5028 std::fs::read_to_string(f.repo.join("source")).unwrap(),
5029 "base\n"
5030 );
5031 }
5032
5033 #[tokio::test]
5034 async fn automatic_requeues_cannot_refund_failed_input_generation() {
5035 let f = Fx::new().await;
5036 let mut run = f.run("1234", false).await;
5037 run.prep_failure = Some("cherry-pick conflict".into());
5038 let mut task = f.task(std::slice::from_ref(&run));
5039 let inputs = crate::queue::recovery_inputs(&run.sources);
5040 for _ in 0..108 {
5041 task.recovery_failed(&run);
5042 task.requeue();
5043 assert_eq!(task.attempts, 0);
5044 }
5045 let bytes = serde_json::to_vec(&task).unwrap();
5046 let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
5047 assert!(
5048 restored
5049 .recovery_exhausted(&run.base_commit, &inputs, 2)
5050 .is_some()
5051 );
5052 assert!(
5053 restored
5054 .recovery_exhausted("different-base", &inputs, 2)
5055 .is_none()
5056 );
5057 let original_base = run.base_commit.clone();
5058 run.base_commit = "different-base".into();
5059 restored.recovery_failed(&run);
5060 assert!(
5061 restored
5062 .recovery_exhausted(&original_base, &inputs, 2)
5063 .is_some(),
5064 "returning to old inputs does not replenish their budget"
5065 );
5066 assert!(
5067 restored
5068 .recovery_exhausted(&run.base_commit, &inputs, 2)
5069 .is_none()
5070 );
5071 restored.release_operator("test-owner", None);
5072 assert!(
5073 restored
5074 .recovery_exhausted(&run.base_commit, &inputs, 2)
5075 .is_none()
5076 );
5077 }
5078 async fn inactive_source(f: &Fx, tag: &str, useful: bool) -> RunState {
5079 let mut state = f.run(tag, useful).await;
5080 state.status = RunStatus::Superseded;
5081 state.save_under(&f.home).unwrap();
5082 state
5083 }
5084
5085 fn historical_review(state: &mut RunState, head: &str) {
5086 state.reviews.push(
5087 serde_json::from_value(serde_json::json!({
5088 "round": state.reviews.len() + 1, "head": head, "reviews": []
5089 }))
5090 .unwrap(),
5091 );
5092 }
5093
5094 async fn reviewer(f: &Fx, state: &RunState, head: &str) -> PathBuf {
5095 let path = state.worktree_root().join("review-1");
5096 git::worktree_add_detached(&f.repo, &path, head)
5097 .await
5098 .unwrap();
5099 path
5100 }
5101
5102 #[tokio::test]
5103 async fn source_eviction_separates_run_authority_from_read_seats() {
5104 for status in [RunStatus::Superseded, RunStatus::Ready, RunStatus::Merged] {
5105 let f = Fx::new().await;
5106 let mut state = inactive_source(&f, "ef01", true).await;
5107 state.status = status;
5108 state.pr = Some(
5109 serde_json::from_value(serde_json::json!({
5110 "url":"https://example.invalid/pull/1", "number":1, "state":"merged",
5111 "checks":"green", "round":1, "rounds":1
5112 }))
5113 .unwrap(),
5114 );
5115 state.pr_head = Some(state.candidates[0].branch.clone());
5116 state.pr_merged_head = Some(f.base.clone());
5117 state.recovery = Some(
5118 serde_json::from_value(serde_json::json!({
5119 "from_run": "20260101-000000-abcd", "generation": 1,
5120 "mode": "full-review"
5121 }))
5122 .unwrap(),
5123 );
5124 state.pr_recovery = Some(
5125 serde_json::from_value(serde_json::json!({
5126 "url": "https://example.invalid/pull/1", "branch": "public",
5127 "head": f.base, "source_run": state.id,
5128 "instruction": "keep implementation", "approved_at": Fx::now()
5129 }))
5130 .unwrap(),
5131 );
5132 state.save_under(&f.home).unwrap();
5133 let review = reviewer(&f, &state, &f.base).await;
5134 let mut other = inactive_source(&f, "ef02", true).await;
5135 other.pr_recovery = state.pr_recovery.clone();
5136 other.save_under(&f.home).unwrap();
5137 let mut task = f.task(&[state.clone(), other]);
5138 task.pr_recovery = state.pr_recovery.clone();
5139 crate::queue::Queue::at(f.home.join("queue"))
5140 .put(&mut task)
5141 .unwrap();
5142 let run_path = f.home.join("runs").join(&state.id).join("run.json");
5143 let before = std::fs::read(&run_path).unwrap();
5144 let task_before = serde_json::to_value(&task).unwrap();
5145 let mut report = Report::default();
5146 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5147 assert_eq!(report.evicted_copies, 1, "{status:?}: {report:?}");
5148 assert!(!review.exists());
5149 assert!(state.candidates[0].worktree.exists());
5150 assert!(
5151 git::branch_exists(&f.repo, &state.candidates[0].branch)
5152 .await
5153 .unwrap()
5154 );
5155 assert_eq!(std::fs::read(&run_path).unwrap(), before);
5156 assert_eq!(
5157 serde_json::to_value(
5158 crate::queue::Queue::at(f.home.join("queue"))
5159 .get(&task.id)
5160 .unwrap()
5161 )
5162 .unwrap(),
5163 task_before
5164 );
5165 }
5166 }
5167
5168 #[tokio::test]
5169 async fn source_eviction_retains_own_paths_and_unfinished_continuations() {
5170 for kind in ["own-path", "active-seat", "paused-session"] {
5171 let f = Fx::new().await;
5172 let mut state = inactive_source(&f, "ef03", true).await;
5173 let review = reviewer(&f, &state, &f.base).await;
5174 let expected = match kind {
5175 "own-path" => {
5176 state.attachments.push(review.join("source"));
5177 "run.attachments[0]"
5178 }
5179 "active-seat" => {
5180 state.active.insert(
5181 "review-1".into(),
5182 serde_json::from_value(serde_json::json!({
5183 "node":"review", "started_at":Fx::now(), "timeout_secs":60
5184 }))
5185 .unwrap(),
5186 );
5187 "continuation lease"
5188 }
5189 _ => {
5190 state.status = RunStatus::Blocked;
5191 state.parked = true;
5192 let mut seat = crate::agent::SeatState::new("review-1", "fixture", 1);
5193 seat.turns = 2;
5194 seat.captured_session = Some("saved-session".into());
5195 state.seats.insert("review-1".into(), seat);
5196 "resume contract"
5197 }
5198 };
5199 state.save_under(&f.home).unwrap();
5200 let mut report = Report::default();
5201 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5202 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5203 assert!(review.exists());
5204 assert!(
5205 report.retained.iter().any(|r| r.contains(expected)),
5206 "{kind}: {report:?}"
5207 );
5208 restore_execution_seats(&f.home, &state).await.unwrap();
5211 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), f.base);
5212 }
5213 }
5214
5215 #[tokio::test]
5216 async fn source_eviction_distinguishes_historical_cwd_from_question_dependencies() {
5217 for kind in ["waiting", "lease", "historical", "unknown"] {
5218 let f = Fx::new().await;
5219 let state = inactive_source(&f, "ef06", true).await;
5220 let review = reviewer(&f, &state, &f.base).await;
5221 let store = crate::ask::Questions::at(f.home.join("questions"));
5222 let mut question = crate::ask::Question::new(
5223 "20260101-000000-ffff".into(),
5224 "review".into(),
5225 "review-1".into(),
5226 "fixture".into(),
5227 "fixture".into(),
5228 Vec::new(),
5229 );
5230 question.cwd = Some(review.to_string_lossy().into_owned());
5231 if kind != "waiting" {
5232 question.status = crate::ask::QuestionStatus::Abandoned;
5233 }
5234 store.put(&mut question).unwrap();
5235 if kind == "lease" {
5236 store.beat(&question.id, crate::ask::WaiterKind::Daemon);
5237 }
5238 if kind == "unknown" {
5239 let path = store.path_of(&question.id);
5240 let mut raw: serde_json::Value =
5241 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5242 raw["future_cwd_authority"] = serde_json::Value::Null;
5243 std::fs::write(path, serde_json::to_vec(&raw).unwrap()).unwrap();
5244 }
5245 let mut report = Report::default();
5246 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5247 assert_eq!(
5248 report.evicted_copies,
5249 usize::from(kind == "historical"),
5250 "{kind}: {report:?}"
5251 );
5252 assert_eq!(review.exists(), kind != "historical");
5253 if kind == "waiting" || kind == "lease" {
5254 assert!(
5255 report
5256 .retained
5257 .iter()
5258 .any(|r| r.contains("question") && r.contains("cwd")),
5259 "{report:?}"
5260 );
5261 }
5262 }
5263 }
5264
5265 #[tokio::test]
5266 async fn source_eviction_failed_execution_setup_keeps_its_checkpoint() {
5267 let f = Fx::new().await;
5268 let mut state = inactive_source(&f, "ef07", true).await;
5269 let review = reviewer(&f, &state, &f.base).await;
5270 let mut report = Report::default();
5271 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5272 assert_eq!(report.evicted_copies, 1, "{report:?}");
5273 state.config.worktree.setup = vec![crate::config::SetupStep {
5274 run: Some("printf preserved > source; exit 1".into()),
5275 ..Default::default()
5276 }];
5277 state.save_under(&f.home).unwrap();
5278 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5279 let marker = f
5280 .home
5281 .join("runs")
5282 .join(&state.id)
5283 .join("artifacts/restore-review-1.pending");
5284 assert!(marker.exists());
5285 assert_eq!(
5286 std::fs::read_to_string(review.join("source")).unwrap(),
5287 "preserved"
5288 );
5289 assert!(
5290 restore_execution_seats(&f.home, &state)
5291 .await
5292 .unwrap_err()
5293 .to_string()
5294 .contains("interrupted execution restoration")
5295 );
5296 let mut retry = Report::default();
5297 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5298 assert_eq!(retry.evicted_copies, 0, "{retry:?}");
5299 assert!(
5300 retry
5301 .retained
5302 .iter()
5303 .any(|r| r.contains("restoration checkpoint")),
5304 "{retry:?}"
5305 );
5306 assert!(marker.exists());
5307 assert_eq!(
5308 std::fs::read_to_string(review.join("source")).unwrap(),
5309 "preserved"
5310 );
5311 }
5312
5313 #[tokio::test]
5314 async fn source_eviction_retains_borrowed_or_promisor_anchor_stores() {
5315 for kind in [
5316 "alternates",
5317 "promisor-config",
5318 "promisor-pack",
5319 "objects-symlink",
5320 ] {
5321 let f = Fx::new().await;
5322 let state = inactive_source(&f, "ef11", true).await;
5323 let review = reviewer(&f, &state, &f.base).await;
5324 let common = crate::repo_lock::common_dir(&f.repo)
5325 .await
5326 .unwrap()
5327 .unwrap();
5328 match kind {
5329 "alternates" => {
5330 let borrowed = f._temp.path().join("borrowed-objects");
5331 std::fs::create_dir(&borrowed).unwrap();
5332 std::fs::write(
5333 common.join("objects/info/alternates"),
5334 format!("{}\n", borrowed.display()),
5335 )
5336 .unwrap();
5337 }
5338 "promisor-config" => {
5339 git::git(&f.repo, &["config", "remote.origin.promisor", "true"])
5340 .await
5341 .unwrap();
5342 }
5343 "promisor-pack" => {
5344 std::fs::write(common.join("objects/pack/fixture.promisor"), "").unwrap();
5345 }
5346 _ => {
5347 let borrowed = f._temp.path().join("borrowed-objects");
5348 std::fs::rename(common.join("objects"), &borrowed).unwrap();
5349 std::os::unix::fs::symlink(borrowed, common.join("objects")).unwrap();
5350 }
5351 }
5352 let mut report = Report::default();
5353 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5354 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5355 assert!(review.exists());
5356 assert!(
5357 report.retained.iter().any(|r| r.contains("source anchor")),
5358 "{kind}: {report:?}"
5359 );
5360 }
5361 }
5362
5363 #[tokio::test]
5364 async fn source_eviction_bounds_long_shared_history_and_requires_its_retained_anchor() {
5365 let mut f = Fx::new().await;
5366 crate::fold_evidence::fixtures::shared_history(&f.repo, &f.base, 5000).await;
5367 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5368 assert_eq!(
5369 git::git(&f.repo, &["rev-list", "--count", &f.base])
5370 .await
5371 .unwrap(),
5372 "5001"
5373 );
5374 let state = inactive_source(&f, "ef10", true).await;
5375 let review = reviewer(&f, &state, &f.base).await;
5376 std::fs::write(review.join("source"), "bounded review delta").unwrap();
5377 git::git(&review, &["add", "."]).await.unwrap();
5378 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5379 let head = git::git(
5380 &review,
5381 &["commit-tree", &tree, "-p", &f.base, "-m", "review delta"],
5382 )
5383 .await
5384 .unwrap();
5385 git::git(&review, &["checkout", "--detach", &head])
5386 .await
5387 .unwrap();
5388 let run_file = f.home.join("runs").join(&state.id).join("run.json");
5389 let run_before = std::fs::read(&run_file).unwrap();
5390 let mut report = Report::default();
5391 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5392 assert_eq!(report.evicted_copies, 1, "{report:?}");
5393 assert!(!review.exists());
5394 assert!(state.candidates[0].worktree.exists());
5395 assert!(
5396 git::branch_exists(&f.repo, &state.candidates[0].branch)
5397 .await
5398 .unwrap()
5399 );
5400 assert_eq!(std::fs::read(&run_file).unwrap(), run_before);
5401 let archive = f
5402 .home
5403 .join("runs")
5404 .join(&state.id)
5405 .join("artifacts/source-evidence")
5406 .join(format!("{head}-{}", f.base));
5407 let manifest: serde_json::Value =
5408 serde_json::from_slice(&std::fs::read(archive.join("manifest.json")).unwrap()).unwrap();
5409 assert_eq!(manifest["anchor"], f.base);
5410 assert!(
5411 std::fs::metadata(archive.join("objects.pack"))
5412 .unwrap()
5413 .len()
5414 < 16 * 1024
5415 );
5416 let pin = format!("refs/magi/source-anchors/{}", f.base);
5417 assert_eq!(git::rev_parse(&f.repo, &pin).await.unwrap(), f.base);
5418 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5419 .await
5420 .unwrap();
5421 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5422 assert!(
5423 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5424 .await
5425 .unwrap()
5426 .ok()
5427 );
5428 let other = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5429 .await
5430 .unwrap();
5431 for kind in ["missing", "moved", "symbolic", "symlink"] {
5433 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5434 .await
5435 .unwrap();
5436 match kind {
5437 "moved" => {
5438 git::git(&f.repo, &["update-ref", &pin, &other])
5439 .await
5440 .unwrap();
5441 }
5442 "symbolic" => {
5443 git::git(&f.repo, &["symbolic-ref", &pin, "refs/heads/main"])
5444 .await
5445 .unwrap();
5446 }
5447 "symlink" => {
5448 let common = crate::repo_lock::common_dir(&f.repo)
5449 .await
5450 .unwrap()
5451 .unwrap();
5452 let target = f._temp.path().join("operator-anchor");
5453 std::fs::write(&target, format!("{}\n", f.base)).unwrap();
5454 let file = common.join(&pin);
5455 std::fs::create_dir_all(file.parent().unwrap()).unwrap();
5456 std::os::unix::fs::symlink(target, file).unwrap();
5457 }
5458 _ => {}
5459 }
5460 assert!(
5461 restore_execution_seats(&f.home, &state).await.is_err(),
5462 "{kind}"
5463 );
5464 assert!(!review.exists());
5465 assert!(
5466 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5467 .await
5468 .unwrap()
5469 .ok()
5470 );
5471 if kind == "symlink" {
5472 let common = crate::repo_lock::common_dir(&f.repo)
5473 .await
5474 .unwrap()
5475 .unwrap();
5476 std::fs::remove_file(common.join(&pin)).unwrap();
5477 } else {
5478 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
5479 .await
5480 .unwrap();
5481 }
5482 git::git(&f.repo, &["update-ref", &pin, &f.base])
5483 .await
5484 .unwrap();
5485 }
5486 assert!(
5489 git::worktree_remove_clean(&f.repo, &state.candidates[0].worktree)
5490 .await
5491 .unwrap()
5492 );
5493 git::git(
5494 &f.repo,
5495 &[
5496 "update-ref",
5497 "-d",
5498 &format!("refs/heads/{}", state.candidates[0].branch),
5499 ],
5500 )
5501 .await
5502 .unwrap();
5503 git::git(&f.repo, &["update-ref", "-d", "refs/heads/main"])
5504 .await
5505 .unwrap();
5506 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5507 .await
5508 .unwrap();
5509 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5510 assert_eq!(
5511 git::git(&f.repo, &["for-each-ref", "--format=%(refname)"])
5512 .await
5513 .unwrap(),
5514 pin
5515 );
5516 assert_eq!(
5517 git::git(&f.repo, &["rev-list", "--count", &f.base])
5518 .await
5519 .unwrap(),
5520 "5001"
5521 );
5522 restore_execution_seats(&f.home, &state).await.unwrap();
5523 assert_eq!(
5524 git::git(&review, &["rev-list", "--count", "HEAD"])
5525 .await
5526 .unwrap(),
5527 "5002"
5528 );
5529 assert_eq!(
5530 git::git(&f.repo, &["rev-parse", "--is-shallow-repository"])
5531 .await
5532 .unwrap(),
5533 "false"
5534 );
5535 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5536 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5537 let patch = seat_diff(&f.home, &state.id, &head).await.unwrap();
5538 assert!(String::from_utf8_lossy(&patch).contains("bounded review delta"));
5539 std::fs::remove_dir_all(&f.repo).unwrap();
5540 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), patch);
5541 let inspection = f._temp.path().join("shared-history-inspection");
5542 restore_seat(&f.home, &state.id, &head, &inspection)
5543 .await
5544 .unwrap();
5545 assert_eq!(git::rev_parse(&inspection, "HEAD").await.unwrap(), head);
5546 assert_eq!(
5547 git::git(&inspection, &["rev-parse", "--is-shallow-repository"])
5548 .await
5549 .unwrap(),
5550 "true"
5551 );
5552 assert_eq!(
5553 git::git(&inspection, &["log", "--format=%H"])
5554 .await
5555 .unwrap(),
5556 format!("{head}\n{}", f.base)
5557 );
5558 assert!(!inspection.join(".git/objects/info/alternates").exists());
5559 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5560 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5561 }
5562
5563 #[tokio::test]
5564 async fn source_eviction_retains_tree_only_archives_and_rejects_import_before_writes() {
5565 let f = Fx::new().await;
5566 let state = inactive_source(&f, "ef08", true).await;
5567 let review = reviewer(&f, &state, &f.base).await;
5568 let tree = git::rev_parse(&review, "HEAD^{tree}").await.unwrap();
5569 let parent = git::git(&review, &["commit-tree", &tree, "-m", "U0"])
5570 .await
5571 .unwrap();
5572 let head = git::git(&review, &["commit-tree", &tree, "-p", &parent, "-m", "U1"])
5573 .await
5574 .unwrap();
5575 git::git(&review, &["checkout", "--detach", &head])
5576 .await
5577 .unwrap();
5578 let archive =
5579 crate::fold_evidence::fixtures::tree_only_archive(&state, &f.home, &head).await;
5580 let mut report = Report::default();
5581 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5582 assert_eq!(report.evicted_copies, 0, "{report:?}");
5583 assert!(review.exists());
5584 assert!(
5585 report
5586 .retained
5587 .iter()
5588 .any(|r| r.contains("incomplete archived commit ancestry")),
5589 "{report:?}"
5590 );
5591 assert!(git::worktree_remove_clean(&f.repo, &review).await.unwrap());
5593 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5594 .await
5595 .unwrap();
5596 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5597 for oid in [&parent, &head] {
5598 assert!(
5599 !git::git_raw(&f.repo, &["cat-file", "-e", oid])
5600 .await
5601 .unwrap()
5602 .ok()
5603 );
5604 }
5605 let patch = source_diff(&archive).await.unwrap();
5606 assert!(
5607 crate::fold_evidence::import_snapshot(&archive, &f.repo)
5608 .await
5609 .is_err()
5610 );
5611 assert!(
5612 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5613 .await
5614 .unwrap()
5615 .ok()
5616 );
5617 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5618 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5619 let inspection = f._temp.path().join("legacy-tree-inspection");
5620 restore_source(&archive, &inspection).await.unwrap();
5621 assert_eq!(git::git(&inspection, &["write-tree"]).await.unwrap(), tree);
5622 assert!(
5623 !git::git_raw(&inspection, &["rev-parse", "--verify", "HEAD"])
5624 .await
5625 .unwrap()
5626 .ok()
5627 );
5628 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5629 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5630 assert_eq!(source_diff(&archive).await.unwrap(), patch);
5631 assert!(state.candidates[0].worktree.exists());
5632 }
5633
5634 #[tokio::test]
5635 async fn source_eviction_retains_ancestry_above_expanded_budget() {
5636 let f = Fx::new().await;
5637 let state = inactive_source(&f, "ef09", true).await;
5638 let review = reviewer(&f, &state, &f.base).await;
5639 std::fs::File::create(review.join("ancestor-only"))
5640 .unwrap()
5641 .set_len(64 * 1024 * 1024 + 1)
5642 .unwrap();
5643 git::git(&review, &["add", "."]).await.unwrap();
5644 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5645 let parent = git::git(
5646 &review,
5647 &["commit-tree", &parent_tree, "-m", "large ancestor"],
5648 )
5649 .await
5650 .unwrap();
5651 std::fs::remove_file(review.join("ancestor-only")).unwrap();
5652 git::git(&review, &["add", "."]).await.unwrap();
5653 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5654 let head = git::git(
5655 &review,
5656 &[
5657 "commit-tree",
5658 &tree,
5659 "-p",
5660 &parent,
5661 "-m",
5662 "small final tree",
5663 ],
5664 )
5665 .await
5666 .unwrap();
5667 git::git(&review, &["checkout", "--detach", &head])
5668 .await
5669 .unwrap();
5670 let mut report = Report::default();
5671 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5672 assert_eq!(report.evicted_copies, 0, "{report:?}");
5673 assert!(
5674 report
5675 .retained
5676 .iter()
5677 .any(|r| r.contains("expanded object limit exceeded")),
5678 "{report:?}"
5679 );
5680 assert!(review.exists());
5681 assert!(state.candidates[0].worktree.exists());
5682 }
5683
5684 #[tokio::test]
5685 async fn source_eviction_restores_execution_after_gc_and_preserves_generations() {
5686 let f = Fx::new().await;
5687 let state = inactive_source(&f, "ef04", true).await;
5688 let review = reviewer(&f, &state, &f.base).await;
5689 std::fs::write(review.join("ancestor-only"), "ancestral content").unwrap();
5690 git::git(&review, &["add", "."]).await.unwrap();
5691 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
5692 let parent = git::git(
5693 &review,
5694 &["commit-tree", &parent_tree, "-m", "unrelated parent"],
5695 )
5696 .await
5697 .unwrap();
5698 std::fs::remove_file(review.join("ancestor-only")).unwrap();
5699 std::fs::write(review.join("source"), "unrelated review checkpoint").unwrap();
5700 git::git(&review, &["add", "."]).await.unwrap();
5701 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5702 let head = git::git(
5703 &review,
5704 &[
5705 "commit-tree",
5706 &tree,
5707 "-p",
5708 &parent,
5709 "-m",
5710 "unrelated review checkpoint",
5711 ],
5712 )
5713 .await
5714 .unwrap();
5715 git::git(&review, &["checkout", "--detach", &head])
5716 .await
5717 .unwrap();
5718 let mut report = Report::default();
5719 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5720 assert_eq!(report.evicted_copies, 1, "{report:?}");
5721 let ledger = f
5722 .home
5723 .join("runs")
5724 .join(&state.id)
5725 .join("artifacts/eviction-review-1.json");
5726 let original = std::fs::read(&ledger).unwrap();
5727 let diff = seat_diff(&f.home, &state.id, &head).await.unwrap();
5728 assert!(String::from_utf8_lossy(&diff).contains("unrelated review checkpoint"));
5729 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5730 .await
5731 .unwrap();
5732 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5733 assert!(
5734 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5735 .await
5736 .unwrap()
5737 .ok()
5738 );
5739 assert!(
5740 !git::git_raw(&f.repo, &["cat-file", "-e", &parent])
5741 .await
5742 .unwrap()
5743 .ok()
5744 );
5745 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5746 assert!(
5747 seat_heads(&f.home, &state.id)
5748 .await
5749 .unwrap()
5750 .contains_key(&head)
5751 );
5752 restore_execution_seats(&f.home, &state).await.unwrap();
5753 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), head);
5754 assert_eq!(
5755 git::git(&review, &["log", "--format=%H"]).await.unwrap(),
5756 format!("{head}\n{parent}")
5757 );
5758 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
5759 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5760 let inspection = f._temp.path().join("complete-history-inspection");
5761 restore_seat(&f.home, &state.id, &head, &inspection)
5762 .await
5763 .unwrap();
5764 assert_eq!(
5765 git::git(&inspection, &["log", "--format=%H"])
5766 .await
5767 .unwrap(),
5768 format!("{head}\n{parent}")
5769 );
5770 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
5771 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
5772
5773 assert_eq!(git::git(&review, &["write-tree"]).await.unwrap(), tree);
5774 assert!(
5775 !git::git_raw(&review, &["symbolic-ref", "-q", "HEAD"])
5776 .await
5777 .unwrap()
5778 .ok()
5779 );
5780 assert_eq!(
5781 std::fs::read_to_string(review.join("source")).unwrap(),
5782 "unrelated review checkpoint"
5783 );
5784 std::fs::write(review.join("operator-notes"), "keep").unwrap();
5785 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5786 assert_eq!(
5787 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5788 "keep"
5789 );
5790 std::fs::remove_file(review.join("operator-notes")).unwrap();
5791 restore_execution_seats(&f.home, &state).await.unwrap();
5792 let mut second = Report::default();
5793 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
5794 assert_eq!(second.evicted_copies, 1, "{second:?}");
5795 assert_eq!(std::fs::read(&ledger).unwrap(), original);
5796 assert!(ledger.with_file_name("eviction-review-1-2.json").exists());
5797 let mut legacy: serde_json::Value = serde_json::from_slice(&original).unwrap();
5800 legacy["version"] = 1.into();
5801 legacy.as_object_mut().unwrap().remove("base");
5802 std::fs::write(&ledger, serde_json::to_vec(&legacy).unwrap()).unwrap();
5803 let mut advanced = state.clone();
5804 advanced.base_commit = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5805 .await
5806 .unwrap();
5807 advanced.save_under(&f.home).unwrap();
5808 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
5809 let raw: serde_json::Value = serde_json::from_slice(&original).unwrap();
5811 let archive = PathBuf::from(raw["evidence"].as_str().unwrap());
5812 let patch = std::fs::read(archive.join("change.patch")).unwrap();
5813 std::fs::write(archive.join("change.patch"), "corrupt").unwrap();
5814 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5815 assert!(!review.exists());
5816 std::fs::write(archive.join("change.patch"), patch).unwrap();
5817 std::fs::create_dir(&review).unwrap();
5818 std::fs::write(review.join("operator-notes"), "keep").unwrap();
5819 assert!(restore_execution_seats(&f.home, &state).await.is_err());
5820 assert_eq!(
5821 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
5822 "keep"
5823 );
5824 }
5825
5826 #[tokio::test]
5827 async fn source_eviction_preserves_history_and_restores_binary_modes_after_gc() {
5828 use std::os::unix::fs::PermissionsExt;
5829 let mut f = Fx::new().await;
5830 std::fs::write(f.repo.join("binary"), b"old\0binary\xff").unwrap();
5831 git::git(&f.repo, &["add", "."]).await.unwrap();
5832 git::git(&f.repo, &["commit", "-m", "binary base"])
5833 .await
5834 .unwrap();
5835 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5836 let mut state = inactive_source(&f, "ec01", true).await;
5837 let candidate = state.candidates[0].worktree.clone();
5838 let earlier = git::rev_parse(&candidate, "HEAD").await.unwrap();
5839 historical_review(&mut state, &earlier);
5840 let review = reviewer(&f, &state, &earlier).await;
5841 std::fs::remove_file(candidate.join("source")).unwrap();
5842 std::fs::write(candidate.join("binary"), b"new\0binary\xfe").unwrap();
5843 std::fs::write(candidate.join("script"), "run\n").unwrap();
5844 std::fs::set_permissions(
5845 candidate.join("script"),
5846 std::fs::Permissions::from_mode(0o755),
5847 )
5848 .unwrap();
5849 git::git(&candidate, &["add", "."]).await.unwrap();
5850 git::git(&candidate, &["commit", "-m", "binary deletion and mode"])
5851 .await
5852 .unwrap();
5853 let head = git::rev_parse(&candidate, "HEAD").await.unwrap();
5854 let tree = git::rev_parse(&candidate, "HEAD^{tree}").await.unwrap();
5855 let base_tree = git::rev_parse(&candidate, &format!("{}^{{tree}}", f.base))
5856 .await
5857 .unwrap();
5858 historical_review(&mut state, &head);
5859 historical_review(&mut state, &head);
5860 state.status = RunStatus::Superseded;
5861 state.save_under(&f.home).unwrap();
5862 let task = f.task(std::slice::from_ref(&state));
5863 let before = serde_json::to_value(&state).unwrap();
5864 git::git(&review, &["reset", "--hard", &head])
5865 .await
5866 .unwrap();
5867 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5868 assert_eq!(report.evicted_copies, 1, "{report:?}");
5869 assert!(!review.exists());
5870 assert!(candidate.exists(), "selected implementation survives");
5871 assert!(report.source_unlinked_logical_bytes > 0);
5872 assert!(report.source_unlinked_allocated_bytes.is_some());
5873 assert!(report.source_evidence_added_allocated_bytes.is_some());
5874 assert_eq!(
5875 serde_json::to_value(RunState::load_under(&state.id, &f.home).unwrap()).unwrap(),
5876 before
5877 );
5878 assert_eq!(
5879 crate::queue::Queue::at(f.home.join("queue"))
5880 .get(&task.id)
5881 .unwrap()
5882 .runs,
5883 task.runs
5884 );
5885 let archive = f
5886 .home
5887 .join("runs")
5888 .join(&state.id)
5889 .join("artifacts/source-evidence")
5890 .join(format!("{head}-{}", f.base));
5891 let patch = source_diff(&archive).await.unwrap();
5892 assert!(String::from_utf8_lossy(&patch).contains("GIT binary patch"));
5893 assert!(String::from_utf8_lossy(&patch).contains("100755"));
5894 assert!(
5895 git::worktree_remove_clean(&f.repo, &candidate)
5896 .await
5897 .unwrap()
5898 );
5899 git::git(&f.repo, &["branch", "-D", &state.candidates[0].branch])
5900 .await
5901 .unwrap();
5902 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5903 .await
5904 .unwrap();
5905 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5906 assert!(
5907 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5908 .await
5909 .unwrap()
5910 .ok()
5911 );
5912 std::fs::remove_dir_all(&f.repo).unwrap();
5914 assert_eq!(source_diff(&archive).await.unwrap(), patch);
5915 let restored = f._temp.path().join("restored");
5916 restore_source(&archive, &restored).await.unwrap();
5917 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5918 assert_eq!(
5919 std::fs::read(restored.join("binary")).unwrap(),
5920 b"new\0binary\xfe"
5921 );
5922 assert!(!restored.join("source").exists());
5923 assert_ne!(
5924 std::fs::metadata(restored.join("script"))
5925 .unwrap()
5926 .permissions()
5927 .mode()
5928 & 0o111,
5929 0
5930 );
5931 assert!(restore_source(&archive, &restored).await.is_err());
5932 git::git(&restored, &["read-tree", "--reset", "-u", &base_tree])
5935 .await
5936 .unwrap();
5937 let patch_file = f._temp.path().join("saved.patch");
5938 std::fs::write(&patch_file, &patch).unwrap();
5939 git::git(
5940 &restored,
5941 &["apply", "--index", "--binary", patch_file.to_str().unwrap()],
5942 )
5943 .await
5944 .unwrap();
5945 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5946 assert_eq!(
5947 std::fs::read(restored.join("binary")).unwrap(),
5948 b"new\0binary\xfe"
5949 );
5950 }
5951
5952 #[tokio::test]
5953 async fn source_eviction_archives_unrelated_tips_and_allows_lineage_references() {
5954 let f = Fx::new().await;
5955 let state = inactive_source(&f, "ec10", true).await;
5956 let review = reviewer(&f, &state, &f.base).await;
5957 let mut successor = inactive_source(&f, "ec11", true).await;
5958 successor.retired_to = Some(state.id.clone());
5959 successor.save_under(&f.home).unwrap();
5960 f.task(&[state.clone(), successor]);
5961
5962 let mut tips = Vec::new();
5963 for contents in ["reflog-only evidence", "ORIG_HEAD-only evidence"] {
5964 std::fs::write(review.join("source"), contents).unwrap();
5965 git::git(&review, &["add", "."]).await.unwrap();
5966 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5967 let tip = git::git(
5968 &review,
5969 &["commit-tree", &tree, "-p", &f.base, "-m", contents],
5970 )
5971 .await
5972 .unwrap();
5973 git::git(&review, &["reset", "--hard", &tip]).await.unwrap();
5974 tips.push(tip);
5975 git::git(&review, &["reset", "--hard", &f.base])
5976 .await
5977 .unwrap();
5978 }
5979 git::git(&review, &["reflog", "delete", "--rewrite", "HEAD@{1}"])
5981 .await
5982 .unwrap();
5983 git::git(&review, &["update-ref", "ORIG_HEAD", &tips[1]])
5984 .await
5985 .unwrap();
5986 let mut report = Report::default();
5987 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5988 assert_eq!(report.evicted_copies, 1, "{report:?}");
5989 assert!(!review.exists());
5990 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5991 .await
5992 .unwrap();
5993 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5994 for tip in &tips {
5995 assert!(
5996 !git::git_raw(&f.repo, &["cat-file", "-e", tip])
5997 .await
5998 .unwrap()
5999 .ok()
6000 );
6001 }
6002 std::fs::remove_dir_all(&f.repo).unwrap();
6004 let shown = seat_evidence(&f.home, &state.id).await.unwrap();
6005 for (i, tip) in tips.iter().enumerate() {
6006 assert!(
6007 shown
6008 .iter()
6009 .any(|row| row.contains(tip) && row.contains("patch_bytes="))
6010 );
6011 let destination = f._temp.path().join(format!("restore-history-{i}"));
6012 restore_seat(&f.home, &state.id, tip, &destination)
6013 .await
6014 .unwrap();
6015 assert_eq!(
6016 std::fs::read_to_string(destination.join("source")).unwrap(),
6017 ["reflog-only evidence", "ORIG_HEAD-only evidence"][i]
6018 );
6019 }
6020 assert!(
6021 restore_seat(
6022 &f.home,
6023 &state.id,
6024 "unknown",
6025 &f._temp.path().join("missing")
6026 )
6027 .await
6028 .is_err()
6029 );
6030 }
6031
6032 #[tokio::test]
6033 async fn source_eviction_revalidates_archive_immediately_before_unlink() {
6034 fn corrupt(path: &Path) -> Result<()> {
6035 let root = path.parent().unwrap();
6036 let marker = root.join("archive-to-corrupt");
6039 let archive = std::fs::read_to_string(&marker)?;
6040 std::fs::write(Path::new(&archive).join("change.patch"), "corrupt")?;
6041 Ok(())
6042 }
6043 let f = Fx::new().await;
6044 let state = inactive_source(&f, "ec12", true).await;
6045 let review = reviewer(&f, &state, &f.base).await;
6046 let archive = f
6047 .home
6048 .join("runs")
6049 .join(&state.id)
6050 .join("artifacts/source-evidence")
6051 .join(format!("{}-{}", f.base, f.base));
6052 std::fs::write(
6053 state.worktree_root().join("archive-to-corrupt"),
6054 archive.to_str().unwrap(),
6055 )
6056 .unwrap();
6057 let mut report = Report::default();
6058 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut report, corrupt).await;
6059 assert_eq!(report.evicted_copies, 0, "{report:?}");
6060 assert!(review.exists());
6061 assert!(
6062 report
6063 .retained
6064 .iter()
6065 .any(|r| r.contains("corrupt snapshot")),
6066 "{report:?}"
6067 );
6068 }
6069
6070 #[tokio::test]
6071 async fn source_eviction_fails_closed_for_dirty_ignored_borrowed_and_authoritative_copies() {
6072 for kind in [
6073 "dirty",
6074 "hidden",
6075 "ignored",
6076 "empty-directory",
6077 "untracked",
6078 "borrowed",
6079 "borrowed-symlink",
6080 "lease",
6081 "owner",
6082 "driver",
6083 "task-pin",
6084 "task-replacement",
6085 "malformed-orig-head",
6086 "grace",
6087 "resumable",
6088 "archive-symlink",
6089 "root-symlink",
6090 "checkpoint",
6091 "symlink",
6092 "retained",
6093 "unreadable",
6094 "unknown-field",
6095 "unknown-task-field",
6096 ] {
6097 let f = Fx::new().await;
6098 let mut state = inactive_source(&f, "ec02", true).await;
6099 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6100 .await
6101 .unwrap();
6102 historical_review(&mut state, &head);
6103 let review = reviewer(&f, &state, &head).await;
6104 let mut output = None;
6105 let mut owner = None;
6106 let mut driver = None;
6107 let mut disk = Fx::disk();
6108 match kind {
6109 "dirty" => std::fs::write(review.join("source"), "notes").unwrap(),
6110 "hidden" => {
6111 git::git(&review, &["update-index", "--assume-unchanged", "source"])
6112 .await
6113 .unwrap();
6114 std::fs::write(review.join("source"), "notes").unwrap();
6115 }
6116 "ignored" => {
6117 std::fs::create_dir(review.join("scratch")).unwrap();
6118 std::fs::write(review.join("scratch/notes"), "notes").unwrap();
6119 }
6120 "empty-directory" => std::fs::create_dir(review.join("owner-notes")).unwrap(),
6121 "untracked" => std::fs::write(review.join("notes"), "notes").unwrap(),
6122 "borrowed" => {
6123 let mut other = f.run("ec03", true).await;
6124 other.candidates[0].worktree = review.clone();
6125 other.save_under(&f.home).unwrap();
6126 }
6127 "borrowed-symlink" => {
6128 let alias = f._temp.path().join("borrowed-alias");
6129 std::os::unix::fs::symlink(&review, &alias).unwrap();
6130 let mut other = f.run("ec03", true).await;
6131 other.candidates[0].worktree = alias;
6132 other.save_under(&f.home).unwrap();
6133 }
6134 "lease" => output = OutputLease::borrow(&review).unwrap(),
6135 "owner" => owner = Some(crate::pr_head::owner_lock(&f.home, &state.id).unwrap()),
6136 "driver" => driver = Some(crate::driver::maintenance(&f.home, &state.id).unwrap()),
6137 "task-pin" => {
6138 let mut task = f.task(std::slice::from_ref(&state));
6139 task.resume_override = Some(crate::queue::OperatorResume {
6140 question_id: "fixture-pin".into(),
6141 at: Fx::now(),
6142 conductor_rehold: None,
6143 forced: true,
6144 pinned_run: Some(state.id.clone()),
6145 });
6146 crate::queue::Queue::at(f.home.join("queue"))
6147 .put(&mut task)
6148 .unwrap();
6149 }
6150 "task-replacement" => {
6151 let mut task = f.task(std::slice::from_ref(&state));
6152 task.replacement = Some(crate::merge_decision::Replacement {
6153 question: "fixture-replacement".into(),
6154 run: state.id.clone(),
6155 pr: "https://example.invalid/pull/1".into(),
6156 agent: "worker".into(),
6157 context: String::new(),
6158 });
6159 crate::queue::Queue::at(f.home.join("queue"))
6160 .put(&mut task)
6161 .unwrap();
6162 }
6163 "malformed-orig-head" => {
6164 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6165 .await
6166 .unwrap();
6167 std::fs::write(Path::new(&admin).join("ORIG_HEAD"), "not-an-object-id")
6168 .unwrap();
6169 }
6170 "grace" => disk.fold_grace_secs = 3600,
6171 "resumable" => state.status = RunStatus::Blocked,
6172 "archive-symlink" => {
6173 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6174 std::fs::create_dir_all(&artifacts).unwrap();
6175 let outside = f._temp.path().join("outside");
6176 std::fs::create_dir(&outside).unwrap();
6177 std::os::unix::fs::symlink(&outside, artifacts.join("source-evidence"))
6178 .unwrap();
6179 }
6180 "root-symlink" => {
6181 let root = state.worktree_root();
6182 let outside = f._temp.path().join("outside-root");
6183 std::fs::rename(&root, &outside).unwrap();
6184 std::os::unix::fs::symlink(&outside, &root).unwrap();
6185 }
6186 "checkpoint" => {
6187 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6188 .await
6189 .unwrap();
6190 std::fs::write(Path::new(&admin).join("MERGE_MSG"), "checkpoint").unwrap();
6191 }
6192 "symlink" => {
6193 std::fs::remove_file(review.join("source")).unwrap();
6194 std::os::unix::fs::symlink(".gitignore", review.join("source")).unwrap();
6195 }
6196 "retained" => std::fs::write(
6197 f.home
6198 .join("runs")
6199 .join(&state.id)
6200 .join("retain-recovery-sources"),
6201 "owner",
6202 )
6203 .unwrap(),
6204 "unreadable" => {
6205 std::fs::create_dir_all(f.home.join("runs/20260101-000000-ffff")).unwrap();
6206 }
6207 "unknown-field" | "unknown-task-field" => {}
6208 _ => unreachable!(),
6209 }
6210 state.save_under(&f.home).unwrap();
6211 if kind == "unknown-field" {
6212 let path = f.home.join("runs").join(&state.id).join("run.json");
6213 let mut raw: serde_json::Value =
6214 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6215 raw["future_source_owner"] = serde_json::json!({"path": review});
6216 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6217 }
6218 if kind == "unknown-task-field" {
6219 let task = f.task(std::slice::from_ref(&state));
6220 let path = crate::queue::Queue::at(f.home.join("queue")).path_of(&task.id);
6221 let mut raw: serde_json::Value =
6222 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6223 raw["future_source_owner"] = serde_json::json!({"path": review});
6224 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6225 }
6226 let mut report = Report::default();
6227 copies::evict(&f.home, &disk, Fx::now(), &mut report).await;
6228 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
6229 assert!(review.exists(), "{kind}: {report:?}");
6230 assert!(!report.retained.is_empty(), "{kind}: {report:?}");
6231 drop(output);
6232 drop(owner);
6233 drop(driver);
6234 }
6235 }
6236
6237 #[tokio::test]
6238 async fn source_eviction_reconciles_interruption_and_refuses_corrupt_archives() {
6239 let f = Fx::new().await;
6240 let state = inactive_source(&f, "ec04", true).await;
6241 let review = reviewer(&f, &state, &f.base).await;
6242 let mut report = Report::default();
6243 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6244 assert_eq!(report.evicted_copies, 1, "{report:?}");
6245 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6246 let ledger = artifacts.join("eviction-review-1.json");
6247 let mut record: serde_json::Value =
6248 serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6249 record["phase"] = "pending".into();
6250 std::fs::write(&ledger, serde_json::to_vec(&record).unwrap()).unwrap();
6251 let archive = PathBuf::from(record["evidence"].as_str().unwrap());
6252 let pack = std::fs::read(archive.join("objects.pack")).unwrap();
6253 std::fs::write(archive.join("objects.pack"), b"corrupt").unwrap();
6254 let mut blocked = Report::default();
6255 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6256 assert!(
6257 blocked
6258 .retained
6259 .iter()
6260 .any(|r| r.contains("corrupt snapshot")),
6261 "{blocked:?}"
6262 );
6263 assert_eq!(
6264 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6265 ["phase"],
6266 "pending"
6267 );
6268 assert!(
6269 restore_source(&archive, &f._temp.path().join("bad-restore"))
6270 .await
6271 .is_err()
6272 );
6273 std::fs::write(archive.join("objects.pack"), &pack).unwrap();
6274 let mut retried = Report::default();
6275 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6276 assert_eq!(
6277 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6278 ["phase"],
6279 "evicted"
6280 );
6281 assert_eq!(retried.evicted_copies, 0);
6282 assert!(!review.exists());
6283 let pending = archive.with_extension("pending");
6285 std::fs::rename(&archive, &pending).unwrap();
6286 assert_eq!(
6287 crate::fold_evidence::snapshot(&state, &f.home, &f.base, 64 * 1024 * 1024)
6288 .await
6289 .unwrap(),
6290 archive
6291 );
6292 assert!(!pending.exists());
6293 }
6294
6295 #[tokio::test]
6296 async fn source_eviction_reclaims_superseded_empty_attempt_without_retiring_lineage() {
6297 let f = Fx::new().await;
6298 let mut empty = inactive_source(&f, "ec05", false).await;
6299 historical_review(&mut empty, &f.base);
6300 empty.status = RunStatus::Superseded;
6301 empty.save_under(&f.home).unwrap();
6302 let source = inactive_source(&f, "ec06", true).await;
6303 let task = f.task(&[empty.clone(), source.clone()]);
6304 let before = serde_json::to_value(&empty).unwrap();
6305 let mut report = Report::default();
6306 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6307 assert_eq!(report.evicted_copies, 1, "{report:?}");
6308 assert!(!empty.candidates[0].worktree.exists());
6309 assert!(source.candidates[0].worktree.exists());
6310 assert!(
6311 git::branch_exists(&f.repo, &empty.candidates[0].branch)
6312 .await
6313 .unwrap()
6314 );
6315 assert_eq!(
6316 serde_json::to_value(RunState::load_under(&empty.id, &f.home).unwrap()).unwrap(),
6317 before
6318 );
6319 let mut retry = Report::default();
6320 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6321 assert_eq!(retry.removed_empty_roots, 1, "{retry:?}");
6322 assert!(!empty.worktree_root().exists());
6323 assert_eq!(
6324 crate::queue::Queue::at(f.home.join("queue"))
6325 .get(&task.id)
6326 .unwrap()
6327 .runs,
6328 task.runs
6329 );
6330 }
6331 #[tokio::test]
6332 async fn source_eviction_retries_pending_intent_and_exact_missing_registration() {
6333 fn interrupt(_: &Path) -> Result<()> {
6334 anyhow::bail!("fixture interruption before unlink")
6335 }
6336 fn vanish(path: &Path) -> Result<()> {
6337 std::fs::remove_dir_all(path)?;
6338 anyhow::bail!("fixture interruption after checkout unlink")
6339 }
6340 for stage in [
6341 "intent",
6342 "missing-registration",
6343 "changed-admin",
6344 "missing-archive",
6345 "wrong-identity",
6346 ] {
6347 let f = Fx::new().await;
6348 let state = inactive_source(&f, "ec07", true).await;
6349 let review = reviewer(&f, &state, &f.base).await;
6350 let admin = PathBuf::from(
6351 git::git(&review, &["rev-parse", "--absolute-git-dir"])
6352 .await
6353 .unwrap(),
6354 );
6355 let mut interrupted = Report::default();
6356 copies::evict_with(
6357 &f.home,
6358 &Fx::disk(),
6359 Fx::now(),
6360 &mut interrupted,
6361 if stage == "missing-registration" || stage == "changed-admin" {
6362 vanish
6363 } else {
6364 interrupt
6365 },
6366 )
6367 .await;
6368 assert_eq!(interrupted.evicted_copies, 0, "{stage}: {interrupted:?}");
6369 let ledger = f
6370 .home
6371 .join("runs")
6372 .join(&state.id)
6373 .join("artifacts/eviction-review-1.json");
6374 let record: serde_json::Value =
6375 serde_json::from_slice(&std::fs::read(&ledger).unwrap_or_else(|error| {
6376 panic!("{stage}: missing pending ledger: {error}; {interrupted:?}")
6377 }))
6378 .unwrap();
6379 assert_eq!(record["phase"], "pending");
6380 if stage == "changed-admin" {
6381 std::fs::write(admin.join("owner-note"), "keep").unwrap();
6382 }
6383 if stage == "missing-archive" {
6384 std::fs::remove_file(
6385 Path::new(record["evidence"].as_str().unwrap()).join("objects.pack"),
6386 )
6387 .unwrap();
6388 }
6389 if stage == "wrong-identity" {
6390 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6391 .await
6392 .unwrap();
6393 let other =
6394 crate::fold_evidence::snapshot(&state, &f.home, &head, 64 * 1024 * 1024)
6395 .await
6396 .unwrap();
6397 let original = Path::new(record["evidence"].as_str().unwrap());
6398 for name in [
6399 "manifest.json",
6400 "objects.pack",
6401 "change.patch",
6402 "change.index",
6403 ] {
6404 std::fs::copy(other.join(name), original.join(name)).unwrap();
6405 }
6406 }
6407 let mut retried = Report::default();
6408 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6409 let phase =
6410 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap())
6411 .unwrap()["phase"]
6412 .clone();
6413 if stage == "changed-admin" || stage == "missing-archive" || stage == "wrong-identity" {
6414 assert_eq!(phase, "pending", "{stage}: {retried:?}");
6415 assert!(!retried.retained.is_empty());
6416 if stage == "changed-admin" {
6417 assert!(admin.join("owner-note").exists());
6418 } else {
6419 assert!(review.exists());
6420 }
6421 } else {
6422 assert_eq!(phase, "evicted", "{stage}: {retried:?}");
6423 assert!(!review.exists());
6424 assert!(!admin.exists());
6425 }
6426 }
6427 }
6428 fn source_census(home: &Path) -> serde_json::Value {
6429 serde_json::from_slice(&std::fs::read(home.join("source-eviction-cursor")).unwrap())
6430 .unwrap()
6431 }
6432
6433 #[tokio::test]
6434 async fn source_eviction_scan_bounds_missing_root_checkpoints_and_eventually_reclaims() {
6435 let f = Fx::new().await;
6436 let state = inactive_source(&f, "ffff", true).await;
6437 let review = reviewer(&f, &state, &f.base).await;
6438 for i in 0..130 {
6439 let mut old = state.clone();
6440 old.id = format!("20260101-000000-{i:04x}");
6441 old.candidates.clear();
6442 old.save_under(&f.home).unwrap();
6443 if i == 0 {
6445 let artifacts = f.home.join("runs").join(&old.id).join("artifacts");
6446 std::fs::create_dir_all(&artifacts).unwrap();
6447 std::fs::write(artifacts.join("eviction-review-1.json"), "{}").unwrap();
6448 }
6449 }
6450 for cycle in 1..=2 {
6455 for (pass, expected_next) in [65, 129, 131].into_iter().enumerate() {
6456 let mut report = Report::default();
6457 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6458 assert!(report.failures.is_empty(), "{report:?}");
6459 assert_eq!(report.evicted_copies, usize::from(cycle == 1 && pass == 2));
6460 assert_eq!(review.exists(), cycle == 1 && pass < 2);
6461 let census = source_census(&f.home);
6462 assert_eq!(census["current"]["next"], expected_next);
6463 assert_eq!(census["current"]["completed_at"].is_null(), pass < 2);
6464 assert_eq!(census["current"]["number"], cycle);
6465 assert_eq!(
6466 census["current"]["runs"]["20260101-000000-0000"]["root"]["outcome"],
6467 "error"
6468 );
6469 assert_eq!(
6470 census["current"]["runs"]["20260101-000000-0001"]["root"]["detail"],
6471 "root absent and no eviction records; nothing to reclaim"
6472 );
6473 if pass < 2 {
6474 let next_id = format!("20260101-000000-{expected_next:04x}");
6475 assert_eq!(
6476 census["current"]["runs"][next_id]["root"]["outcome"],
6477 "not-yet-inspected"
6478 );
6479 }
6480 }
6481 }
6482 }
6483
6484 #[tokio::test]
6485 async fn source_eviction_scan_reaches_old_seats_despite_empty_roots_and_timestamp_churn() {
6486 let f = Fx::new().await;
6487 let mut state = inactive_source(&f, "ffff", true).await;
6488 let review = reviewer(&f, &state, &f.base).await;
6489 let mut history = Vec::new();
6492 for i in 0..130 {
6493 let mut old = state.clone();
6494 old.id = format!("20260101-000000-{i:04x}");
6495 old.candidates.clear();
6496 old.save_under(&f.home).unwrap();
6497 std::fs::create_dir_all(old.worktree_root()).unwrap();
6498 history.push(old);
6499 }
6500 state
6501 .save_under_at(
6502 &f.home,
6503 Timestamp::now() + jiff::SignedDuration::from_secs(10),
6504 )
6505 .unwrap();
6506 for pass in 0..3 {
6507 let mut report = Report::default();
6508 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6509 assert!(report.failures.is_empty(), "{report:?}");
6510 let census = source_census(&f.home);
6511 assert_eq!(
6512 census["current"]["next"].as_u64().unwrap(),
6513 ((pass + 1) * 64).min(131)
6514 );
6515 if pass < 2 {
6516 assert!(review.exists());
6517 assert!(census["current"]["completed_at"].is_null());
6518 assert_eq!(
6519 census["current"]["runs"][&state.id]["root"]["outcome"],
6520 "not-yet-inspected"
6521 );
6522 }
6523 for old in history.iter_mut().take(64) {
6526 old.updated_at = Timestamp::now();
6527 old.save_under(&f.home).unwrap();
6528 }
6529 if pass == 1 {
6530 let older = state.updated_at - jiff::SignedDuration::from_secs(100);
6537 for old in history.iter_mut().take(128) {
6538 if f.home.join("runs").join(&old.id).exists() {
6539 old.save_under_at(&f.home, older).unwrap();
6540 }
6541 }
6542 history[127]
6543 .save_under_at(&f.home, older - jiff::SignedDuration::from_secs(100))
6544 .unwrap();
6545 let mut legacy: Vec<_> = history
6546 .iter()
6547 .filter(|r| f.home.join("runs").join(&r.id).exists())
6548 .cloned()
6549 .collect();
6550 legacy.push(state.clone());
6551 legacy.sort_by_key(|r| r.updated_at);
6552 let index = legacy.iter().position(|r| r.id == history[127].id).unwrap();
6553 let len = legacy.len();
6554 legacy.rotate_left((index + 1) % len);
6555 assert!(legacy.iter().take(64).all(|r| r.id != state.id));
6556 }
6557 if pass == 0 {
6558 std::fs::remove_dir_all(f.home.join("runs").join(&history[64].id)).unwrap();
6559 let mut added = state.clone();
6560 added.id = "20260101-000000-fffe".into();
6561 added.candidates.clear();
6562 added.save_under(&f.home).unwrap();
6563 }
6564 }
6565 assert!(!review.exists());
6566 let census = source_census(&f.home);
6567 assert!(!census["current"]["completed_at"].is_null());
6568 assert_eq!(
6569 census["current"]["runs"][&state.id]["seats"]["review-1"]["outcome"],
6570 "evicted"
6571 );
6572 assert_eq!(
6573 census["current"]["runs"][&history[64].id]["root"]["outcome"],
6574 "error"
6575 );
6576 let mut next = Report::default();
6577 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
6578 let wrapped = source_census(&f.home);
6579 assert_eq!(wrapped["current"]["number"], 2);
6580 assert_eq!(wrapped["previous"], census["current"]);
6581 assert!(
6582 wrapped["current"]["runs"]
6583 .get("20260101-000000-fffe")
6584 .is_some()
6585 );
6586 }
6587
6588 #[tokio::test]
6589 async fn source_eviction_scan_resumes_within_run_and_bounds_failed_seats() {
6590 fn fail_one(path: &Path) -> Result<()> {
6591 ensure!(
6592 path.file_name().unwrap() != "review-2",
6593 "injected checkpoint failure"
6594 );
6595 Ok(())
6596 }
6597 let f = Fx::new().await;
6598 let state = inactive_source(&f, "ef20", true).await;
6599 let mut paths = Vec::new();
6600 for i in 1..=8 {
6601 let path = state.worktree_root().join(format!("review-{i}"));
6602 git::worktree_add_detached(&f.repo, &path, &f.base)
6603 .await
6604 .unwrap();
6605 paths.push(path);
6606 }
6607 std::fs::write(paths[2].join("owner-data"), "retain me").unwrap();
6608 let mut first = Report::default();
6609 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut first, fail_one).await;
6610 assert_eq!(first.evicted_copies, 3, "{first:?}");
6611 let census = source_census(&f.home);
6612 let seats = &census["current"]["runs"][&state.id]["seats"];
6613 assert_eq!(census["current"]["after_seat"], "review-5");
6614 assert_eq!(seats["review-2"]["outcome"], "error");
6615 assert_eq!(seats["review-2"]["stage"], "unlink");
6616 assert_eq!(seats["review-3"]["outcome"], "protected");
6617 assert_eq!(seats["review-6"]["outcome"], "not-yet-inspected");
6618 assert!(first.source_unlinked_logical_bytes > 0);
6619 assert!(first.source_evidence_added_logical_bytes > 0);
6620 let mut second = Report::default();
6621 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
6622 assert_eq!(second.evicted_copies, 3, "{second:?}");
6623 assert!(paths[1].exists(), "failed seat waits until next cycle");
6624 assert!(paths[2].exists());
6625 assert!(!source_census(&f.home)["current"]["completed_at"].is_null());
6626 let mut third = Report::default();
6627 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut third).await;
6628 assert_eq!(third.evicted_copies, 1, "{third:?}");
6629 assert!(!paths[1].exists());
6630 assert_eq!(
6631 std::fs::read_to_string(paths[2].join("owner-data")).unwrap(),
6632 "retain me"
6633 );
6634 let shown = seat_inspections(&f.home, &state.id).unwrap().join("\n");
6635 assert!(
6636 shown.contains("cycle=1 complete=true review-2 error stage=unlink"),
6637 "{shown}"
6638 );
6639 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6640 let destination = f.bay.join("restored-scan-seat");
6641 restore_seat(&f.home, &state.id, &f.base, &destination)
6642 .await
6643 .unwrap();
6644 assert_eq!(git::rev_parse(&destination, "HEAD").await.unwrap(), f.base);
6645 }
6646
6647 #[tokio::test]
6648 async fn source_eviction_scan_continues_after_archive_failure_and_busy_output() {
6649 let f = Fx::new().await;
6650 let failed = inactive_source(&f, "ef30", true).await;
6651 let failed_seat = reviewer(&f, &failed, &f.base).await;
6652 let evidence = f
6653 .home
6654 .join("runs")
6655 .join(&failed.id)
6656 .join("artifacts/source-evidence");
6657 std::fs::create_dir_all(&evidence).unwrap();
6658 std::fs::File::create(evidence.join("reserved-evidence"))
6659 .unwrap()
6660 .set_len(64 * 1024 * 1024)
6661 .unwrap();
6662 let state = inactive_source(&f, "ef31", true).await;
6663 let busy = reviewer(&f, &state, &f.base).await;
6664 let available = state.worktree_root().join("review-2");
6665 git::worktree_add_detached(&f.repo, &available, &f.base)
6666 .await
6667 .unwrap();
6668 let output = OutputLease::take(&busy, true).unwrap();
6669 let mut report = Report::default();
6670 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6671 assert_eq!(report.evicted_copies, 1, "{report:?}");
6672 assert!(failed_seat.exists());
6673 assert!(busy.exists());
6674 assert!(!available.exists());
6675 let census = source_census(&f.home);
6676 assert!(!census["current"]["completed_at"].is_null());
6677 assert_eq!(
6678 census["current"]["runs"][&failed.id]["seats"]["review-1"]["outcome"],
6679 "error"
6680 );
6681 assert_eq!(
6682 census["current"]["runs"][&state.id]["seats"]["review-1"]["outcome"],
6683 "protected"
6684 );
6685 drop(output);
6686 let mut retry = Report::default();
6687 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6688 assert_eq!(retry.evicted_copies, 1, "{retry:?}");
6689 assert!(!busy.exists());
6690 assert!(failed_seat.exists());
6691 }
6692
6693 #[tokio::test]
6694 async fn source_eviction_scan_checkpoint_failure_stops_before_later_seats() {
6695 fn block_checkpoint(path: &Path) -> Result<()> {
6696 let home = path
6697 .parent()
6698 .unwrap()
6699 .parent()
6700 .unwrap()
6701 .parent()
6702 .unwrap()
6703 .join("home");
6704 let cursor = home.join("source-eviction-cursor");
6705 std::fs::rename(&cursor, home.join("saved-cursor"))?;
6706 std::fs::create_dir(&cursor)?;
6707 Ok(())
6708 }
6709 let f = Fx::new().await;
6710 let state = inactive_source(&f, "ef32", true).await;
6711 let first = reviewer(&f, &state, &f.base).await;
6712 let second = state.worktree_root().join("review-2");
6713 git::worktree_add_detached(&f.repo, &second, &f.base)
6714 .await
6715 .unwrap();
6716 let mut report = Report::default();
6717 copies::evict_with(
6718 &f.home,
6719 &Fx::disk(),
6720 Fx::now(),
6721 &mut report,
6722 block_checkpoint,
6723 )
6724 .await;
6725 assert_eq!(report.evicted_copies, 1, "{report:?}");
6726 assert!(!first.exists());
6727 assert!(second.exists());
6728 assert!(
6729 report
6730 .failures
6731 .iter()
6732 .any(|e| e.contains("persist source scan census"))
6733 );
6734 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "error");
6735 std::fs::remove_dir(f.home.join("source-eviction-cursor")).unwrap();
6736 std::fs::rename(
6737 f.home.join("saved-cursor"),
6738 f.home.join("source-eviction-cursor"),
6739 )
6740 .unwrap();
6741 let mut retry = Report::default();
6742 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6743 assert_eq!(retry.evicted_copies, 1, "{retry:?}");
6744 assert!(!second.exists());
6745 }
6746
6747 #[tokio::test]
6748 async fn source_eviction_scan_reports_invalid_cursor_and_global_ownership() {
6749 let f = Fx::new().await;
6750 let state = inactive_source(&f, "ef21", true).await;
6751 let review = reviewer(&f, &state, &f.base).await;
6752 let cursor = f.home.join("source-eviction-cursor");
6753 std::fs::write(&cursor, "broken cursor").unwrap();
6754 let mut report = Report::default();
6755 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6756 assert_eq!(std::fs::read_to_string(&cursor).unwrap(), "broken cursor");
6757 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "error");
6758 assert!(review.exists());
6759 std::fs::write(&cursor, "20260101-000000-abcd").unwrap();
6761 let mut disabled = Fx::disk();
6762 disabled.auto_fold = false;
6763 let mut report = Report::default();
6764 copies::evict(&f.home, &disabled, Fx::now(), &mut report).await;
6765 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "disabled");
6766 let record = f.home.join("runs").join(&state.id).join("run.json");
6767 let original = std::fs::read(&record).unwrap();
6768 std::fs::write(&record, "invalid ownership").unwrap();
6769 let mut report = Report::default();
6770 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6771 assert!(
6772 report
6773 .failures
6774 .iter()
6775 .any(|e| e.contains("global run ownership validation"))
6776 );
6777 assert!(review.exists());
6778 std::fs::write(record, original).unwrap();
6779 let mut report = Report::default();
6780 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6781 assert_eq!(report.evicted_copies, 1, "{report:?}");
6782 assert_eq!(source_census(&f.home)["version"], 1);
6783 }
6784
6785 #[tokio::test]
6786 async fn source_eviction_batches_reproducible_seats_without_dropping_task_history() {
6787 let f = Fx::new().await;
6788 let mut runs = Vec::new();
6789 let mut paths = Vec::new();
6790 for tag in ["ec10", "ec11", "ec12", "ec13"] {
6791 let state = inactive_source(&f, tag, true).await;
6792 for name in ["judge-1", "advisor-1"] {
6793 let path = state.worktree_root().join(name);
6794 git::worktree_add_detached(&f.repo, &path, &f.base)
6795 .await
6796 .unwrap();
6797 paths.push(path);
6798 }
6799 runs.push(state);
6800 }
6801 let task = f.task(&runs);
6802 let mut report = Report::default();
6803 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6804 assert_eq!(report.evicted_copies, 5, "{report:?}");
6805 let mut next = Report::default();
6806 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
6807 assert_eq!(next.evicted_copies, 3, "{next:?}");
6808 assert!(paths.iter().all(|p| !p.exists()));
6809 assert!(runs.iter().all(|r| r.candidates[0].worktree.exists()));
6810 assert_eq!(
6811 crate::queue::Queue::at(f.home.join("queue"))
6812 .get(&task.id)
6813 .unwrap()
6814 .runs,
6815 task.runs
6816 );
6817 let limited = Fx::new().await;
6818 let state = inactive_source(&limited, "ec15", true).await;
6819 let copy = reviewer(&limited, &state, &limited.base).await;
6820 let evidence = limited
6821 .home
6822 .join("runs")
6823 .join(&state.id)
6824 .join("artifacts/source-evidence");
6825 std::fs::create_dir_all(&evidence).unwrap();
6826 let retained = evidence.join("reserved-evidence");
6827 std::fs::File::create(&retained)
6828 .unwrap()
6829 .set_len(64 * 1024 * 1024)
6830 .unwrap();
6831 let mut blocked = Report::default();
6832 copies::evict(&limited.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6833 assert_eq!(blocked.evicted_copies, 0, "{blocked:?}");
6834 assert!(copy.exists());
6835 assert_eq!(
6836 std::fs::metadata(&retained).unwrap().len(),
6837 64 * 1024 * 1024
6838 );
6839 assert!(
6840 blocked
6841 .retained
6842 .iter()
6843 .any(|r| r.contains("per-run storage limit")),
6844 "{blocked:?}"
6845 );
6846 }
6847 #[tokio::test]
6848 async fn source_eviction_migrates_historical_records_without_rewriting_them() {
6849 let f = Fx::new().await;
6850 let mut state = inactive_source(&f, "ec14", true).await;
6851 historical_review(&mut state, &f.base);
6852 historical_review(&mut state, &f.base);
6853 state.status = RunStatus::Superseded;
6854 state.save_under(&f.home).unwrap();
6855 let review = reviewer(&f, &state, &f.base).await;
6856 let task = f.task(std::slice::from_ref(&state));
6857 let path = f.home.join("runs").join(&state.id).join("run.json");
6858 let mut raw: serde_json::Value =
6859 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6860 raw["schema"] = 10.into();
6861 raw.as_object_mut().unwrap().remove("output_activity_at");
6862 raw["resource_wait"] = serde_json::Value::Null;
6865 raw["resource_block"] = serde_json::Value::Null;
6866 raw["driver_claimed"] = serde_json::Value::Null;
6867 raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6868 let graph = raw["config"]["graph"].as_object_mut().unwrap();
6870 let count = graph.remove("implementers").unwrap();
6871 graph.insert("candidates".into(), count);
6872 let queue = crate::queue::Queue::at(f.home.join("queue"));
6873 let task_path = queue.path_of(&task.id);
6874 let mut task_raw: serde_json::Value =
6875 serde_json::from_slice(&std::fs::read(&task_path).unwrap()).unwrap();
6876 task_raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
6877 let task_historical = serde_json::to_vec_pretty(&task_raw).unwrap();
6878 std::fs::write(&task_path, &task_historical).unwrap();
6879 let historical = serde_json::to_vec_pretty(&raw).unwrap();
6880 std::fs::write(&path, &historical).unwrap();
6881 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
6882 assert_eq!(report.evicted_copies, 1, "{report:?}");
6883 assert!(!review.exists());
6884 assert_eq!(std::fs::read(&path).unwrap(), historical);
6885 assert_eq!(std::fs::read(&task_path).unwrap(), task_historical);
6886 assert!(state.candidates[0].worktree.exists());
6887 assert_eq!(
6888 crate::queue::Queue::at(f.home.join("queue"))
6889 .get(&task.id)
6890 .unwrap()
6891 .runs,
6892 task.runs
6893 );
6894 }
6895 #[tokio::test]
6896 async fn source_eviction_archives_old_only_reflog_oid_after_unrewritten_deletion() {
6897 let f = Fx::new().await;
6898 let state = inactive_source(&f, "ed01", true).await;
6899 let review = reviewer(&f, &state, &f.base).await;
6900 std::fs::write(review.join("old-only"), b"old-only\0binary\xff").unwrap();
6901 git::git(&review, &["add", "."]).await.unwrap();
6902 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6903 let tip = git::git(
6904 &review,
6905 &[
6906 "commit-tree",
6907 &tree,
6908 "-p",
6909 &f.base,
6910 "-m",
6911 "temporary detached source",
6912 ],
6913 )
6914 .await
6915 .unwrap();
6916 git::git(&review, &["checkout", "--detach", &tip])
6917 .await
6918 .unwrap();
6919 git::git(&review, &["checkout", "--detach", &f.base])
6920 .await
6921 .unwrap();
6922 git::git(&review, &["reflog", "delete", "HEAD@{1}"])
6923 .await
6924 .unwrap();
6925 git::git(&review, &["update-ref", "-d", "ORIG_HEAD"])
6926 .await
6927 .unwrap();
6928 let shown = git::git(&review, &["reflog", "show", "--format=%H", "HEAD"])
6929 .await
6930 .unwrap();
6931 assert!(!shown.lines().any(|oid| oid == tip));
6932 let admin = PathBuf::from(
6933 git::git(&review, &["rev-parse", "--absolute-git-dir"])
6934 .await
6935 .unwrap(),
6936 );
6937 let raw = std::fs::read_to_string(admin.join("logs/HEAD")).unwrap();
6938 assert!(raw.lines().any(|line| line.starts_with(&format!("{tip} "))));
6939 fn interrupt(_: &Path) -> Result<()> {
6940 anyhow::bail!("fixture interruption before unlink")
6941 }
6942 let mut pending = Report::default();
6943 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut pending, interrupt).await;
6944 assert!(review.exists());
6945 let ledger = f
6946 .home
6947 .join("runs")
6948 .join(&state.id)
6949 .join("artifacts/eviction-review-1.json");
6950 let complete = std::fs::read(&ledger).unwrap();
6951 let mut incomplete: serde_json::Value = serde_json::from_slice(&complete).unwrap();
6952 incomplete["history"]
6953 .as_object_mut()
6954 .unwrap()
6955 .remove(&tip)
6956 .unwrap();
6957 std::fs::write(&ledger, serde_json::to_vec(&incomplete).unwrap()).unwrap();
6958 let mut blocked = Report::default();
6959 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6960 assert!(review.exists());
6961 assert!(
6962 blocked
6963 .retained
6964 .iter()
6965 .any(|reason| reason.contains("incomplete reflog checkpoint")),
6966 "{blocked:?}"
6967 );
6968 std::fs::write(&ledger, complete).unwrap();
6969 let mut report = Report::default();
6970 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6971 assert_eq!(report.evicted_copies, 1, "{report:?}");
6972 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6973 .await
6974 .unwrap();
6975 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6976 assert!(
6977 !git::git_raw(&f.repo, &["cat-file", "-e", &tip])
6978 .await
6979 .unwrap()
6980 .ok()
6981 );
6982 std::fs::remove_dir_all(&f.repo).unwrap();
6983 let destination = f._temp.path().join("restore-old-only");
6984 restore_seat(&f.home, &state.id, &tip, &destination)
6985 .await
6986 .unwrap();
6987 assert_eq!(
6988 std::fs::read(destination.join("old-only")).unwrap(),
6989 b"old-only\0binary\xff"
6990 );
6991 }
6992
6993 #[tokio::test]
6994 async fn source_eviction_crashed_driver_lease_does_not_block_cleanup_or_retry() {
6995 fn interrupt(_: &Path) -> Result<()> {
6996 anyhow::bail!("fixture interruption before unlink")
6997 }
6998 fn vanish(path: &Path) -> Result<()> {
6999 std::fs::remove_dir_all(path)?;
7000 anyhow::bail!("fixture interruption before unregistration")
7001 }
7002 for stage in ["ordinary", "intent", "missing-registration", "live-driver"] {
7003 let f = Fx::new().await;
7004 let mut state = inactive_source(&f, "ed02", true).await;
7005 state.driver_exited = false;
7006 state.driver_claimed = Some(true);
7007 state.driver_pid = Some(std::process::id());
7008 state.driver_started_at = crate::proc::process_started_at(std::process::id());
7009 state.begin_pass();
7010 state.status = RunStatus::Superseded;
7011 state.save_under(&f.home).unwrap();
7012 let review = reviewer(&f, &state, &f.base).await;
7013 let identity = serde_json::json!({"pid":state.driver_pid,"started_at":state.driver_started_at,
7016 "pass":state.passes.last().unwrap().started_at,"pass_index":state.passes.last().unwrap().index});
7017 std::fs::write(
7018 f.home.join("runs").join(&state.id).join("driver.json"),
7019 serde_json::to_vec(&identity).unwrap(),
7020 )
7021 .unwrap();
7022 let live = if stage == "live-driver" {
7023 Some(crate::driver::maintenance(&f.home, &state.id).unwrap())
7024 } else {
7025 None
7026 };
7027 assert_eq!(
7028 state.liveness_under(&f.home, false),
7029 if live.is_some() {
7030 Liveness::Live
7031 } else {
7032 Liveness::Dead
7033 }
7034 );
7035 let mut report = Report::default();
7036 if stage == "intent" || stage == "missing-registration" {
7037 copies::evict_with(
7038 &f.home,
7039 &Fx::disk(),
7040 Fx::now(),
7041 &mut report,
7042 if stage == "intent" { interrupt } else { vanish },
7043 )
7044 .await;
7045 assert_eq!(report.evicted_copies, 0, "{stage}: {report:?}");
7046 let mut retry = Report::default();
7047 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
7048 assert!(!review.exists(), "{stage}: {retry:?}");
7049 let ledger = f
7050 .home
7051 .join("runs")
7052 .join(&state.id)
7053 .join("artifacts/eviction-review-1.json");
7054 let receipt: serde_json::Value =
7055 serde_json::from_slice(&std::fs::read(ledger).unwrap()).unwrap();
7056 assert_eq!(receipt["phase"], "evicted", "{stage}: {retry:?}");
7057 } else {
7058 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7059 assert_eq!(
7060 report.evicted_copies,
7061 usize::from(live.is_none()),
7062 "{stage}: {report:?}"
7063 );
7064 assert_eq!(review.exists(), live.is_some());
7065 }
7066 drop(live);
7067 assert!(
7068 !RunState::load_under(&state.id, &f.home)
7069 .unwrap()
7070 .driver_exited
7071 );
7072 }
7073 }
7074}