1mod copies;
4pub mod reconcile;
5
6pub async fn source_diff(evidence: &Path) -> Result<Vec<u8>> {
8 crate::fold_evidence::validate_snapshot(evidence).await?;
9 Ok(std::fs::read(evidence.join("change.patch"))?)
10}
11
12pub async fn restore_source(evidence: &Path, destination: &Path) -> Result<()> {
14 crate::fold_evidence::restore_snapshot(evidence, destination).await
15}
16
17pub async fn seat_evidence(home: &Path, id: &str) -> Result<Vec<String>> {
19 copies::seat_evidence(home, id).await
20}
21
22pub async fn restore_seat(home: &Path, id: &str, head: &str, destination: &Path) -> Result<()> {
24 copies::restore_seat(home, id, head, destination).await
25}
26
27use crate::{
28 git,
29 proc::Quiet as _,
30 run::{Liveness, RunState},
31};
32use anyhow::{Context, Result, ensure};
33use jiff::Timestamp;
34use serde::{Deserialize, Serialize};
35use std::{
36 fs::{File, OpenOptions},
37 path::{Path, PathBuf},
38};
39
40#[derive(Debug)]
44pub(crate) struct OutputLease {
45 _file: File,
46 _activity: bool,
47}
48
49impl OutputLease {
50 #[cfg(unix)]
51 pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
52 use std::os::fd::AsRawFd;
53 self._file.as_raw_fd()
54 }
55
56 pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
57 if !worktree.join(".git").is_file() {
58 return Ok(None);
59 }
60 Self::take(worktree, false).map(Some)
61 }
62 pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
63 use std::hash::{Hash, Hasher};
64 let mut hash = std::collections::hash_map::DefaultHasher::new();
65 let mut normalized = cache.to_owned();
66 let mut missing = Vec::new();
67 while !normalized.exists() {
68 let Some(name) = normalized.file_name() else {
69 break;
70 };
71 missing.push(name.to_owned());
72 if !normalized.pop() {
73 break;
74 }
75 }
76 normalized = normalized.canonicalize().unwrap_or(normalized);
77 for part in missing.into_iter().rev() {
78 normalized.push(part);
79 }
80 normalized.hash(&mut hash);
81 let dir = home.join("cache-leases");
82 std::fs::create_dir_all(&dir)?;
83 Self::lock(
84 &dir.join(format!("outputs-{:x}.lock", hash.finish())),
85 exclusive,
86 )
87 }
88 #[cfg(unix)]
92 pub(crate) fn admit(home: &Path, paths: &[PathBuf], min_free: u64) -> Result<Vec<Self>> {
93 Self::admit_with(home, paths, min_free, crate::disk::free_bytes)
94 }
95
96 #[cfg(unix)]
97 pub(crate) fn admit_with(
98 home: &Path,
99 paths: &[PathBuf],
100 min_free: u64,
101 free_bytes: impl Fn(&Path) -> Result<u64>,
102 ) -> Result<Vec<Self>> {
103 use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
104 if min_free == 0 {
105 return Ok(Vec::new());
106 }
107 const RESERVE: u64 = 1024 * 1024 * 1024;
108 std::fs::create_dir_all(home)?;
109 let dir = home.canonicalize()?.join("disk-admission");
110 std::fs::create_dir_all(&dir)?;
111 ensure!(
112 plain_dir(&dir) && dir.canonicalize()? == dir,
113 "disk admission directory is not canonical"
114 );
115 let mut devices = std::collections::BTreeSet::new();
116 let mut reservations = Vec::new();
117 for path in paths {
118 let probe = path
119 .ancestors()
120 .find(|p| p.exists())
121 .context("disk admission path missing")?;
122 let device = std::fs::metadata(probe)?.dev();
123 if !devices.insert(device) {
124 continue;
125 }
126 let open = |path: &Path| -> Result<File> {
127 Ok(OpenOptions::new()
128 .read(true)
129 .write(true)
130 .create(true)
131 .truncate(false)
132 .custom_flags(libc::O_NOFOLLOW)
133 .open(path)?)
134 };
135 let admission = Self {
139 _file: open(&dir.join(format!("{device}.lock")))?,
140 _activity: false,
141 };
142 let started = std::time::Instant::now();
143 loop {
144 match crate::pr_head::try_lock_file(&admission._file) {
145 Ok(()) => break,
146 Err(error)
147 if matches!(
148 error.kind(),
149 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
150 ) && started.elapsed() < std::time::Duration::from_secs(1) =>
151 {
152 std::thread::sleep(std::time::Duration::from_millis(5));
153 }
154 Err(error) => {
155 return Err(error)
156 .context("disk admission busy or unavailable; retry without dispatch");
157 }
158 }
159 }
160 let mut active = 0u64;
161 let mut available = None;
162 for slot in 0..1024 {
163 let file = dir.join(format!("{device}-{slot}.reserve"));
164 if !file.exists() {
165 if available.is_none() {
166 available = Some(file);
167 }
168 continue;
169 }
170 let file = open(&file)?;
171 match crate::pr_head::try_lock_file(&file) {
172 Ok(()) if available.is_none() => {
173 available = Some(dir.join(format!("{device}-{slot}.reserve")))
174 }
175 Ok(()) => {}
176 Err(_) => active = active.saturating_add(RESERVE),
177 }
178 }
179 let free = free_bytes(probe)?;
180 let required = min_free.saturating_add(active).saturating_add(RESERVE);
181 ensure!(
182 free >= required,
183 "disk admission blocked: {free} bytes free; floor {min_free}, active writer reservations {active}, next writer reservation {RESERVE} bytes (estimates, not quotas)"
184 );
185 let file = open(&available.context("disk admission reservation slots exhausted")?)?;
186 crate::pr_head::try_lock_file(&file)?;
187 reservations.push(Self {
188 _file: file,
189 _activity: false,
190 });
191 }
192 Ok(reservations)
193 }
194
195 #[cfg(not(unix))]
196 pub(crate) fn admit(_home: &Path, _paths: &[PathBuf], _min_free: u64) -> Result<Vec<Self>> {
197 Ok(Vec::new())
199 }
200
201 pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
202 let out = std::process::Command::new("git")
203 .args(["rev-parse", "--absolute-git-dir"])
204 .current_dir(worktree)
205 .quiet()
206 .output()?;
207 ensure!(
208 out.status.success(),
209 "cannot identify output lease directory"
210 );
211 let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
212 Self::lock(&dir.join("magi-output.lock"), exclusive)
213 }
214 fn lock(path: &Path, exclusive: bool) -> Result<Self> {
215 let file = OpenOptions::new()
216 .create(true)
217 .truncate(false)
218 .read(true)
219 .write(true)
220 .open(path)
221 .with_context(|| format!("output lease: open {}", path.display()))?;
222 #[cfg(unix)]
223 {
224 use std::os::fd::AsRawFd;
225 let mode = if exclusive {
226 libc::LOCK_EX
227 } else {
228 libc::LOCK_SH
229 };
230 ensure!(
232 unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
233 "output lease: outputs are in use or being reclaimed: {}",
234 path.display()
235 );
236 }
237 #[cfg(not(unix))]
238 let _ = exclusive;
239 if !exclusive {
240 file.set_len(1)?;
242 file.set_modified(std::time::SystemTime::now())
243 .with_context(|| format!("record output activity {}", path.display()))?;
244 }
245 Ok(Self {
246 _file: file,
247 _activity: !exclusive,
248 })
249 }
250}
251#[cfg(unix)]
252impl Drop for OutputLease {
253 fn drop(&mut self) {
254 use std::os::fd::AsRawFd;
255 if self._activity
256 && let Err(error) = self._file.set_modified(std::time::SystemTime::now())
257 {
258 let _ = self._file.set_len(2);
261 tracing::warn!("could not record output lease release activity: {error}");
262 }
263 unsafe {
265 libc::flock(self._file.as_raw_fd(), libc::LOCK_UN);
266 }
267 }
268}
269
270#[derive(Default, Debug, Serialize, Deserialize)]
271#[serde(default)]
272pub(crate) struct Report {
273 pub checked_at: Option<Timestamp>,
274 pub limit_bytes: u64,
275 pub reclaim_budget_bytes: u64,
276 pub measured_bytes: u64,
278 pub freed_bytes: u64,
280 pub measured_allocated_file_bytes: Option<u64>,
281 pub proven_reclaimable_bytes: u64,
283 pub proven_reclaimable_allocated_bytes: Option<u64>,
286 pub protected_or_unproven_bytes: u64,
287 pub estimated_released_allocated_bytes: Option<u64>,
290 pub observed_home_free_change_bytes: Option<i64>,
293 pub retired: usize,
294 pub evicted_copies: usize,
295 pub removed_empty_roots: usize,
296 pub source_unlinked_logical_bytes: u64,
297 pub source_unlinked_allocated_bytes: Option<u64>,
298 pub source_evidence_added_logical_bytes: u64,
299 pub source_evidence_added_allocated_bytes: Option<u64>,
300 pub retained: Vec<String>,
301 pub failures: Vec<String>,
302}
303
304#[cfg(unix)]
305fn allocated_inventory(
306 root: &Path,
307 seen: &mut std::collections::BTreeSet<(u64, u64)>,
308) -> Result<u64> {
309 use std::os::unix::fs::MetadataExt;
310 ensure!(
311 plain_dir(root) && root.canonicalize()? == root,
312 "allocated inventory root is not canonical"
313 );
314 let mut dirs = vec![root.to_owned()];
315 let mut bytes = 0u64;
316 while let Some(dir) = dirs.pop() {
317 for entry in std::fs::read_dir(dir)? {
318 let path = entry?.path();
319 let metadata = std::fs::symlink_metadata(&path)?;
320 if metadata.is_dir() {
321 dirs.push(path);
322 } else if metadata.is_file() && seen.insert((metadata.dev(), metadata.ino())) {
323 bytes = bytes.saturating_add(metadata.blocks().saturating_mul(512));
324 }
325 }
326 }
327 Ok(bytes)
328}
329
330#[derive(Default)]
331struct ReleasedAllocation {
332 #[cfg(unix)]
333 files: std::collections::BTreeMap<PathBuf, (u64, u64, u64, u64)>,
334 #[cfg(unix)]
335 removed: std::collections::BTreeMap<(u64, u64), u64>,
336}
337
338impl ReleasedAllocation {
339 fn of(paths: &[PathBuf]) -> Result<Self> {
340 #[cfg(unix)]
341 {
342 use std::os::unix::fs::MetadataExt;
343 let mut files = std::collections::BTreeMap::new();
344 for path in paths {
345 let m = std::fs::symlink_metadata(path)?;
346 ensure!(m.is_file(), "allocation candidate is not regular");
347 files.insert(
348 path.clone(),
349 (m.dev(), m.ino(), m.nlink(), m.blocks().saturating_mul(512)),
350 );
351 }
352 Ok(Self {
353 files,
354 removed: Default::default(),
355 })
356 }
357 #[cfg(not(unix))]
358 {
359 let _ = paths;
360 Ok(Self::default())
361 }
362 }
363 fn reclaimable_allocated_bytes(&self) -> Option<u64> {
364 #[cfg(unix)]
365 {
366 let mut inodes = std::collections::BTreeMap::new();
367 for &(device, inode, links, blocks) in self.files.values() {
368 let entry = inodes
369 .entry((device, inode))
370 .or_insert((0u64, links, blocks));
371 entry.0 += 1;
372 }
373 Some(
374 inodes
375 .values()
376 .filter(|(count, links, _)| count == links)
377 .fold(0u64, |sum, (_, _, blocks)| sum.saturating_add(*blocks)),
378 )
379 }
380 #[cfg(not(unix))]
381 {
382 None
383 }
384 }
385
386 fn unlinked(&mut self, path: &Path) -> Option<u64> {
387 #[cfg(unix)]
388 {
389 let (device, inode, links, blocks) = self.files[path];
390 let removed = self.removed.entry((device, inode)).or_default();
391 *removed += 1;
392 Some(if *removed == links { blocks } else { 0 })
393 }
394 #[cfg(not(unix))]
395 {
396 let _ = path;
397 None
398 }
399 }
400}
401
402fn plain_dir(path: &Path) -> bool {
403 std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
404}
405
406fn run_paths_missing(state: &RunState) -> Result<bool> {
409 for path in [state.worktree_root(), state.repo.clone()] {
410 match std::fs::metadata(&path) {
411 Ok(_) => {}
412 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
413 Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
414 }
415 }
416 Ok(false)
417}
418
419async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
421 let root = state.worktree_root();
422 if !plain_dir(&root) {
423 return Ok(Vec::new());
424 }
425 let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
426 registered_worktrees(state, &list)
427}
428
429fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
430 let root = state.worktree_root();
431 if !plain_dir(&root) {
432 return Ok(Vec::new());
433 }
434 let root = root.canonicalize()?;
435 let mut paths = Vec::new();
436 for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
437 let path = PathBuf::from(path);
438 if plain_dir(&path)
439 && path.parent() == Some(root.as_path())
440 && path.canonicalize()? == path
441 && path.join(".git").is_file()
442 {
443 paths.push(path);
444 }
445 }
446 Ok(paths)
447}
448
449async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
452 validate_cargo_target(worktree, false).await
453}
454
455async fn validate_cargo_target(worktree: &Path, allow_missing_tag: bool) -> Result<PathBuf> {
456 let target = worktree.join("target");
457 ensure!(plain_dir(&target), "target absent or not a plain directory");
458 ensure!(target.canonicalize()? == target, "target path changed");
459 ensure!(
460 git::git(worktree, &["ls-files", "--", "target"])
461 .await?
462 .is_empty(),
463 "tracked target contents"
464 );
465 ensure!(
466 git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
467 .await?
468 .ok(),
469 "target is not ignored"
470 );
471 cache_tag(&target, allow_missing_tag)?;
472 let path = target.join(".rustc_info.json");
473 plain_file(&path, "rustc identity")?;
474 let info: serde_json::Value = serde_json::from_slice(
475 &std::fs::read(&path).with_context(|| format!("read rustc identity {}", path.display()))?,
476 )
477 .with_context(|| format!("parse rustc identity {}", path.display()))?;
478 ensure!(
479 info.get("rustc_fingerprint").is_some(),
480 "unknown Cargo identity {}",
481 path.display()
482 );
483 if allow_missing_tag && !target.join("CACHEDIR.TAG").exists() {
484 ensure!(
485 info.get("rustc_fingerprint")
486 .is_some_and(serde_json::Value::is_u64)
487 && info
488 .get("outputs")
489 .and_then(serde_json::Value::as_object)
490 .is_some_and(|outputs| outputs.values().any(|output| {
491 output.get("success").and_then(serde_json::Value::as_bool) == Some(true)
492 && output
493 .get("stdout")
494 .and_then(serde_json::Value::as_str)
495 .is_some_and(|stdout| stdout.starts_with("rustc "))
496 && output
497 .get("stderr")
498 .is_some_and(serde_json::Value::is_string)
499 }))
500 && info
501 .get("successes")
502 .is_some_and(serde_json::Value::is_object),
503 "missing-tag rustc identity structure invalid: {}",
504 path.display()
505 );
506 let mut profiles = 0;
507 for profile in ["debug", "release"] {
508 let dir = target.join(profile);
509 if !dir.exists() {
510 continue;
511 }
512 for subdir in [dir.clone(), dir.join("deps"), dir.join(".fingerprint")] {
513 ensure!(
514 plain_dir(&subdir) && subdir.canonicalize()? == subdir,
515 "missing-tag profile proof: {} is not a plain directory",
516 subdir.display()
517 );
518 }
519 plain_file(&dir.join(".cargo-lock"), "missing-tag Cargo lock")?;
520 profiles += 1;
521 }
522 ensure!(
523 profiles > 0,
524 "missing-tag proof: no Cargo profile in {}",
525 target.display()
526 );
527 }
528 for name in ["debug", "release"] {
529 let path = target.join(name);
530 match std::fs::symlink_metadata(&path) {
531 Ok(metadata) => ensure!(
532 metadata.is_dir() && path.canonicalize()? == path,
533 "profile validation: not a plain directory: {}",
534 path.display()
535 ),
536 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
537 Err(error) => {
538 return Err(error)
539 .with_context(|| format!("profile validation: inspect {}", path.display()));
540 }
541 }
542 }
543 Ok(target)
544}
545
546fn plain_file(path: &Path, stage: &str) -> Result<()> {
547 let meta = std::fs::symlink_metadata(path)
548 .with_context(|| format!("{stage}: inspect {}", path.display()))?;
549 ensure!(
550 meta.is_file() && path.canonicalize()? == path,
551 "{stage}: not a plain file: {}",
552 path.display()
553 );
554 Ok(())
555}
556
557fn cache_tag(target: &Path, allow_missing: bool) -> Result<()> {
558 let path = target.join("CACHEDIR.TAG");
559 match std::fs::symlink_metadata(&path) {
560 Err(error) if allow_missing && error.kind() == std::io::ErrorKind::NotFound => {
561 return Ok(());
562 }
563 result => {
564 result.with_context(|| format!("cache tag: inspect {}", path.display()))?;
565 }
566 }
567 plain_file(&path, "cache tag")?;
568 let tag = std::fs::read_to_string(&path)
569 .with_context(|| format!("cache tag: read {}", path.display()))?;
570 ensure!(
571 tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55"),
572 "unknown Cargo cache tag: {}",
573 path.display()
574 );
575 Ok(())
576}
577
578fn output_age(target: &Path, now: Timestamp, grace: u64) -> Result<()> {
582 let mut dirs = vec![target.to_owned()];
583 let mut entries = 0;
584 while let Some(dir) = dirs.pop() {
585 for entry in std::fs::read_dir(&dir)
586 .with_context(|| format!("output activity: list {}", dir.display()))?
587 {
588 let path = entry?.path();
589 entries += 1;
590 ensure!(
591 entries <= 100_000,
592 "output activity inventory too large: {}",
593 target.display()
594 );
595 let metadata = std::fs::symlink_metadata(&path)
596 .with_context(|| format!("output activity: inspect {}", path.display()))?;
597 if metadata.is_dir() {
598 if dir != target
600 || matches!(
601 path.file_name().and_then(|s| s.to_str()),
602 Some("debug" | "release")
603 )
604 {
605 dirs.push(path);
606 }
607 } else if metadata.is_file()
608 && (dir != target || path.file_name().is_some_and(|n| n == ".rustc_info.json"))
609 {
610 let at =
611 Timestamp::try_from(metadata.modified().with_context(|| {
612 format!("output activity: modified {}", path.display())
613 })?)?;
614 ensure!(
615 crate::clean::due(now, at, grace),
616 "output activity within grace period: {} modified {at} (now {now}, grace {grace}s)",
617 path.display()
618 );
619 }
620 }
621 }
622 Ok(())
623}
624
625pub(crate) fn artifact_signature(path: &Path) -> Result<bool> {
626 use std::io::Read as _;
627 let mut header = [0; 8];
628 let n = File::open(path)?.read(&mut header)?;
629 let header = &header[..n];
630 Ok(header.starts_with(b"\x7fELF")
631 || header.starts_with(b"!<arch>\n")
632 || header.starts_with(b"rust\0\0\0")
633 || header.starts_with(b"MZ\x90\0")
634 || [
635 [0xfe, 0xed, 0xfa, 0xce],
636 [0xce, 0xfa, 0xed, 0xfe],
637 [0xfe, 0xed, 0xfa, 0xcf],
638 [0xcf, 0xfa, 0xed, 0xfe],
639 [0xca, 0xfe, 0xba, 0xbe],
640 [0xca, 0xfe, 0xba, 0xbf],
641 ]
642 .iter()
643 .any(|magic| header.starts_with(magic)))
644}
645
646fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
650 let mut outputs = std::collections::BTreeSet::new();
651 for profile in ["debug", "release"] {
652 for dir in [target.join(profile), target.join(profile).join("deps")] {
653 if !plain_dir(&dir) || dir.canonicalize()? != dir {
654 continue;
655 }
656 for entry in std::fs::read_dir(&dir)? {
657 let path = entry?.path();
658 if path.extension().is_none_or(|ext| ext != "d")
659 || !std::fs::symlink_metadata(&path)?.is_file()
660 || std::fs::metadata(&path)?.len() > 1024 * 1024
661 || path.canonicalize()? != path
662 {
663 continue;
664 }
665 let body = std::fs::read_to_string(&path)?;
666 for line in body.lines() {
667 let Some((names, dependencies)) = line.split_once(": ") else {
668 continue;
669 };
670 if dependencies.trim().is_empty() {
671 continue;
672 }
673 let mut words = Vec::new();
676 let mut word = String::new();
677 let mut escaped = false;
678 for c in names.chars() {
679 if escaped {
680 word.push(c);
681 escaped = false;
682 } else if c == '\\' {
683 escaped = true;
684 } else if c.is_whitespace() {
685 if !word.is_empty() {
686 words.push(std::mem::take(&mut word));
687 }
688 } else {
689 word.push(c);
690 }
691 }
692 if escaped || word.contains(['$', '%']) {
693 continue;
694 }
695 if !word.is_empty() {
696 words.push(word);
697 }
698 for name in words {
699 let output = PathBuf::from(name);
700 let stem = path.file_stem().unwrap_or_default().to_string_lossy();
701 let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
702 let artifact_name =
703 output_stem == stem || output_stem == format!("lib{stem}");
704 let artifact_kind = output.extension().is_none_or(|ext| {
705 matches!(
706 ext.to_str(),
707 Some(
708 "rlib"
709 | "rmeta"
710 | "so"
711 | "dylib"
712 | "a"
713 | "lib"
714 | "dll"
715 | "exe"
716 | "pdb"
717 )
718 )
719 });
720 if artifact_name
721 && artifact_kind
722 && output.parent() == Some(dir.as_path())
723 && output.extension().is_none_or(|ext| ext != "d")
724 && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
725 && output.canonicalize().is_ok_and(|p| p == output)
726 && artifact_signature(&output)?
727 {
728 outputs.insert(output);
729 }
730 }
731 }
732 }
733 }
734 }
735 Ok(outputs.into_iter().collect())
736}
737
738#[derive(Serialize, Deserialize)]
739struct CargoResidue {
740 target: PathBuf,
741 files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
742}
743
744fn residue_files(
747 target: &Path,
748 outputs: &[PathBuf],
749) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
750 let mut files = std::collections::BTreeMap::new();
751 let mut dirs = vec![target.to_owned()];
752 let mut bytes = 0usize;
753 let mut entries = 0usize;
754 while let Some(dir) = dirs.pop() {
755 for entry in std::fs::read_dir(dir)? {
756 let path = entry?.path();
757 entries += 1;
758 ensure!(entries <= 4096, "residue inventory too large");
759 let metadata = std::fs::symlink_metadata(&path)?;
760 ensure!(
761 !metadata.file_type().is_symlink(),
762 "symlink in Cargo residue"
763 );
764 if metadata.is_dir() {
765 dirs.push(path);
766 } else {
767 ensure!(metadata.is_file(), "non-file in Cargo residue");
768 if outputs.contains(&path) {
769 continue;
770 }
771 ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
772 let body = std::fs::read(&path)?;
773 bytes = bytes.saturating_add(body.len());
774 ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
775 files.insert(path.strip_prefix(target)?.to_owned(), body);
776 }
777 }
778 }
779 Ok(files)
780}
781
782fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
783 Ok(home
784 .join("runs")
785 .join(id)
786 .join("build-residue")
787 .join(worktree.file_name().context("missing worktree name")?))
788}
789
790fn residue_root(target: &Path) -> Result<()> {
793 for entry in std::fs::read_dir(target)? {
794 let name = entry?.file_name();
795 ensure!(
796 matches!(
797 name.to_str(),
798 Some("debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json")
799 ),
800 "unrecognized Cargo residue root entry"
801 );
802 }
803 Ok(())
804}
805
806fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
807 cache_tag(target, false)?;
808 residue_root(target)?;
809 let files = residue_files(target, outputs)?;
810 for (relative, body) in &files {
811 let path = target.join(relative);
812 let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
813 let lock = matches!(
814 relative.to_str(),
815 Some("debug/.cargo-lock" | "release/.cargo-lock")
816 ) && body.is_empty();
817 let dep_info = path.extension().is_some_and(|ext| ext == "d")
818 && outputs.iter().any(|output| {
819 output.parent() == path.parent()
820 && (output.file_stem() == path.file_stem()
821 || output.file_stem().is_some_and(|stem| {
822 stem.to_string_lossy()
823 == format!(
824 "lib{}",
825 path.file_stem().unwrap_or_default().to_string_lossy()
826 )
827 }))
828 });
829 ensure!(identity || lock || dep_info, "unproven Cargo residue");
830 }
831 let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
832 std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
833 let tmp = proof.with_extension("tmp");
834 std::fs::write(
835 &tmp,
836 serde_json::to_vec(&CargoResidue {
837 target: target.to_owned(),
838 files,
839 })?,
840 )?;
841 std::fs::rename(tmp, proof)?;
842 Ok(())
843}
844
845pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
846 let target = cargo_target(worktree).await?;
847 residue_root(&target)?;
848 let locks = cargo_locks(&target)?;
849 let proof: CargoResidue =
850 serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
851 ensure!(
852 proof.target == target && proof.files == residue_files(&target, &[])?,
853 "Cargo residue changed or contains unproven files"
854 );
855 Ok(locks)
856}
857
858pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
860 let mut held = Vec::new();
861 for (profile, name) in ["debug", "release"].into_iter().flat_map(|profile| {
862 [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"].map(|name| (profile, name))
863 }) {
864 let lock = target.join(profile).join(name);
865 match std::fs::symlink_metadata(&lock) {
866 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
867 result => {
868 result.with_context(|| format!("Cargo lock: inspect {}", lock.display()))?;
869 }
870 }
871 {
872 plain_file(&lock, "Cargo lock")?;
873 let file = OpenOptions::new()
874 .read(true)
875 .write(true)
876 .open(&lock)
877 .with_context(|| format!("Cargo lock: open {}", lock.display()))?;
878 crate::pr_head::try_lock_file(&file)
879 .with_context(|| format!("Cargo build is active: {}", lock.display()))?;
880 held.push(file);
881 }
882 }
883 Ok(held)
884}
885
886pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
889 let home = home.to_owned();
890 let disk = disk.clone();
891 let runtime = tokio::runtime::Handle::current();
892 tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
893 .await
894 .unwrap_or_else(|e| Report {
895 failures: vec![format!("retention worker failed: {e}")],
896 ..Report::default()
897 })
898}
899
900#[derive(Default, Serialize, Deserialize)]
901struct Measurement {
902 at: Option<Timestamp>,
903 bytes: u64,
904}
905
906async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
907 let mut report = Report {
908 checked_at: Some(now),
909 limit_bytes: disk.worktree_limit_bytes,
910 reclaim_budget_bytes: disk.worktree_reclaim_bytes,
911 ..Report::default()
912 };
913 if !cfg!(unix) {
914 report
915 .failures
916 .push("Automatic output reclamation requires Unix process containment".into());
917 return report;
918 }
919 let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
920 return report;
921 };
922 let mut targets = Vec::new();
923 let mut under_pressure = false;
924 let mut registrations = std::collections::HashMap::new();
925 let measurements_path = home.join("output-measurements.json");
926 let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
927 std::fs::read(&measurements_path)
928 .ok()
929 .and_then(|b| serde_json::from_slice(&b).ok())
930 .unwrap_or_default();
931 let mut observed = std::collections::BTreeSet::new();
932 let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
933 .into_iter()
934 .flatten()
935 .flatten()
936 .map(|e| e.file_name().to_string_lossy().into_owned())
937 .collect();
938 ids.sort();
939 for id in ids {
940 if !crate::run::is_run_id(&id) {
941 continue;
942 }
943 let state = match RunState::load_under(&id, home) {
944 Ok(state) => state,
945 Err(e) => {
946 report
947 .retained
948 .push(format!("{id}: unreadable run ownership: {e:#}"));
949 continue;
950 }
951 };
952 if disk.worktree_limit_bytes == 0 {
953 continue;
954 }
955 match run_paths_missing(&state) {
956 Ok(true) => continue,
957 Ok(false) => {}
958 Err(e) => {
959 report.failures.push(format!("{id}: {e:#}"));
960 continue;
961 }
962 }
963 let list = match registrations.entry(state.repo.clone()) {
964 std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
965 std::collections::hash_map::Entry::Vacant(entry) => {
966 match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
967 Ok(list) => entry.insert(list),
968 Err(e) => {
969 if matches!(run_paths_missing(&state), Ok(true)) {
972 continue;
973 }
974 report.failures.push(format!("{id}: {e:#}"));
975 continue;
976 }
977 }
978 }
979 };
980 let paths = match registered_worktrees(&state, list) {
981 Ok(paths) => paths,
982 Err(e) => {
983 report.failures.push(format!("{id}: {e:#}"));
984 continue;
985 }
986 };
987 for path in paths {
988 if !path.join("target").exists() {
989 continue;
990 }
991 let target = path.join("target");
992 observed.insert(target.clone());
993 let pressure =
994 crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
995 under_pressure |= pressure;
996 let measurement = measurements.entry(target).or_default();
997 if pressure
998 || measurement
999 .at
1000 .is_none_or(|at| crate::clean::due(now, at, 300))
1001 {
1002 measurement.bytes = crate::disk::dir_size(&path.join("target"));
1003 measurement.at = Some(now);
1004 }
1005 let size = measurement.bytes;
1006 report.measured_bytes = report.measured_bytes.saturating_add(size);
1007 targets.push((state.output_activity(), id.clone(), path, size));
1008 }
1009 }
1010 if under_pressure {
1011 targets.sort_by_key(|t| (std::cmp::Reverse(t.3), t.0));
1014 } else {
1015 targets.sort_by_key(|t| t.0);
1016 }
1017 let free_before = crate::disk::free_bytes(home).ok();
1018 #[cfg(unix)]
1019 {
1020 let mut seen = std::collections::BTreeSet::new();
1021 report.measured_allocated_file_bytes = Some(0);
1022 report.estimated_released_allocated_bytes = Some(0);
1023 report.proven_reclaimable_allocated_bytes = Some(0);
1024 for (_, _, path, _) in &targets {
1025 report.measured_allocated_file_bytes = report
1026 .measured_allocated_file_bytes
1027 .zip(allocated_inventory(&path.join("target"), &mut seen).ok())
1028 .map(|(a, b)| a.saturating_add(b));
1029 }
1030 }
1031 let mut remaining = report.measured_bytes;
1032 let pressure_paths: Vec<_> = targets.iter().map(|t| t.2.clone()).collect();
1033 let mut deletion_started = false;
1034 let mut oversized_started = false;
1035 for (updated, id, path, _) in targets {
1036 if disk.worktree_limit_bytes == 0
1037 || remaining <= disk.worktree_limit_bytes
1038 && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
1039 {
1040 report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
1041 continue;
1042 }
1043 if oversized_started || report.freed_bytes >= disk.worktree_reclaim_bytes {
1044 report
1045 .retained
1046 .push(format!("{id}: per-pass byte budget spent"));
1047 continue;
1048 }
1049 let result = async {
1050 ensure!(
1051 crate::clean::due(now, updated, disk.fold_grace_secs),
1052 "execution activity within grace period: {updated} (now {now}, grace {}s)", disk.fold_grace_secs
1053 );
1054 let _owner = crate::pr_head::owner_lock(home, &id)?;
1055 let state = RunState::load_under(&id, home)?;
1056 ensure!(
1057 state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
1058 == Liveness::Dead,
1059 "run live or ownership unknown"
1060 );
1061 ensure!(crate::clean::due(now, state.output_activity(), disk.fold_grace_secs), "execution activity after inventory: {}", state.output_activity());
1062 let _driver = crate::driver::maintenance(home, &id)?;
1063 let _outputs = OutputLease::take(&path, true)?;
1064 ensure!(
1065 worktrees(&state).await?.contains(&path),
1066 "worktree registration changed"
1067 );
1068 let target = validate_cargo_target(&path, true).await
1069 .with_context(|| format!("stage validate-target: {}", path.join("target").display()))?;
1070 let _cargo = cargo_locks(&target)
1071 .with_context(|| format!("stage cargo-locks: {}", target.display()))?;
1072 validate_cargo_target(&path, true).await
1075 .with_context(|| format!("stage validate-target (locked): {}", target.display()))?;
1076 let lease_meta = _outputs._file.metadata().context("output lease activity: metadata")?;
1077 if lease_meta.len() > 0 {
1078 ensure!(lease_meta.len() == 1, "output lease activity witness is unreadable or incomplete for {}", path.display());
1079 let lease_at = Timestamp::try_from(lease_meta.modified().context("output lease activity: modified")?)?;
1080 ensure!(crate::clean::due(now, lease_at, disk.fold_grace_secs),
1081 "output lease activity within grace period: {lease_at} (now {now}, grace {}s)", disk.fold_grace_secs);
1082 }
1083 output_age(&target, now, disk.fold_grace_secs)
1084 .with_context(|| format!("stage output-age: {}", target.display()))?;
1085 let mut outputs = declared_outputs(&target)
1086 .with_context(|| format!("stage declared-outputs: {}", target.display()))?;
1087 let incremental = match crate::incremental::candidates(&path) {
1088 Ok(candidates) => candidates,
1089 Err(error) => {
1090 report.retained.push(format!("{id}: incremental cache retained: {error:#}"));
1091 crate::incremental::Reclaim::default()
1092 }
1093 };
1094 let incremental_files = incremental.files.clone();
1095 outputs.extend(incremental_files.iter().cloned());
1096 outputs.sort();
1097 outputs.dedup();
1098 ensure!(!outputs.is_empty(), "no dep-info-proven outputs or provenance-proven incremental caches; legacy incremental/build and unknown files retained");
1099 let sizes = outputs.iter().map(|p| std::fs::metadata(p).map(|m| m.len())
1100 .with_context(|| format!("stage measure: {}", p.display())))
1101 .collect::<Result<Vec<_>>>()?;
1102 let current_size = sizes.iter().fold(0u64, |bytes, size| bytes.saturating_add(*size));
1103 report.proven_reclaimable_bytes = report.proven_reclaimable_bytes.saturating_add(current_size);
1104 let mut allocation = ReleasedAllocation::of(&outputs)?;
1105 report.proven_reclaimable_allocated_bytes = report.proven_reclaimable_allocated_bytes
1106 .zip(allocation.reclaimable_allocated_bytes()).map(|(a, b)| a.saturating_add(b));
1107 let exceeds_budget = current_size > disk.worktree_reclaim_bytes.saturating_sub(report.freed_bytes);
1108 ensure!(!deletion_started || !exceeds_budget,
1112 "declared outputs exceed remaining per-pass byte budget");
1113 let ledger = home.join("runs").join(&id).join("build-outputs.json");
1116 std::fs::write(
1117 &ledger,
1118 serde_json::to_vec(
1119 &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","identity":if target.join("CACHEDIR.TAG").exists() { "cache-tag" } else { "rustc-profile-dep-info" },"files":outputs}),
1120 )?,
1121 ).with_context(|| format!("stage ledger: {}", ledger.display()))?;
1122 if !incremental_files.is_empty() {
1125 report.retained.push(format!("{id}: incremental proof does not authorize retirement"));
1126 } else if let Err(error) = record_residue(home, &id, &target, &outputs) {
1127 report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
1128 }
1129 ensure!(worktrees(&state).await?.contains(&path),
1130 "worktree registration changed before deletion");
1131 deletion_started = true;
1132 oversized_started = exceeds_budget;
1133 measurements.remove(&target);
1134 for (output, size) in outputs.iter().zip(sizes) {
1135 std::fs::remove_file(output).with_context(|| format!("stage remove: {}", output.display()))?;
1136 report.estimated_released_allocated_bytes = report.estimated_released_allocated_bytes
1137 .zip(allocation.unlinked(output)).map(|(a, b)| a.saturating_add(b));
1138 report.freed_bytes = report.freed_bytes.saturating_add(size);
1140 remaining = remaining.saturating_sub(size);
1141 }
1142 incremental.finish().context("finish incremental reclamation")?;
1143 report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
1144 Ok::<_, anyhow::Error>(())
1145 }
1146 .await;
1147 if let Err(e) = result {
1148 report.retained.push(format!(
1149 "{id}/{}: {e:#}",
1150 path.file_name().unwrap_or_default().to_string_lossy()
1151 ));
1152 }
1153 }
1154 measurements.retain(|path, _| observed.contains(path));
1155 if let Ok(body) = serde_json::to_vec(&measurements)
1156 && let Err(e) = std::fs::write(&measurements_path, body)
1157 {
1158 report.failures.push(format!("output measurements: {e}"));
1159 }
1160 copies::evict(home, disk, now, &mut report).await;
1161 retire_empty(home, disk, now, &mut report).await;
1162 report.protected_or_unproven_bytes = report
1163 .measured_bytes
1164 .saturating_sub(report.proven_reclaimable_bytes);
1165 let pressure_remaining = pressure_paths
1166 .iter()
1167 .any(|path| crate::disk::free_bytes(path).map_or(true, |free| free < disk.min_free_bytes));
1168 if !report.failures.is_empty()
1169 || disk.worktree_limit_bytes > 0
1170 && (remaining > disk.worktree_limit_bytes || pressure_remaining)
1171 {
1172 let details = report
1173 .retained
1174 .iter()
1175 .filter(|reason| {
1176 !reason.contains("within policy")
1177 && !reason.ends_with("unmanifested files and Cargo metadata retained")
1178 })
1179 .take(4)
1180 .chain(report.failures.iter().take(2))
1181 .cloned()
1182 .collect::<Vec<_>>()
1183 .join("; ");
1184 crate::notices::raise_in(
1185 home,
1186 crate::notices::Notice::warn(
1187 "disk:retention",
1188 format!(
1189 "Retention remains blocked by the disk cap, free-space floor or inspection failures (logical inventory {}B, proven reclaimable {}B, protected or unproven {}B, unlinked {}B). {} Retained/held targets: {details}. Inspect the cleanup report for active owners, retained recovery sources and failures.",
1190 report.measured_bytes,
1191 report.proven_reclaimable_bytes,
1192 report.protected_or_unproven_bytes,
1193 report.freed_bytes,
1194 if report.proven_reclaimable_bytes == 0 {
1195 "No inspected contents have deletion authority."
1196 } else {
1197 "Authorized reclamation was insufficient; remaining protections and budgets still apply."
1198 }
1199 ),
1200 ),
1201 );
1202 }
1203 report.observed_home_free_change_bytes = free_before
1204 .zip(crate::disk::free_bytes(home).ok())
1205 .map(|(before, after)| {
1206 (i128::from(after) - i128::from(before))
1207 .clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
1208 });
1209 let store = home.join("retention.json");
1210 if let Ok(body) = serde_json::to_vec_pretty(&report) {
1211 let tmp = home.join("retention.json.tmp");
1212 if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
1213 tracing::warn!("retention report could not be saved: {e}");
1214 }
1215 }
1216 tracing::info!(
1217 measured_bytes = report.measured_bytes,
1218 freed_bytes = report.freed_bytes,
1219 retired = report.retired,
1220 retained = report.retained.len(),
1221 failures = report.failures.len(),
1222 "retained worktree cleanup"
1223 );
1224 for reason in report.retained.iter().chain(&report.failures) {
1225 tracing::debug!(%reason, "worktree retained");
1226 }
1227 report
1228}
1229
1230async fn retire_empty(
1231 home: &Path,
1232 disk: &crate::config::Disk,
1233 now: Timestamp,
1234 report: &mut Report,
1235) {
1236 if !disk.auto_fold {
1237 return;
1238 }
1239 let queue = crate::queue::Queue::at(home.join("queue"));
1240 let questions = crate::ask::Questions::at(home.join("questions"));
1241 let open = questions.list();
1242 for listed in queue.list() {
1243 let Ok(_claim) = queue.claim(&listed.id) else {
1244 continue;
1245 };
1246 let Ok(task) = queue.get(&listed.id) else {
1247 continue;
1248 };
1249 if open
1250 .iter()
1251 .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
1252 {
1253 report.retained.push(format!(
1254 "{}: task has an unanswered owner decision",
1255 task.id
1256 ));
1257 continue;
1258 }
1259 let sources: Vec<_> = task
1260 .runs
1261 .iter()
1262 .collect::<std::collections::BTreeSet<_>>()
1263 .into_iter()
1264 .cloned()
1265 .collect();
1266 let mut useful = Vec::new();
1267 let mut selected = Vec::new();
1268 let mut public = Vec::new();
1269 let mut unreadable = false;
1270 for id in &sources {
1271 let Ok(state) = RunState::load_under(id, home) else {
1272 unreadable = true;
1273 continue;
1274 };
1275 if state.released() || state.retired_to.is_some() {
1276 continue;
1277 }
1278 if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
1279 selected.push(state.clone());
1280 }
1281 if state.pr.is_some() && state.pr_taken_over_by.is_none() {
1282 public.push(state.clone());
1283 }
1284 if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
1285 useful.push(state);
1286 }
1287 }
1288 let pool = if !selected.is_empty() {
1292 selected
1293 } else if !public.is_empty() {
1294 public
1295 } else {
1296 useful
1297 };
1298 if unreadable || pool.len() != 1 {
1299 report.retained.push(format!(
1300 "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
1301 task.id,
1302 sources.len()
1303 ));
1304 continue;
1305 }
1306 let authority = &pool[0];
1307 if authority.candidates.is_empty() {
1308 continue;
1309 }
1310 let latest = &authority.id;
1311 for id in &sources {
1312 if id == latest || report.retired >= 5 {
1313 continue;
1314 }
1315 if let Ok(state) = RunState::load_under(id, home)
1316 && state.retired_to.as_ref() == Some(latest)
1317 && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
1318 {
1319 continue;
1320 }
1321 let result = async {
1322 ensure!(
1323 !task
1324 .resume_override
1325 .as_ref()
1326 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
1327 "operator-pinned recovery run"
1328 );
1329 ensure!(
1330 !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
1331 "selected replacement recovery source"
1332 );
1333 let _owner = crate::pr_head::owner_lock(home, id)?;
1334 let mut state = RunState::load_under(id, home)?;
1335 ensure!(
1336 state.repo == authority.repo,
1337 "repository differs from recovery authority"
1338 );
1339 ensure!(
1340 !crate::pr_recovery::explicit_sources_retained(home, id),
1341 "explicitly retained recovery source"
1342 );
1343 ensure!(
1344 state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
1345 == Liveness::Dead,
1346 "run live or ownership unknown"
1347 );
1348 let _driver = crate::driver::maintenance(home, id)?;
1349 ensure!(
1350 crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
1351 "within grace period"
1352 );
1353 ensure!(
1354 !open.iter().any(|q| q.run == *id && q.status.open()),
1355 "unanswered owner decision"
1356 );
1357 ensure!(
1358 state.pr.is_none()
1359 && state.pr_head.is_none()
1360 && state.pr_recovery.is_none()
1361 && state.land_approval.is_none()
1362 && state.merge_decision.is_none()
1363 && state.reviews.is_empty()
1364 && state.recovery.is_none()
1365 && state.candidate_preservation.is_empty()
1366 && state.interrupted_implementations.is_empty()
1367 && !state.released(),
1368 "PR ownership or recovery checkpoint"
1369 );
1370 let proof = prove_empty(home, &state, latest).await?;
1371 commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
1372 for path in &proof.paths {
1375 ensure!(
1376 git::worktree_remove_clean(&state.repo, path).await?,
1377 "Git refused clean worktree removal"
1378 );
1379 }
1380 Ok::<_, anyhow::Error>(())
1381 }
1382 .await;
1383 match result {
1384 Ok(()) => report.retired += 1,
1385 Err(e) => report.retained.push(format!("{id}: {e:#}")),
1386 }
1387 }
1388 }
1389}
1390
1391struct EmptyProof {
1392 paths: Vec<PathBuf>,
1393 _outputs: Vec<OutputLease>,
1394 _cargo: Vec<File>,
1395}
1396
1397pub(crate) async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
1400 let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
1401 let mut files = Vec::new();
1402 let mut blobs = Vec::new();
1403 for record in tree.split('\0').filter(|s| !s.is_empty()) {
1404 let (header, name) = record.split_once('\t').context("unknown tree record")?;
1405 let parts: Vec<_> = header.split_whitespace().collect();
1406 ensure!(
1407 parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
1408 "unsupported tracked source kind"
1409 );
1410 let path = worktree.join(name);
1411 let meta = std::fs::symlink_metadata(&path)?;
1412 ensure!(
1413 meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
1414 "noncanonical tracked source"
1415 );
1416 #[cfg(unix)]
1417 {
1418 use std::os::unix::fs::PermissionsExt as _;
1419 ensure!(
1420 (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
1421 "tracked executable mode differs from original base"
1422 );
1423 }
1424 files.push(path.to_str().context("non UTF-8 source")?.to_owned());
1425 blobs.push(parts[2].to_owned());
1426 }
1427 for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
1428 let mut args = vec!["hash-object", "--no-filters", "--"];
1429 args.extend(paths.iter().map(String::as_str));
1430 let actual = git::git(worktree, &args).await?;
1431 ensure!(
1432 actual.lines().eq(expected.iter().map(String::as_str)),
1433 "tracked bytes differ from original base"
1434 );
1435 }
1436 Ok(())
1437}
1438
1439async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
1440 let id = &state.id;
1441 let paths = worktrees(state).await?;
1442 let mut guards = Vec::new();
1443 let mut cargo_guards = Vec::new();
1444 for path in &paths {
1445 guards.push(OutputLease::take(path, true)?);
1446 ensure!(
1447 git::rev_parse(path, "HEAD").await? == state.base_commit,
1448 "unique or unrecorded commits"
1449 );
1450 ensure!(
1451 git::git(
1452 path,
1453 &[
1454 "--no-optional-locks",
1455 "status",
1456 "--porcelain",
1457 "--untracked-files=all"
1458 ]
1459 )
1460 .await?
1461 .is_empty(),
1462 "tracked or untracked source changes"
1463 );
1464 ensure!(
1465 git::git(
1466 path,
1467 &[
1468 "--no-optional-locks",
1469 "status",
1470 "--porcelain",
1471 "--untracked-files=all",
1472 "--ignored",
1473 "--",
1474 ".",
1475 ":(exclude)target"
1476 ]
1477 )
1478 .await?
1479 .is_empty(),
1480 "source changes, ignored files or build outputs"
1481 );
1482 tracked_base(path, &state.base_commit).await?;
1483 match std::fs::symlink_metadata(path.join("target")) {
1484 Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
1485 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1486 Err(error) => return Err(error.into()),
1487 }
1488 for checkpoint in [
1489 "rebase-merge",
1490 "rebase-apply",
1491 "sequencer",
1492 "BISECT_LOG",
1493 "REVERT_HEAD",
1494 "AUTO_MERGE",
1495 "MERGE_MSG",
1496 "CHERRY_PICK_HEAD",
1497 "MERGE_HEAD",
1498 "REBASE_HEAD",
1499 ] {
1500 let location = git::git(
1501 path,
1502 &[
1503 "rev-parse",
1504 "--path-format=absolute",
1505 "--git-path",
1506 checkpoint,
1507 ],
1508 )
1509 .await?;
1510 ensure!(
1511 matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
1512 "retained Git recovery metadata: {checkpoint}"
1513 );
1514 }
1515 let origin_path = git::git(
1518 path,
1519 &[
1520 "rev-parse",
1521 "--path-format=absolute",
1522 "--git-path",
1523 "ORIG_HEAD",
1524 ],
1525 )
1526 .await?;
1527 match std::fs::symlink_metadata(origin_path.trim()) {
1528 Ok(meta) => {
1529 ensure!(
1530 meta.is_file() && !meta.file_type().is_symlink(),
1531 "unknown ORIG_HEAD artifact"
1532 );
1533 git::rev_parse(path, "ORIG_HEAD")
1534 .await
1535 .context("unreadable ORIG_HEAD")?;
1536 }
1537 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1538 Err(e) => return Err(e.into()),
1539 }
1540 if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1541 ensure!(
1542 git::git_raw(
1543 path,
1544 &["merge-base", "--is-ancestor", &original, &state.base_commit]
1545 )
1546 .await?
1547 .ok(),
1548 "ORIG_HEAD retains commits outside the original base"
1549 );
1550 }
1551 let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1552 for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1553 ensure!(
1554 git::git_raw(
1555 path,
1556 &["merge-base", "--is-ancestor", head, &state.base_commit]
1557 )
1558 .await?
1559 .ok(),
1560 "worktree reflog retains commits outside the original base"
1561 );
1562 }
1563 for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1564 ensure!(
1565 !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1566 .await?
1567 .ok(),
1568 "unfinished Git operation"
1569 );
1570 }
1571 }
1572 if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1575 return Ok(EmptyProof {
1576 paths,
1577 _outputs: guards,
1578 _cargo: cargo_guards,
1579 });
1580 }
1581 ensure!(!paths.is_empty(), "no worktrees to prove empty");
1582 for candidate in &state.candidates {
1583 ensure!(
1584 (paths.contains(&candidate.worktree)
1585 || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1586 && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1587 "candidate recovery source"
1588 );
1589 }
1590 let pattern = format!("refs/heads/magi/{}/", state.short());
1591 let branches = git::git(
1592 &state.repo,
1593 &["for-each-ref", "--format=%(objectname)", &pattern],
1594 )
1595 .await?;
1596 ensure!(
1597 branches.lines().all(|head| head == state.base_commit),
1598 "unrecorded candidate branch changes"
1599 );
1600 Ok(EmptyProof {
1601 paths,
1602 _outputs: guards,
1603 _cargo: cargo_guards,
1604 })
1605}
1606
1607fn commit_retirement(
1612 home: &Path,
1613 queue: &crate::queue::Queue,
1614 expected: &crate::queue::Task,
1615 state: &mut RunState,
1616 authority: &str,
1617 sources: &[String],
1618 now: Timestamp,
1619) -> Result<()> {
1620 let _write = queue.lock_task(&expected.id)?;
1621 let current = queue.get(&expected.id)?;
1622 ensure!(
1623 serde_json::to_value(¤t)? == serde_json::to_value(expected)?,
1624 "task changed during retirement proof; retaining worktrees for the current owner decision"
1625 );
1626 ensure!(
1627 !current
1628 .resume_override
1629 .as_ref()
1630 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
1631 && !current
1632 .replacement
1633 .as_ref()
1634 .is_some_and(|pin| pin.run == state.id),
1635 "current owner selected this recovery source"
1636 );
1637 ensure!(
1638 !crate::pr_recovery::explicit_sources_retained(home, &state.id),
1639 "explicitly retained recovery source"
1640 );
1641 ensure!(
1642 !crate::ask::Questions::at(home.join("questions"))
1643 .list()
1644 .iter()
1645 .any(|q| q.status.open()
1646 && (q.run == state.id
1647 || q.run == current.id
1648 || current.blocked_by.contains(&q.id))),
1649 "owner decision arrived during retirement proof"
1650 );
1651 let dir = home.join("recovery-lineages");
1652 std::fs::create_dir_all(&dir)?;
1653 let tmp = dir.join(format!("{}.tmp", current.id));
1654 let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
1655 "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
1656 std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
1657 std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
1658 state.retired_to = Some(authority.to_owned());
1659 state.event(
1660 "retention",
1661 format!(
1662 "empty attempt retired to {authority}; recovery sources: {}",
1663 sources.join(", ")
1664 ),
1665 );
1666 state.save_under(home)?;
1667 Ok(())
1669}
1670
1671pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64) -> String {
1675 if min_free == 0 {
1676 return std::future::pending().await;
1677 }
1678 loop {
1679 for path in &paths {
1680 let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
1681 match crate::disk::free_bytes(existing) {
1682 Ok(free) if free >= min_free / 2 => {}
1683 Ok(free) => {
1684 return format!(
1685 "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
1686 );
1687 }
1688 Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
1689 }
1690 }
1691 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
1692 }
1693}
1694
1695#[cfg(all(test, unix))]
1696mod tests {
1697 use super::*;
1698 use crate::{
1699 config::{Config, Disk},
1700 queue::{Source, Task},
1701 run::{Candidate, RunStatus},
1702 };
1703
1704 struct Fx {
1705 _temp: tempfile::TempDir,
1706 home: PathBuf,
1707 repo: PathBuf,
1708 bay: PathBuf,
1709 base: String,
1710 }
1711 impl Fx {
1712 async fn new() -> Self {
1713 let temp = tempfile::tempdir().unwrap();
1714 let root = temp.path().canonicalize().unwrap();
1715 let home = root.join("home");
1716 let repo = root.join("repo");
1717 let bay = root.join("worktrees");
1718 std::fs::create_dir_all(&repo).unwrap();
1719 std::fs::create_dir_all(home.join("runs")).unwrap();
1720 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
1721 git::git(&repo, &["config", "user.name", "Fixture"])
1722 .await
1723 .unwrap();
1724 git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
1725 .await
1726 .unwrap();
1727 git::git(&repo, &["config", "commit.gpgsign", "false"])
1728 .await
1729 .unwrap();
1730 std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
1731 std::fs::write(repo.join("source"), "base\n").unwrap();
1732 git::git(&repo, &["add", "."]).await.unwrap();
1733 git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
1734 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
1735 Self {
1736 _temp: temp,
1737 home,
1738 repo,
1739 bay,
1740 base,
1741 }
1742 }
1743 async fn run(&self, tag: &str, useful: bool) -> RunState {
1744 let mut cfg = Config::default();
1745 cfg.graph.worktree_root = Some(self.bay.clone());
1746 let mut state = RunState::new(
1747 self.repo.clone(),
1748 "main".into(),
1749 self.base.clone(),
1750 "task".into(),
1751 cfg,
1752 );
1753 state.id = format!("20260101-000000-{tag}");
1754 state.status = RunStatus::Blocked;
1755 state.driver_exited = true;
1756 let wt = state.worktree_root().join("cand-A");
1757 let branch = format!("magi/{tag}/A");
1758 git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
1759 .await
1760 .unwrap();
1761 if useful {
1762 std::fs::write(wt.join("source"), tag).unwrap();
1763 git::git(&wt, &["add", "."]).await.unwrap();
1764 git::git(&wt, &["commit", "-m", "candidate change"])
1765 .await
1766 .unwrap();
1767 }
1768 state.candidates.push(Candidate {
1769 index: 0,
1770 label: 'A',
1771 agent: "fixture".into(),
1772 branch,
1773 worktree: wt,
1774 summary: String::new(),
1775 stat: String::new(),
1776 files: usize::from(useful),
1777 commits: usize::from(useful),
1778 empty: !useful,
1779 failed: None,
1780 verified_noop: None,
1781 duration_ms: 0,
1782 folded: false,
1783 });
1784 state.save_under(&self.home).unwrap();
1785 state
1786 }
1787 fn disk() -> Disk {
1788 Disk {
1789 worktree_limit_bytes: 1,
1790 worktree_reclaim_bytes: 1024 * 1024,
1791 fold_grace_secs: 0,
1792 min_free_bytes: 0,
1793 ..Disk::default()
1794 }
1795 }
1796 fn target(wt: &Path) {
1797 let target = wt.join("target");
1798 std::fs::create_dir_all(target.join("debug")).unwrap();
1799 let mut artifact = [1; 200];
1800 artifact[..4].copy_from_slice(b"\x7fELF");
1801 std::fs::write(target.join("debug/output"), artifact).unwrap();
1802 let output = target
1803 .join("debug/output")
1804 .display()
1805 .to_string()
1806 .replace('\\', "\\\\")
1807 .replace(' ', "\\ ");
1808 std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
1809 std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
1810 std::fs::write(
1811 target.join("CACHEDIR.TAG"),
1812 "Signature: 8a477f597d28d172789f06886806bc55\n",
1813 )
1814 .unwrap();
1815 std::fs::write(
1816 target.join(".rustc_info.json"),
1817 "{\"rustc_fingerprint\":123}",
1818 )
1819 .unwrap();
1820 }
1821 fn now() -> Timestamp {
1822 Timestamp::now() + jiff::SignedDuration::from_secs(2)
1823 }
1824 fn task(&self, runs: &[RunState]) -> Task {
1825 let mut task = Task::new(
1826 "fixture".into(),
1827 "task".into(),
1828 self.repo.clone(),
1829 Source::Human,
1830 );
1831 task.runs = runs.iter().map(|s| s.id.clone()).collect();
1832 crate::queue::Queue::at(self.home.join("queue"))
1833 .put(&mut task)
1834 .unwrap();
1835 task
1836 }
1837 }
1838
1839 #[cfg(unix)]
1840 #[tokio::test]
1841 async fn incremental_only_reclamation_preserves_legacy_and_unknown_data() {
1842 let f = Fx::new().await;
1843 let state = f.run("ca01", true).await;
1844 let wt = &state.candidates[0].worktree;
1845 let generation = crate::incremental::tests::generate(wt);
1846 for dir in [wt.join("target/debug"), wt.join("target/debug/deps")] {
1848 for entry in std::fs::read_dir(dir).unwrap() {
1849 let path = entry.unwrap().path();
1850 if path.extension().is_some_and(|x| x == "d") {
1851 std::fs::remove_file(path).unwrap();
1852 }
1853 }
1854 }
1855 assert!(declared_outputs(&wt.join("target")).unwrap().is_empty());
1856 assert_eq!(
1857 sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes,
1858 0,
1859 "live generation is protected"
1860 );
1861 drop(generation);
1862 let candidates = crate::incremental::candidates(wt).unwrap();
1863 let logical: u64 = candidates
1864 .files
1865 .iter()
1866 .map(|p| std::fs::metadata(p).unwrap().len())
1867 .sum();
1868 let unknown = candidates.files[0].parent().unwrap().join("operator-note");
1869 std::fs::write(&unknown, "retain unknown data").unwrap();
1870 drop(candidates);
1871 let refused = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1872 assert_eq!(refused.freed_bytes, 0, "{refused:?}");
1873 assert!(
1874 refused
1875 .retained
1876 .iter()
1877 .any(|s| s.contains("provenance mismatch"))
1878 );
1879 std::fs::remove_file(&unknown).unwrap();
1880 let legacy = wt.join("target/debug/incremental/legacy-session");
1881 std::fs::create_dir(&legacy).unwrap();
1882 let candidates = crate::incremental::candidates(wt).unwrap();
1884 let bin = candidates
1885 .files
1886 .iter()
1887 .find(|p| p.extension().is_some_and(|x| x == "bin"))
1888 .unwrap();
1889 std::fs::copy(bin, legacy.join("query-cache.bin")).unwrap();
1890 drop(candidates);
1891 let cargo = cargo_locks(&wt.join("target")).unwrap();
1892 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1893 drop(cargo);
1894 let outputs = OutputLease::borrow(wt).unwrap().unwrap();
1895 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
1896 drop(outputs);
1897 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1898 assert_eq!(report.freed_bytes, logical, "{report:?}");
1899 assert_eq!(report.proven_reclaimable_bytes, logical);
1900 assert!(report.measured_bytes > logical);
1901 assert_eq!(
1902 report.protected_or_unproven_bytes,
1903 report.measured_bytes - logical
1904 );
1905 #[cfg(unix)]
1906 {
1907 assert!(report.estimated_released_allocated_bytes.unwrap() > 0);
1908 assert!(
1909 report.measured_allocated_file_bytes.unwrap()
1910 >= report.estimated_released_allocated_bytes.unwrap()
1911 );
1912 }
1913 assert!(report.observed_home_free_change_bytes.is_some());
1914 assert!(legacy.join("query-cache.bin").is_file());
1915 assert!(wt.join("src/lib.rs").is_file());
1916 assert_eq!(report.retired, 0);
1917 }
1918
1919 #[cfg(unix)]
1920 #[tokio::test]
1921 async fn incremental_and_deps_links_account_for_retained_third_link() {
1922 use std::os::unix::fs::MetadataExt;
1923 let f = Fx::new().await;
1924 let state = f.run("ca02", true).await;
1925 let wt = &state.candidates[0].worktree;
1926 let generation = crate::incremental::tests::generate(wt);
1927 drop(generation);
1928 let candidates = crate::incremental::candidates(wt).unwrap();
1929 let objects = candidates
1930 .files
1931 .iter()
1932 .filter(|p| p.extension().is_some_and(|x| x == "o"))
1933 .take(2)
1934 .cloned()
1935 .collect::<Vec<_>>();
1936 assert_eq!(
1937 objects.len(),
1938 2,
1939 "fixture must produce real incremental objects"
1940 );
1941 let mut outputs = candidates.files.clone();
1942 drop(candidates);
1943 let target = wt.join("target");
1944 for dir in [target.join("debug"), target.join("debug/deps")] {
1945 for entry in std::fs::read_dir(dir).unwrap() {
1946 let path = entry.unwrap().path();
1947 if path.extension().is_some_and(|x| x == "d") {
1948 std::fs::remove_file(path).unwrap();
1949 }
1950 }
1951 }
1952 for entry in std::fs::read_dir(target.join("debug/deps")).unwrap() {
1955 let path = entry.unwrap().path();
1956 if path.is_file() {
1957 std::fs::remove_file(path).unwrap();
1958 }
1959 }
1960 for (index, object) in objects.iter().enumerate() {
1961 assert_eq!(std::fs::metadata(object).unwrap().nlink(), 1);
1962 let dep = target.join(format!("debug/deps/link{index}"));
1963 std::fs::hard_link(object, &dep).unwrap();
1964 std::fs::write(
1965 dep.with_extension("d"),
1966 format!("{}: src/lib.rs\n", dep.display()),
1967 )
1968 .unwrap();
1969 outputs.push(dep);
1970 }
1971 let retained = wt.join("retained-input");
1972 std::fs::hard_link(&objects[1], &retained).unwrap();
1973 let sentinel = std::fs::read(&retained).unwrap();
1974 let logical: u64 = outputs
1975 .iter()
1976 .map(|p| std::fs::metadata(p).unwrap().len())
1977 .sum();
1978 let mut seen = std::collections::BTreeSet::new();
1979 let retained_meta = std::fs::metadata(&retained).unwrap();
1980 let retained_inode = (retained_meta.dev(), retained_meta.ino());
1981 let mut expected = 0u64;
1982 for output in &outputs {
1983 let m = std::fs::metadata(output).unwrap();
1984 let inode = (m.dev(), m.ino());
1985 if inode != retained_inode && seen.insert(inode) {
1986 expected += m.blocks() * 512;
1987 }
1988 }
1989 assert!(expected > 0);
1990 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
1991 assert_eq!(report.proven_reclaimable_bytes, logical, "{report:?}");
1992 assert_eq!(report.freed_bytes, logical, "{report:?}");
1993 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(expected));
1994 assert_eq!(report.estimated_released_allocated_bytes, Some(expected));
1995 assert!(outputs.iter().all(|p| !p.exists()));
1996 assert_eq!(std::fs::read(retained).unwrap(), sentinel);
1997 assert!(wt.join("src/lib.rs").is_file());
1998 }
1999
2000 #[tokio::test]
2001 async fn retention_notice_explains_unproven_inventory_and_live_owner() {
2002 let f = Fx::new().await;
2003 let state = f.run("ca03", true).await;
2004 let wt = &state.candidates[0].worktree;
2005 Fx::target(wt);
2006 std::fs::remove_file(wt.join("target/debug/output")).unwrap();
2007 let unknown = wt.join("target/debug/operator-note");
2008 std::fs::write(&unknown, vec![1; 4096]).unwrap();
2009 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2010 assert_eq!(report.proven_reclaimable_bytes, 0);
2011 #[cfg(unix)]
2012 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(0));
2013 assert!(report.protected_or_unproven_bytes >= 4096);
2014 let message = || {
2015 crate::notices::Notices::at(f.home.join("notifications"))
2016 .list()
2017 .into_iter()
2018 .find(|n| n.key == "disk:retention")
2019 .unwrap()
2020 .message
2021 };
2022 let notice = message();
2023 assert!(notice.contains("proven reclaimable 0B"), "{notice}");
2024 assert!(notice.contains("protected or unproven"), "{notice}");
2025 assert!(
2026 notice.contains("legacy incremental/build and unknown files retained"),
2027 "{notice}"
2028 );
2029 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2030 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2031 assert_eq!(report.freed_bytes, 0);
2032 let notice = message();
2033 assert!(notice.contains(&state.id), "{notice}");
2034 assert!(
2035 notice.contains("lock") || notice.contains("owner"),
2036 "{notice}"
2037 );
2038 drop(owner);
2039 assert_eq!(std::fs::read(unknown).unwrap(), vec![1; 4096]);
2040 }
2041
2042 #[tokio::test]
2043 async fn real_cargo_build_holds_the_target_against_reclamation() {
2044 let f = Fx::new().await;
2045 let state = f.run("ca02", true).await;
2046 let wt = &state.candidates[0].worktree;
2047 Fx::target(wt);
2048 std::fs::create_dir(wt.join("src")).unwrap();
2049 std::fs::write(
2050 wt.join("Cargo.toml"),
2051 "[package]\nname='live-cache-fixture'\nversion='0.1.0'\nedition='2021'\n[workspace]\n",
2052 )
2053 .unwrap();
2054 std::fs::write(wt.join("src/lib.rs"), "pub fn answer() -> usize { 42 }\n").unwrap();
2055 std::fs::write(
2056 wt.join("build.rs"),
2057 r#"fn main() {
2058 let ready = std::env::var("FIXTURE_READY").unwrap();
2059 let release = std::env::var("FIXTURE_RELEASE").unwrap();
2060 std::fs::write(ready, "ready").unwrap();
2061 let start = std::time::Instant::now();
2062 while !std::path::Path::new(&release).exists() && start.elapsed().as_secs() < 15 {
2063 std::thread::sleep(std::time::Duration::from_millis(10));
2064 }
2065 }"#,
2066 )
2067 .unwrap();
2068 let ready = wt.join("ready");
2069 let release = wt.join("release");
2070 let mut cargo = tokio::process::Command::new("cargo");
2071 cargo.env_clear();
2072 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2073 if let Some(value) = std::env::var_os(key) {
2074 cargo.env(key, value);
2075 }
2076 }
2077 let mut child = cargo
2078 .current_dir(wt)
2079 .env("RUSTUP_TOOLCHAIN", "stable")
2080 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2081 .env("CARGO_INCREMENTAL", "1")
2082 .env("FIXTURE_READY", &ready)
2083 .env("FIXTURE_RELEASE", &release)
2084 .args(["build", "--offline", "--jobs", "1", "--target-dir"])
2085 .arg(wt.join("target"))
2086 .stdout(std::process::Stdio::null())
2087 .stderr(std::process::Stdio::null())
2088 .kill_on_drop(true)
2089 .spawn()
2090 .unwrap();
2091 let arrived = tokio::time::timeout(std::time::Duration::from_secs(10), async {
2092 while !ready.exists() {
2093 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2094 }
2095 })
2096 .await
2097 .is_ok();
2098 let report = if arrived {
2099 Some(sweep(&f.home, &Fx::disk(), Fx::now()).await)
2100 } else {
2101 None
2102 };
2103 std::fs::write(release, "release").unwrap();
2105 let status = tokio::time::timeout(std::time::Duration::from_secs(10), child.wait()).await;
2106 if status.is_err() {
2107 child.kill().await.unwrap();
2108 child.wait().await.unwrap();
2109 }
2110 assert!(
2111 arrived,
2112 "fixture build never reached its active build script"
2113 );
2114 assert!(status.unwrap().unwrap().success());
2115 let report = report.unwrap();
2116 assert_eq!(report.freed_bytes, 0, "{report:?}");
2117 assert!(
2118 report
2119 .retained
2120 .iter()
2121 .any(|s| s.contains("Cargo build is active")),
2122 "{report:?}"
2123 );
2124 assert!(wt.join("target/debug/output").is_file());
2125 assert!(crate::disk::dir_size(&wt.join("target/debug/incremental")) > 0);
2126 }
2127
2128 #[cfg(unix)]
2129 #[test]
2130 fn allocated_accounting_deduplicates_links_and_requires_the_last_link() {
2131 let temp = tempfile::tempdir().unwrap();
2132 let a = temp.path().join("a");
2133 let b = temp.path().join("b");
2134 let outside = temp.path().join("retained-input");
2135 std::fs::write(&a, vec![1; 65536]).unwrap();
2136 std::fs::hard_link(&a, &b).unwrap();
2137 std::fs::hard_link(&a, &outside).unwrap();
2138 let mut seen = Default::default();
2139 let blocks = allocated_inventory(&temp.path().canonicalize().unwrap(), &mut seen).unwrap();
2140 assert!(blocks > 0);
2141 let mut accounting = ReleasedAllocation::of(&[a.clone(), b.clone()]).unwrap();
2142 std::fs::remove_file(&a).unwrap();
2143 assert_eq!(accounting.unlinked(&a), Some(0));
2144 std::fs::remove_file(&b).unwrap();
2145 assert_eq!(accounting.unlinked(&b), Some(0));
2146 assert_eq!(std::fs::read(&outside).unwrap().len(), 65536);
2147 let mut accounting = ReleasedAllocation::of(std::slice::from_ref(&outside)).unwrap();
2148 std::fs::remove_file(&outside).unwrap();
2149 assert_eq!(accounting.unlinked(&outside), Some(blocks));
2150 }
2151
2152 #[cfg(unix)]
2153 #[test]
2154 fn relative_admission_home_matches_absolute_home_and_retains_symlink_checks() {
2155 let temp = tempfile::tempdir_in(".").unwrap();
2156 let home = PathBuf::from(temp.path().file_name().unwrap()).join("state");
2157 assert!(home.is_relative());
2158 let paths = vec![std::env::current_dir().unwrap()];
2159 let gib = 1024 * 1024 * 1024;
2160 let reservation = OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).unwrap();
2161 assert!(
2162 OutputLease::admit_with(&home.canonicalize().unwrap(), &paths, 8 * gib, |_| Ok(
2163 9 * gib
2164 ))
2165 .is_err()
2166 );
2167 drop(reservation);
2168 assert!(OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).is_ok());
2169 let bad = temp.path().join("symlink-state");
2170 std::fs::create_dir(&bad).unwrap();
2171 std::os::unix::fs::symlink(
2172 home.canonicalize().unwrap().join("disk-admission"),
2173 bad.join("disk-admission"),
2174 )
2175 .unwrap();
2176 assert!(OutputLease::admit_with(&bad, &paths, 8 * gib, |_| Ok(9 * gib)).is_err());
2177 }
2178
2179 #[cfg(unix)]
2180 #[test]
2181 fn concurrent_disk_admission_reserves_once_per_volume_until_release() {
2182 let temp = tempfile::tempdir().unwrap();
2183 let home = temp.path().canonicalize().unwrap();
2184 let paths = vec![home.clone(), home.join("not-created-yet")];
2185 let gib = 1024 * 1024 * 1024;
2186 let floor = 8 * gib;
2187 let first = OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap();
2188 assert_eq!(first.len(), 1);
2189 let blocked =
2190 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap_err();
2191 assert!(blocked.to_string().contains("active writer reservations"));
2192 let second =
2193 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + 2 * gib)).unwrap();
2194 drop(first);
2195 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_err());
2196 drop(second);
2197 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_ok());
2198 assert!(
2199 OutputLease::admit_with(&home, &paths, floor, |_| anyhow::bail!("unmeasured")).is_err()
2200 );
2201 }
2202
2203 #[cfg(unix)]
2204 #[test]
2205 fn simultaneous_writers_cannot_spend_the_same_volume_headroom() {
2206 let temp = tempfile::tempdir().unwrap();
2207 let home = temp.path().canonicalize().unwrap();
2208 let barrier = std::sync::Barrier::new(2);
2209 let gib = 1024 * 1024 * 1024;
2210 let results = std::thread::scope(|scope| {
2211 let threads: Vec<_> = (0..2)
2212 .map(|_| {
2213 scope.spawn(|| {
2214 barrier.wait();
2215 OutputLease::admit_with(&home, std::slice::from_ref(&home), 100, |_| {
2216 Ok(100 + gib)
2217 })
2218 })
2219 })
2220 .collect();
2221 threads
2222 .into_iter()
2223 .map(|thread| thread.join().unwrap())
2224 .collect::<Vec<_>>()
2225 });
2226 assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
2227 assert!(
2228 results
2229 .iter()
2230 .filter_map(|r| r.as_ref().err())
2231 .all(|e| e.to_string().contains("active writer reservations"))
2232 );
2233 }
2234
2235 #[tokio::test]
2236 async fn sweep_reclaims_real_cargo_outputs_and_preserves_source_and_unknown_files() {
2237 real_cargo_fixture(false).await;
2238 }
2239
2240 #[tokio::test]
2241 async fn missing_tag_from_precreated_target_reclaims_only_real_cargo_artifacts() {
2242 real_cargo_fixture(true).await;
2243 }
2244
2245 async fn real_cargo_fixture(precreate: bool) {
2246 let f = Fx::new().await;
2247 let state = f.run("ca00", true).await;
2248 let wt = &state.candidates[0].worktree;
2249 let source = "fn main() { println!(\"retained fixture\"); }\n";
2250 std::fs::create_dir(wt.join("src")).unwrap();
2251 std::fs::write(wt.join("src/main.rs"), source).unwrap();
2252 std::fs::write(
2253 wt.join("Cargo.toml"),
2254 "[package]\nname = \"retained-fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n",
2255 )
2256 .unwrap();
2257 if precreate {
2258 std::fs::create_dir_all(wt.join("target/magi-tools")).unwrap();
2259 }
2260 let mut cargo = std::process::Command::new("cargo");
2261 cargo.env_clear();
2262 for key in ["PATH", "HOME", "RUSTUP_HOME"] {
2263 if let Some(value) = std::env::var_os(key) {
2264 cargo.env(key, value);
2265 }
2266 }
2267 let build = cargo
2268 .current_dir(wt)
2269 .env("RUSTUP_TOOLCHAIN", "stable")
2270 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2271 .env("CARGO_INCREMENTAL", "0")
2272 .args([
2273 "build",
2274 "--offline",
2275 "--release",
2276 "--jobs",
2277 "1",
2278 "--target-dir",
2279 ])
2280 .arg(wt.join("target"))
2281 .output()
2282 .unwrap();
2283 assert!(
2284 build.status.success(),
2285 "{}",
2286 String::from_utf8_lossy(&build.stderr)
2287 );
2288 assert_eq!(wt.join("target/CACHEDIR.TAG").exists(), !precreate);
2289 if precreate {
2290 let again = cargo.output().unwrap();
2292 assert!(
2293 again.status.success(),
2294 "{}",
2295 String::from_utf8_lossy(&again.stderr)
2296 );
2297 assert!(!wt.join("target/CACHEDIR.TAG").exists());
2298 assert!(cargo_target(wt).await.is_err());
2299 }
2300 git::git(wt, &["add", "Cargo.toml", "Cargo.lock", "src"])
2301 .await
2302 .unwrap();
2303 git::git(wt, &["commit", "-m", "real Cargo fixture"])
2304 .await
2305 .unwrap();
2306 let artifact = wt.join("target/release/retained-fixture");
2307 assert!(artifact.is_file());
2308 let unknown = wt.join("target/release/operator-note");
2309 std::fs::write(&unknown, "keep this note\n").unwrap();
2310
2311 let target = wt.join("target");
2312 let mut preserved = std::collections::BTreeMap::new();
2313 for relative in [
2314 "tmp/sentinel",
2315 "magi-tools/sentinel",
2316 "lifecycle-client-check.log",
2317 ] {
2318 let path = target.join(relative);
2319 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
2320 std::fs::write(&path, format!("retain {relative}\n")).unwrap();
2321 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2322 }
2323 let decoy = target.join("magi-tools/retained-fixture");
2326 std::fs::copy(&artifact, &decoy).unwrap();
2327 let dep_info = decoy.with_extension("d");
2328 std::fs::write(&dep_info, format!("{}: src/main.rs\n", decoy.display())).unwrap();
2329 for path in [
2330 decoy,
2331 dep_info,
2332 wt.join("Cargo.toml"),
2333 wt.join("Cargo.lock"),
2334 ] {
2335 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2336 }
2337 let proven = declared_outputs(&target).unwrap();
2338
2339 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2340 assert!(out.freed_bytes > 0, "{out:?}");
2341 for (path, bytes) in preserved {
2342 assert_eq!(std::fs::read(&path).unwrap(), bytes, "{}", path.display());
2343 }
2344 assert!(!artifact.exists(), "{out:?}");
2345 assert_eq!(
2346 std::fs::read_to_string(wt.join("src/main.rs")).unwrap(),
2347 source
2348 );
2349 assert_eq!(
2350 std::fs::read_to_string(&unknown).unwrap(),
2351 "keep this note\n"
2352 );
2353 assert_eq!(std::fs::read_to_string(wt.join("source")).unwrap(), "ca00");
2354 let ledger: serde_json::Value = serde_json::from_slice(
2355 &std::fs::read(
2356 f.home
2357 .join("runs")
2358 .join(&state.id)
2359 .join("build-outputs.json"),
2360 )
2361 .unwrap(),
2362 )
2363 .unwrap();
2364 assert_eq!(
2365 ledger["identity"],
2366 if precreate {
2367 "rustc-profile-dep-info"
2368 } else {
2369 "cache-tag"
2370 }
2371 );
2372 if precreate {
2373 assert!(!target.join("CACHEDIR.TAG").exists());
2374 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2375 }
2376 let deleted: Vec<PathBuf> = serde_json::from_value(ledger["files"].clone()).unwrap();
2377 assert!(deleted.contains(&artifact));
2378 assert_eq!(deleted, proven);
2379 for path in deleted {
2380 assert!(!path.exists());
2381 assert!(
2382 [target.join("release"), target.join("release/deps")]
2383 .contains(&path.parent().unwrap().to_owned())
2384 );
2385 }
2386 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2387 }
2388
2389 fn missing_tag_target(wt: &Path) {
2390 Fx::target(wt);
2391 let target = wt.join("target");
2392 std::fs::remove_file(target.join("CACHEDIR.TAG")).unwrap();
2393 for subdir in ["debug/deps", "debug/.fingerprint"] {
2394 std::fs::create_dir_all(target.join(subdir)).unwrap();
2395 }
2396 std::fs::write(target.join(".rustc_info.json"),
2397 r#"{"rustc_fingerprint":123,"outputs":{"1":{"success":true,"stdout":"rustc fixture","stderr":""}},"successes":{}}"#).unwrap();
2398 }
2399
2400 #[tokio::test]
2401 async fn bookkeeping_flushes_do_not_extend_artifact_grace_including_legacy_records() {
2402 for legacy in [false, true] {
2403 let f = Fx::new().await;
2404 let mut state = f.run("ac00", true).await;
2405 Fx::target(&state.candidates[0].worktree);
2406 let original = state.updated_at;
2407 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2408 if legacy {
2409 let path = f.home.join("runs").join(&state.id).join("run.json");
2410 let mut value = serde_json::to_value(&state).unwrap();
2411 value.as_object_mut().unwrap().remove("output_activity_at");
2412 std::fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
2413 state = RunState::load_under(&state.id, &f.home).unwrap();
2414 assert_eq!(state.output_activity(), original);
2415 }
2416 let age = state.output_activity();
2417 for offset in [30, 20, 10] {
2418 state
2419 .save_under_at(&f.home, now - jiff::SignedDuration::from_secs(offset))
2420 .unwrap();
2421 state = RunState::load_under(&state.id, &f.home).unwrap();
2422 assert_eq!(state.output_activity(), age);
2423 }
2424 let mut disk = Fx::disk();
2425 disk.fold_grace_secs = 3600;
2426 let report = sweep(&f.home, &disk, now).await;
2427 assert_eq!(report.freed_bytes, 200, "legacy={legacy}: {report:?}");
2428 }
2429 }
2430
2431 #[tokio::test]
2432 async fn recent_execution_outputs_and_released_output_leases_retain_artifacts() {
2433 for evidence in [
2434 "execution",
2435 "artifact",
2436 "dep-info",
2437 "fingerprint",
2438 "lease",
2439 "future",
2440 ] {
2441 let f = Fx::new().await;
2442 let mut state = f.run("ac01", true).await;
2443 let wt = state.candidates[0].worktree.clone();
2444 missing_tag_target(&wt);
2445 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
2446 let recent = now - jiff::SignedDuration::from_secs(10);
2447 match evidence {
2448 "execution" => {
2449 state.note_output_activity(recent);
2450 state.save_under_at(&f.home, recent).unwrap();
2451 }
2452 "lease" => {
2453 let lease = OutputLease::borrow(&wt).unwrap().unwrap();
2454 let file = lease._file.try_clone().unwrap();
2455 drop(lease);
2456 file.set_modified(std::time::SystemTime::from(recent))
2457 .unwrap();
2458 }
2459 _ => {
2460 let relative = match evidence {
2461 "dep-info" => "debug/output.d",
2462 "fingerprint" => "debug/.fingerprint/invoked.timestamp",
2463 _ => "debug/output",
2464 };
2465 let path = wt.join("target").join(relative);
2466 let file = OpenOptions::new()
2467 .write(true)
2468 .create(true)
2469 .truncate(false)
2470 .open(path)
2471 .unwrap();
2472 let at = if evidence == "future" {
2473 now + jiff::SignedDuration::from_secs(10)
2474 } else {
2475 recent
2476 };
2477 file.set_modified(std::time::SystemTime::from(at)).unwrap();
2478 }
2479 }
2480 let mut disk = Fx::disk();
2481 disk.fold_grace_secs = 3600;
2482 let report = sweep(&f.home, &disk, now).await;
2483 assert_eq!(report.freed_bytes, 0, "{evidence}: {report:?}");
2484 assert!(
2485 report.retained.iter().any(|s| s.contains("grace period")
2486 || s.contains("execution activity after inventory")),
2487 "{evidence}: {report:?}"
2488 );
2489 let aged = now + jiff::SignedDuration::from_secs(7200);
2490 let report = sweep(&f.home, &disk, aged).await;
2491 assert_eq!(report.freed_bytes, 200, "{evidence}: {report:?}");
2492 }
2493 }
2494
2495 #[tokio::test]
2496 async fn missing_tag_proof_preserves_locks_unknown_files_and_retirement_boundary() {
2497 let f = Fx::new().await;
2498 let state = f.run("ac02", true).await;
2499 let wt = &state.candidates[0].worktree;
2500 missing_tag_target(wt);
2501 let target = wt.join("target");
2502 let unknown = target.join("debug/notes");
2503 std::fs::write(&unknown, "operator data").unwrap();
2504 for lock in [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"] {
2505 let path = target.join("debug").join(lock);
2506 let file = OpenOptions::new()
2507 .create(true)
2508 .truncate(false)
2509 .read(true)
2510 .write(true)
2511 .open(&path)
2512 .unwrap();
2513 crate::pr_head::try_lock_file(&file).unwrap();
2514 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2515 assert_eq!(report.freed_bytes, 0, "{lock}: {report:?}");
2516 assert!(
2517 report
2518 .retained
2519 .iter()
2520 .any(|s| s.contains(path.to_str().unwrap())),
2521 "{report:?}"
2522 );
2523 }
2524 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2525 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2526 drop(owner);
2527 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
2528 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2529 drop(driver);
2530 let output = OutputLease::borrow(wt).unwrap().unwrap();
2531 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2532 drop(output);
2533 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2534 assert_eq!(report.freed_bytes, 200, "{report:?}");
2535 assert_eq!(std::fs::read_to_string(unknown).unwrap(), "operator data");
2536 assert!(!target.join("CACHEDIR.TAG").exists());
2537 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2538 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2539 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2540 }
2541
2542 #[tokio::test]
2543 async fn missing_tag_partial_reclamation_can_resume_without_adopting_residue() {
2544 let f = Fx::new().await;
2545 let state = f.run("ac04", true).await;
2546 let wt = &state.candidates[0].worktree;
2547 missing_tag_target(wt);
2548 let target = wt.join("target");
2549 let first = target.join("debug/output");
2550 let second = target.join("debug/second");
2551 std::fs::copy(&first, &second).unwrap();
2552 std::fs::write(
2553 second.with_extension("d"),
2554 format!("{}: source\n", second.display()),
2555 )
2556 .unwrap();
2557 let outputs = declared_outputs(&target).unwrap();
2558 assert_eq!(outputs.len(), 2);
2559 std::fs::write(f.home.join("runs").join(&state.id).join("build-outputs.json"),
2561 serde_json::to_vec(&serde_json::json!({"target":target,"files":outputs,"action":"reclaim-declared-outputs"})).unwrap()).unwrap();
2562 std::fs::remove_file(first).unwrap();
2563 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2564 assert_eq!(report.freed_bytes, 200, "{report:?}");
2565 assert!(!second.exists());
2566 assert!(second.with_extension("d").exists());
2567 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2568 }
2569
2570 #[tokio::test]
2571 async fn missing_tag_proof_rejects_invalid_identity_structure_and_symlinks() {
2572 for invalid in [
2573 "identity",
2574 "deps",
2575 "fingerprint",
2576 "lock",
2577 "dep-info",
2578 "tag-symlink",
2579 "bad-tag",
2580 ] {
2581 let f = Fx::new().await;
2582 let state = f.run("ac03", true).await;
2583 let wt = &state.candidates[0].worktree;
2584 missing_tag_target(wt);
2585 let target = wt.join("target");
2586 match invalid {
2587 "identity" => std::fs::write(
2588 target.join(".rustc_info.json"),
2589 r#"{"rustc_fingerprint":123}"#,
2590 )
2591 .unwrap(),
2592 "deps" | "fingerprint" => {
2593 let path = target.join("debug").join(if invalid == "deps" {
2594 "deps"
2595 } else {
2596 ".fingerprint"
2597 });
2598 std::fs::remove_dir(&path).unwrap();
2599 std::os::unix::fs::symlink(&f.repo, path).unwrap();
2600 }
2601 "lock" => {
2602 std::fs::remove_file(target.join("debug/.cargo-lock")).unwrap();
2603 }
2604 "dep-info" => std::fs::write(target.join("debug/output.d"), "").unwrap(),
2605 "tag-symlink" => {
2606 std::os::unix::fs::symlink(target.join("absent"), target.join("CACHEDIR.TAG"))
2607 .unwrap()
2608 }
2609 "bad-tag" => std::fs::write(target.join("CACHEDIR.TAG"), "unknown").unwrap(),
2610 _ => unreachable!(),
2611 }
2612 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2613 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
2614 assert!(target.join("debug/output").is_file());
2615 }
2616 }
2617
2618 #[tokio::test]
2619 async fn sweep_budget_oversized_target_makes_progress_without_a_second_deletion() {
2620 let f = Fx::new().await;
2621 let first = f.run("ba00", true).await;
2622 let second = f.run("ba01", true).await;
2623 for state in [&first, &second] {
2624 Fx::target(&state.candidates[0].worktree);
2625 }
2626 let mut disk = Fx::disk();
2627 disk.worktree_reclaim_bytes = 100;
2628 let out = sweep(&f.home, &disk, Fx::now()).await;
2629 assert_eq!(out.freed_bytes, 200, "{out:?}");
2630 assert!(
2631 out.retained
2632 .contains(&format!("{}: per-pass byte budget spent", second.id)),
2633 "{out:?}"
2634 );
2635 assert!(
2636 !first.candidates[0]
2637 .worktree
2638 .join("target/debug/output")
2639 .exists()
2640 );
2641 assert!(
2642 second.candidates[0]
2643 .worktree
2644 .join("target/debug/output")
2645 .exists()
2646 );
2647 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2648 }
2649
2650 #[tokio::test]
2651 async fn sweep_budget_pressure_prioritizes_largest_over_oldest() {
2652 let f = Fx::new().await;
2653 let small = f.run("bb00", true).await;
2654 let large = f.run("bb01", true).await;
2655 for state in [&small, &large] {
2656 Fx::target(&state.candidates[0].worktree);
2657 }
2658 let output = large.candidates[0].worktree.join("target/debug/output");
2659 std::fs::OpenOptions::new()
2660 .write(true)
2661 .open(&output)
2662 .unwrap()
2663 .set_len(1000)
2664 .unwrap();
2665 let mut disk = Fx::disk();
2666 disk.worktree_reclaim_bytes = 300;
2667 disk.min_free_bytes = u64::MAX;
2668 disk.worktree_limit_bytes = u64::MAX;
2670 let out = sweep(&f.home, &disk, Fx::now()).await;
2671 assert_eq!(out.freed_bytes, 1000, "{out:?}");
2672 assert!(!output.exists());
2673 assert!(
2674 small.candidates[0]
2675 .worktree
2676 .join("target/debug/output")
2677 .exists()
2678 );
2679 }
2680
2681 #[tokio::test]
2682 async fn sweep_budget_zero_remaining_and_inventory_are_distinct() {
2683 let f = Fx::new().await;
2684 let first = f.run("bc00", true).await;
2685 let second = f.run("bc01", true).await;
2686 for state in [&first, &second] {
2687 Fx::target(&state.candidates[0].worktree);
2688 }
2689 std::fs::create_dir_all(
2691 first.candidates[0]
2692 .worktree
2693 .join("target/debug/incremental"),
2694 )
2695 .unwrap();
2696 std::fs::write(
2697 first.candidates[0]
2698 .worktree
2699 .join("target/debug/incremental/cache"),
2700 [0; 1000],
2701 )
2702 .unwrap();
2703 let mut disk = Fx::disk();
2704 disk.worktree_reclaim_bytes = 0;
2705 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2706 disk.worktree_reclaim_bytes = 300;
2707 let out = sweep(&f.home, &disk, Fx::now()).await;
2708 assert_eq!(out.freed_bytes, 200, "{out:?}");
2709 assert!(
2710 !first.candidates[0]
2711 .worktree
2712 .join("target/debug/output")
2713 .exists()
2714 );
2715 assert!(
2716 second.candidates[0]
2717 .worktree
2718 .join("target/debug/output")
2719 .exists()
2720 );
2721 assert!(
2722 out.retained
2723 .iter()
2724 .any(|r| r.contains("declared outputs exceed"))
2725 );
2726 }
2727
2728 #[tokio::test]
2729 async fn sweep_budget_exception_preserves_grace_liveness_and_locks() {
2730 let f = Fx::new().await;
2731 let mut protected = f.run("bd00", true).await;
2732 let later = f.run("bd01", true).await;
2733 for state in [&protected, &later] {
2734 Fx::target(&state.candidates[0].worktree);
2735 }
2736 let mut disk = Fx::disk();
2737 disk.worktree_reclaim_bytes = 100;
2738 disk.fold_grace_secs = 3600;
2739 let out = sweep(&f.home, &disk, Fx::now()).await;
2740 assert_eq!(out.freed_bytes, 0);
2741 assert!(
2742 out.retained
2743 .iter()
2744 .any(|r| r.contains("within grace period"))
2745 );
2746 disk.fold_grace_secs = 0;
2747 protected.driver_exited = false;
2748 protected.driver_pid = Some(std::process::id());
2749 protected.driver_started_at = crate::proc::process_started_at(std::process::id());
2750 protected
2751 .save_under_at(&f.home, protected.updated_at)
2752 .unwrap();
2753 let out = sweep(&f.home, &disk, Fx::now()).await;
2754 assert!(
2755 out.retained
2756 .iter()
2757 .any(|r| r.contains("run live or ownership unknown"))
2758 );
2759 assert_eq!(
2760 out.freed_bytes, 200,
2761 "live target must not consume exception: {out:?}"
2762 );
2763 let output = protected.candidates[0].worktree.join("target/debug/output");
2764 assert!(output.exists());
2765 protected.driver_exited = true;
2766 protected.save_under(&f.home).unwrap();
2767 let owner = crate::pr_head::owner_lock(&f.home, &protected.id).unwrap();
2768 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2769 assert!(output.exists());
2770 drop(owner);
2771 let cargo = cargo_locks(&protected.candidates[0].worktree.join("target")).unwrap();
2772 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2773 assert!(output.exists());
2774 drop(cargo);
2775 let lease = OutputLease::take(&protected.candidates[0].worktree, true).unwrap();
2776 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
2777 assert!(output.exists());
2778 drop(lease);
2779 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
2780 assert!(!output.exists());
2781 }
2782
2783 #[tokio::test]
2784 async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
2785 for remove_root in [true, false] {
2786 let f = Fx::new().await;
2787 let stale = Fx::new().await;
2788 let mut missing = stale.run("aa00", false).await;
2789 Fx::target(&missing.candidates[0].worktree);
2790 missing.save_under(&f.home).unwrap();
2791 if remove_root {
2792 std::fs::remove_dir_all(missing.worktree_root()).unwrap();
2793 std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
2796 } else {
2797 std::fs::remove_dir_all(&stale.repo).unwrap();
2798 }
2799 let healthy = f.run("aa01", false).await;
2800 Fx::target(&healthy.candidates[0].worktree);
2801
2802 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2803 assert!(report.failures.is_empty(), "{report:?}");
2804 assert_eq!(report.freed_bytes, 200, "{report:?}");
2805 assert!(
2806 !healthy.candidates[0]
2807 .worktree
2808 .join("target/debug/output")
2809 .exists()
2810 );
2811 if !remove_root {
2812 assert!(
2814 missing.candidates[0]
2815 .worktree
2816 .join("target/debug/output")
2817 .exists()
2818 );
2819 }
2820 }
2821 }
2822
2823 #[tokio::test]
2824 async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
2825 let f = Fx::new().await;
2826 let mut missing = f.run("aa00", false).await;
2827 Fx::target(&missing.candidates[0].worktree);
2828 missing.repo = missing.candidates[0].worktree.clone();
2829 missing.save_under(&f.home).unwrap();
2830 std::fs::remove_dir_all(&missing.repo).unwrap();
2831 assert!(missing.worktree_root().is_dir());
2832 assert!(f.repo.is_dir());
2833
2834 let healthy = f.run("aa01", false).await;
2835 Fx::target(&healthy.candidates[0].worktree);
2836
2837 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2838 assert!(report.failures.is_empty(), "{report:?}");
2839 assert_eq!(report.freed_bytes, 200, "{report:?}");
2840 assert!(
2841 !healthy.candidates[0]
2842 .worktree
2843 .join("target/debug/output")
2844 .exists()
2845 );
2846 }
2847
2848 #[tokio::test]
2849 async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
2850 let f = Fx::new().await;
2851 let broken = Fx::new().await;
2852 let mut invalid = broken.run("aa00", false).await;
2853 Fx::target(&invalid.candidates[0].worktree);
2854 invalid.save_under(&f.home).unwrap();
2855 std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
2856 let healthy = f.run("aa01", false).await;
2857 Fx::target(&healthy.candidates[0].worktree);
2858
2859 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2860 assert_eq!(report.failures.len(), 1, "{report:?}");
2861 assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
2862 assert_eq!(report.freed_bytes, 200, "{report:?}");
2863 assert!(
2864 !healthy.candidates[0]
2865 .worktree
2866 .join("target/debug/output")
2867 .exists()
2868 );
2869 assert!(
2870 invalid.candidates[0]
2871 .worktree
2872 .join("target/debug/output")
2873 .exists()
2874 );
2875 }
2876
2877 async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
2878 let path = reconcile::run(&f.home, &task.id, &source.id, None)
2879 .await
2880 .unwrap();
2881 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
2882 }
2883
2884 #[tokio::test]
2885 async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
2886 use crate::{
2887 chat_control::{Operation, dispatch},
2888 talk::{self, Talks},
2889 };
2890 let f = Fx::new().await;
2891 let source = f.run("ae00", true).await;
2892 let clean = f.run("ae01", false).await;
2893 let queue = crate::queue::Queue::at(f.home.join("queue"));
2894 let talks = Talks::at(f.home.join("talks"));
2895 let open_talk = |id: &str| -> talk::Talk {
2896 let mut conversation = serde_json::from_value(serde_json::json!({
2897 "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
2898 "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
2899 "seat":crate::agent::SeatState::new("talk","worker",1)
2900 }))
2901 .unwrap();
2902 talks.put(&mut conversation).unwrap();
2903 conversation
2904 };
2905 let conversation = open_talk("talk-owner");
2906 let other = open_talk("talk-other");
2907 let task = f.task(&[source.clone(), clean.clone()]);
2909 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
2910 let operation = |apply: Option<&str>| Operation::ReconcileRetention {
2911 task: task.id.clone(),
2912 authority: source.id.clone(),
2913 apply: apply.map(str::to_owned),
2914 };
2915 let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
2917 .await
2918 .unwrap();
2919 let legacy: serde_json::Value =
2920 serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
2921 let legacy_id = legacy["id"].as_str().unwrap();
2922 assert!(
2923 dispatch(
2924 &queue,
2925 &f.home,
2926 &talks,
2927 &conversation.id,
2928 operation(Some(legacy_id))
2929 )
2930 .await
2931 .is_err()
2932 );
2933 let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
2934 .await
2935 .unwrap();
2936 assert!(summary.contains("Complete review summary"));
2937 assert!(summary.contains(&source.id) && summary.contains(&clean.id));
2938 let root = f.home.join("retention-reconciliations");
2939 let displayed: serde_json::Value =
2940 serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
2941 let id = displayed["plan"].as_str().unwrap();
2942 let dir = root.join(id);
2943 let plan: serde_json::Value =
2944 serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
2945 assert_eq!(plan["version"], 2);
2946 assert!(clean.candidates[0].worktree.exists());
2947 assert!(!dir.join("approved.json").exists());
2948 assert!(
2950 dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
2951 .await
2952 .is_err()
2953 );
2954 assert!(!dir.join("approved.json").exists());
2955 let error = reconcile::interrupt_chat_after_fence(
2958 &f.home,
2959 &task,
2960 id,
2961 reconcile::ChatRequest {
2962 talks: &talks,
2963 conversation: &conversation.id,
2964 },
2965 )
2966 .await
2967 .unwrap_err();
2968 assert!(
2969 error.to_string().contains("simulated interruption"),
2970 "{error:#}"
2971 );
2972 assert!(clean.candidates[0].worktree.exists());
2973 assert!(dir.join("approved.json").exists());
2974 dispatch(
2975 &queue,
2976 &f.home,
2977 &talks,
2978 &conversation.id,
2979 operation(Some(id)),
2980 )
2981 .await
2982 .unwrap();
2983 assert!(!clean.candidates[0].worktree.exists());
2984 assert!(source.candidates[0].worktree.exists());
2985 let audit: serde_json::Value =
2986 serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
2987 assert_eq!(audit["version"], 2);
2988 assert_eq!(
2989 audit["owner"],
2990 serde_json::json!({"conversation": conversation.id})
2991 );
2992 dispatch(
2993 &queue,
2994 &f.home,
2995 &talks,
2996 &conversation.id,
2997 operation(Some(id)),
2998 )
2999 .await
3000 .unwrap();
3001 }
3002
3003 #[tokio::test]
3004 async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
3005 let f = Fx::new().await;
3006 let source = f.run("af00", true).await;
3007 let mut clean = f.run("af01", false).await;
3008 let mut dirty = f.run("af02", false).await;
3009 for state in [&mut clean, &mut dirty] {
3010 state.status = RunStatus::Prep;
3011 state.prep_complete = Some(false);
3012 state.candidates[0].empty = false;
3013 state.save_under(&f.home).unwrap();
3014 }
3015 std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
3016 let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
3017 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3018 let plan = reconciliation_plan(&f, &task, &source).await;
3019 let selected: Vec<_> = plan["candidates"]
3020 .as_array()
3021 .unwrap()
3022 .iter()
3023 .filter(|e| e["selected"] == true)
3024 .collect();
3025 assert_eq!(selected.len(), 1, "{plan:#}");
3026 assert_eq!(selected[0]["run"], clean.id);
3027 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3028 .await
3029 .unwrap();
3030 assert!(!clean.candidates[0].worktree.exists());
3031 assert!(dirty.candidates[0].worktree.exists());
3032 assert!(
3033 RunState::load_under(&dirty.id, &f.home)
3034 .unwrap()
3035 .retired_to
3036 .is_none()
3037 );
3038 }
3039
3040 #[tokio::test]
3041 async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
3042 let f = Fx::new().await;
3043 let mut source = f.run("aa00", true).await;
3044 let alternative = f.run("aa01", true).await;
3045 let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
3046 .await
3047 .unwrap();
3048 source.reviewed_commits = Some(vec![head.clone()]);
3049 source.save_under(&f.home).unwrap();
3050 let mut runs = vec![source.clone(), alternative.clone()];
3051 for i in 2..14 {
3052 runs.push(f.run(&format!("aa{i:02}"), false).await);
3053 }
3054 let mut task = f.task(&runs);
3055 task.pr_recovery = Some(crate::pr_recovery::Selection {
3056 url: "https://example.invalid/pull/1".into(),
3057 branch: "selected".into(),
3058 head: head.clone(),
3059 source_head: Some(head.clone()),
3060 source_run: Some(source.id.clone()),
3061 source_proof: None,
3062 disposition: Some(crate::pr_recovery::Disposition::Land),
3063 instruction: task.instruction.clone(),
3064 approved_at: Timestamp::now(),
3065 });
3066 source.pr_recovery = task.pr_recovery.clone();
3067 source.save_under(&f.home).unwrap();
3068 runs[0] = source.clone();
3069 crate::queue::Queue::at(f.home.join("queue"))
3070 .put(&mut task)
3071 .unwrap();
3072 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3073 let task_before = serde_json::to_value(&task).unwrap();
3074 let mut report = Report::default();
3075 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3076 assert_eq!(
3077 report.retired, 0,
3078 "automatic cleanup must still refuse blanket markers"
3079 );
3080 assert!(
3081 report
3082 .retained
3083 .iter()
3084 .any(|reason| reason.contains("explicitly retained"))
3085 );
3086 let mut total = 0;
3087 for expected in [5, 5, 2, 0] {
3088 let plan = reconciliation_plan(&f, &task, &source).await;
3089 let selected: Vec<_> = plan["candidates"]
3090 .as_array()
3091 .unwrap()
3092 .iter()
3093 .filter(|e| e["selected"] == true)
3094 .collect();
3095 assert_eq!(selected.len(), expected, "{plan:#}");
3096 for e in &selected {
3097 assert!(
3098 RunState::load_under(e["run"].as_str().unwrap(), &f.home)
3099 .unwrap()
3100 .retired_to
3101 .is_none()
3102 );
3103 }
3104 let operation = plan["id"].as_str().unwrap();
3105 for _ in 0..2 {
3106 reconcile::run(&f.home, &task.id, &source.id, Some(operation))
3107 .await
3108 .unwrap();
3109 }
3110 total += expected;
3111 }
3112 assert_eq!(total, 12);
3113 for run in &runs {
3114 let dir = f.home.join("runs").join(&run.id);
3115 assert!(dir.join("run.json").exists());
3116 assert!(dir.join("retain-recovery-sources").exists());
3117 assert!(
3118 git::branch_exists(&f.repo, &run.candidates[0].branch)
3119 .await
3120 .unwrap()
3121 );
3122 }
3123 assert!(source.candidates[0].worktree.exists());
3124 assert!(alternative.candidates[0].worktree.exists());
3125 assert_eq!(
3126 git::rev_parse(&f.repo, &source.candidates[0].branch)
3127 .await
3128 .unwrap(),
3129 head
3130 );
3131 assert_eq!(
3132 serde_json::to_value(
3133 crate::queue::Queue::at(f.home.join("queue"))
3134 .get(&task.id)
3135 .unwrap()
3136 )
3137 .unwrap(),
3138 task_before
3139 );
3140 assert_eq!(
3141 runs.iter()
3142 .filter(|r| RunState::load_under(&r.id, &f.home)
3143 .unwrap()
3144 .retired_to
3145 .is_none())
3146 .count(),
3147 2
3148 );
3149 }
3150
3151 #[tokio::test]
3152 async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
3153 let f = Fx::new().await;
3154 let source = f.run("ab00", true).await;
3155 let mut runs = vec![source.clone()];
3156 for i in 1..9 {
3157 runs.push(f.run(&format!("ab{i:02}"), false).await);
3158 }
3159 std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
3160 std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
3161 std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
3162 std::fs::write(
3163 runs[3].candidates[0].worktree.join("scratch/source"),
3164 "ignored source",
3165 )
3166 .unwrap();
3167 std::fs::write(
3168 runs[4].worktree_root().join("unknown"),
3169 "unregistered source",
3170 )
3171 .unwrap();
3172 let checkpoint = git::git(
3173 &runs[5].candidates[0].worktree,
3174 &[
3175 "rev-parse",
3176 "--path-format=absolute",
3177 "--git-path",
3178 "sequencer",
3179 ],
3180 )
3181 .await
3182 .unwrap();
3183 std::fs::create_dir(checkpoint.trim()).unwrap();
3184 runs[6].driver_exited = false;
3185 runs[6].driver_pid = Some(std::process::id());
3186 runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
3187 runs[6].save_under(&f.home).unwrap();
3188 runs[7].driver_exited = false;
3189 runs[7].driver_pid = None;
3190 runs[7].driver_claimed = None; runs[7].save_under(&f.home).unwrap();
3192 assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
3193
3194 let hidden = &runs[8].candidates[0].worktree;
3195 git::git(hidden, &["update-index", "--assume-unchanged", "source"])
3196 .await
3197 .unwrap();
3198 std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
3199 assert!(
3200 git::git(hidden, &["status", "--porcelain"])
3201 .await
3202 .unwrap()
3203 .is_empty()
3204 );
3205 let task = f.task(&runs);
3206 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3207 let plan = reconciliation_plan(&f, &task, &source).await;
3208 assert!(
3209 plan["candidates"]
3210 .as_array()
3211 .unwrap()
3212 .iter()
3213 .all(|e| e["selected"] == false),
3214 "{plan:#}"
3215 );
3216 for run in &runs {
3217 assert!(run.candidates[0].worktree.exists());
3218 }
3219 }
3220
3221 #[tokio::test]
3222 async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
3223 let f = Fx::new().await;
3224 let source = f.run("ac00", true).await;
3225 let empty = f.run("ac01", false).await;
3226 let task = f.task(&[source.clone(), empty.clone()]);
3227 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3228 let plan = reconciliation_plan(&f, &task, &source).await;
3229 std::fs::write(
3230 f.home.join("runs").join(&source.id).join("new-report"),
3231 "new review checkpoint",
3232 )
3233 .unwrap();
3234 assert!(
3235 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3236 .await
3237 .is_err()
3238 );
3239 assert!(empty.candidates[0].worktree.exists());
3240 assert!(
3241 RunState::load_under(&empty.id, &f.home)
3242 .unwrap()
3243 .retired_to
3244 .is_none()
3245 );
3246 let marker = f
3247 .home
3248 .join("runs")
3249 .join(&empty.id)
3250 .join("retain-recovery-sources");
3251 std::fs::write(&marker, "different owner").unwrap();
3252 let plan = reconciliation_plan(&f, &task, &source).await;
3253 assert!(
3254 plan["candidates"]
3255 .as_array()
3256 .unwrap()
3257 .iter()
3258 .all(|e| e["selected"] == false)
3259 );
3260 }
3261
3262 #[tokio::test]
3263 async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
3264 for partial in [false, true] {
3265 let f = Fx::new().await;
3266 let source = f.run("ab10", true).await;
3267 let first = f.run("ab11", false).await;
3268 let second = f.run("ab12", false).await;
3269 let task = f.task(&[source.clone(), first.clone(), second.clone()]);
3270 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3271 let plan = reconciliation_plan(&f, &task, &source).await;
3272 let id = plan["id"].as_str().unwrap();
3273 let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
3274 std::fs::write(&staging, b"").unwrap();
3276 assert!(
3277 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3278 .await
3279 .is_err()
3280 );
3281 assert!(staging.exists());
3282 std::fs::remove_file(&staging).unwrap();
3283 let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
3284 .await
3285 .unwrap_err();
3286 assert!(
3287 error.to_string().contains("before run state rename"),
3288 "{error:#}"
3289 );
3290 assert!(
3291 RunState::load_under(&first.id, &f.home)
3292 .unwrap()
3293 .retired_to
3294 .is_some()
3295 );
3296 assert!(
3297 RunState::load_under(&second.id, &f.home)
3298 .unwrap()
3299 .retired_to
3300 .is_none()
3301 );
3302 assert!(first.candidates[0].worktree.exists());
3303 assert!(second.candidates[0].worktree.exists());
3304 let interrupted = std::fs::read(&staging).unwrap();
3305 std::fs::write(&staging, b"unrelated recovery source").unwrap();
3307 assert!(
3308 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3309 .await
3310 .is_err()
3311 );
3312 assert_eq!(
3313 std::fs::read(&staging).unwrap(),
3314 b"unrelated recovery source"
3315 );
3316 std::fs::write(&staging, interrupted).unwrap();
3317 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3318 .await
3319 .unwrap();
3320 assert!(!staging.exists());
3321 assert!(!first.candidates[0].worktree.exists());
3322 assert!(!second.candidates[0].worktree.exists());
3323 assert_eq!(
3324 RunState::load_under(&second.id, &f.home)
3325 .unwrap()
3326 .retired_to,
3327 Some(source.id)
3328 );
3329 }
3330 }
3331
3332 #[tokio::test]
3333 async fn reconciliation_resumes_only_its_exact_durable_fence() {
3334 let f = Fx::new().await;
3335 let source = f.run("ad00", true).await;
3336 let empty = f.run("ad01", false).await;
3337 let task = f.task(&[source.clone(), empty.clone()]);
3338 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3339 let plan = reconciliation_plan(&f, &task, &source).await;
3340 let id = plan["id"].as_str().unwrap();
3341 assert!(
3342 reconcile::interrupt_after_fence(&f.home, &task, id)
3343 .await
3344 .is_err()
3345 );
3346 assert!(empty.candidates[0].worktree.exists());
3347 assert_eq!(
3348 RunState::load_under(&empty.id, &f.home)
3349 .unwrap()
3350 .retired_to
3351 .as_ref(),
3352 Some(&source.id)
3353 );
3354 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3355 .await
3356 .unwrap();
3357 assert!(!empty.candidates[0].worktree.exists());
3358 git::worktree_add_branch(
3360 &f.repo,
3361 &empty.candidates[0].worktree,
3362 "restore-empty",
3363 &f.base,
3364 )
3365 .await
3366 .unwrap();
3367 assert!(
3369 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3370 .await
3371 .is_err()
3372 );
3373 assert!(empty.candidates[0].worktree.exists());
3374 let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
3375 changed.prep_failure = Some("new owner edit".into());
3376 changed.save_under(&f.home).unwrap();
3377 assert!(
3378 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3379 .await
3380 .is_err()
3381 );
3382 }
3383
3384 #[tokio::test]
3385 async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
3386 let f = Fx::new().await;
3387 let source = f.run("ae00", true).await;
3388 let mut prep = f.run("ae01", false).await;
3389 let candidate = prep.candidates.pop().unwrap();
3390 prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
3391 prep.save_under(&f.home).unwrap();
3392 let reset = f.run("ae02", false).await;
3393 let wt = &reset.candidates[0].worktree;
3394 std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
3395 git::git(wt, &["add", "."]).await.unwrap();
3396 git::git(wt, &["commit", "-m", "unique implementation"])
3397 .await
3398 .unwrap();
3399 git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
3400 let unknown = f.run("ae03", false).await;
3401 let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
3402 let mut raw: serde_json::Value =
3403 serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
3404 raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
3405 std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
3406 let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
3407 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3408 let plan = reconciliation_plan(&f, &task, &source).await;
3409 let selected: Vec<_> = plan["candidates"]
3410 .as_array()
3411 .unwrap()
3412 .iter()
3413 .filter(|e| e["selected"] == true)
3414 .collect();
3415 assert_eq!(selected.len(), 1, "{plan:#}");
3416 assert_eq!(selected[0]["run"], prep.id);
3417 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3418 .await
3419 .unwrap();
3420 assert!(!candidate.worktree.exists());
3421 assert!(reset.candidates[0].worktree.exists());
3422 assert!(unknown.candidates[0].worktree.exists());
3423 }
3424
3425 #[tokio::test]
3426 async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
3427 let f = Fx::new().await;
3428 let mut state = f.run("aaaa", true).await;
3429 let candidate = state.candidates[0].worktree.clone();
3430 std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
3431 Fx::target(&candidate);
3432 for seat in ["review-1", "review-2"] {
3433 let wt = state.worktree_root().join(seat);
3434 git::worktree_add_detached(&f.repo, &wt, &f.base)
3435 .await
3436 .unwrap();
3437 Fx::target(&wt);
3438 }
3439 state.status = RunStatus::Landing;
3440 state.save_under(&f.home).unwrap();
3441 let second = Fx::new().await;
3442 let other = second.run("bbbb", true).await;
3443 let mut imported = other.clone();
3445 imported.save_under(&f.home).unwrap();
3446 Fx::target(&other.candidates[0].worktree);
3447 let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
3448 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3449 assert!(
3450 candidate.join("target/debug/output").exists(),
3451 "active writer protected"
3452 );
3453 assert!(
3454 !state
3455 .worktree_root()
3456 .join("review-1/target/debug/output")
3457 .exists()
3458 );
3459 assert!(
3460 !state
3461 .worktree_root()
3462 .join("review-2/target/debug/output")
3463 .exists()
3464 );
3465 assert!(
3466 !other.candidates[0]
3467 .worktree
3468 .join("target/debug/output")
3469 .exists()
3470 );
3471 assert!(out.freed_bytes >= 600);
3472 drop(guard);
3473 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3474 assert!(out.freed_bytes >= 200);
3475 assert_eq!(
3476 std::fs::read_to_string(candidate.join("source")).unwrap(),
3477 "uncommitted work"
3478 );
3479 assert!(
3480 git::branch_exists(&f.repo, &state.candidates[0].branch)
3481 .await
3482 .unwrap()
3483 );
3484 }
3485
3486 #[tokio::test]
3487 async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
3488 let f = Fx::new().await;
3489 let state = f.run("cc01", false).await;
3490 let wt = &state.candidates[0].worktree;
3491 Fx::target(wt);
3492 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3493 let dir = wt.join("target").join(dir);
3494 std::fs::create_dir_all(&dir).unwrap();
3495 std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
3496 }
3497 let manifest = wt.join("target/debug/output.d");
3499 let output = wt.join("target/debug/output");
3500 std::fs::write(
3501 &manifest,
3502 format!(
3503 "{}: {}\n",
3504 output.display(),
3505 wt.join("target/debug/notes.txt").display()
3506 ),
3507 )
3508 .unwrap();
3509 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3510 assert_eq!(report.freed_bytes, 200);
3511 assert!(!output.exists());
3512 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
3513 assert_eq!(
3514 std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
3515 "user evidence"
3516 );
3517 }
3518 assert!(manifest.exists());
3520 assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
3521 std::fs::write(&output, "user note replacing an old binary").unwrap();
3522 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3523 assert_eq!(report.freed_bytes, 0);
3524 assert_eq!(
3525 std::fs::read_to_string(&output).unwrap(),
3526 "user note replacing an old binary"
3527 );
3528 }
3529
3530 #[tokio::test]
3531 async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
3532 let f = Fx::new().await;
3533 let state = f.run("cc02", false).await;
3534 let wt = &state.candidates[0].worktree;
3535 Fx::target(wt);
3536 let mut disk = Fx::disk();
3537 disk.worktree_limit_bytes = u64::MAX;
3538 let now = Fx::now();
3539 let first = sweep(&f.home, &disk, now).await;
3540 std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
3541 assert_eq!(
3542 sweep(&f.home, &disk, now).await.measured_bytes,
3543 first.measured_bytes
3544 );
3545 assert!(
3546 sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
3547 .await
3548 .measured_bytes
3549 > first.measured_bytes
3550 );
3551 disk.worktree_limit_bytes = 0;
3552 assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
3553 }
3554
3555 #[tokio::test]
3556 async fn unfinished_source_fold_reports_its_retention_reason() {
3557 let f = Fx::new().await;
3558 let mut source = f.run("cc03", true).await;
3559 let task = f.task(&[source.clone()]);
3560 let error = crate::graph::fold_run(&mut source, true, &f.home)
3561 .await
3562 .unwrap_err();
3563 assert!(error.to_string().contains(&task.id));
3564 assert!(source.candidates[0].worktree.exists());
3565 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3566 let error = crate::graph::fold_run(&mut source, true, &f.home)
3567 .await
3568 .unwrap_err();
3569 assert!(error.to_string().contains("explicitly retained"));
3570 }
3571
3572 #[tokio::test]
3573 async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
3574 let f = Fx::new().await;
3575 let mut source = f.run("cc04", true).await;
3576 let mut task = f.task(&[source.clone()]);
3577 task.hold_manual(Some("queue settlement pending".into()));
3578 let queue = crate::queue::Queue::at(f.home.join("queue"));
3579 queue.put(&mut task).unwrap();
3580 source.merge = Some(crate::run::MergeOutcome {
3581 mode: crate::config::MergeMode::Pr,
3582 ok: true,
3583 detail: "confirmed merged".into(),
3584 empty: false,
3585 });
3586 for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
3588 source.status = status;
3589 assert!(
3590 crate::graph::fold_run(&mut source, true, &f.home)
3591 .await
3592 .is_err()
3593 );
3594 assert!(source.candidates[0].worktree.exists());
3595 }
3596 git::git(
3597 &f.repo,
3598 &["merge", "--ff-only", &source.candidates[0].branch],
3599 )
3600 .await
3601 .unwrap();
3602 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3603 .await
3604 .unwrap();
3605 source.status = RunStatus::Merged;
3606 source.save_under(&f.home).unwrap();
3607 assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
3608 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3609 assert!(
3610 crate::graph::fold_run(&mut source, true, &f.home)
3611 .await
3612 .unwrap_err()
3613 .to_string()
3614 .contains("explicitly retained")
3615 );
3616 std::fs::remove_file(
3619 f.home
3620 .join("runs")
3621 .join(&source.id)
3622 .join("retain-recovery-sources"),
3623 )
3624 .unwrap();
3625 let (folded, unreadable) = crate::clean::fold_due(
3626 &f.home.join("runs"),
3627 &f.home,
3628 &f.bay,
3629 &Fx::disk(),
3630 Fx::now(),
3631 )
3632 .await
3633 .unwrap();
3634 assert_eq!((folded, unreadable), (1, 0));
3635 assert!(!source.candidates[0].worktree.exists());
3636 let evidence = f
3637 .home
3638 .join("runs")
3639 .join(&source.id)
3640 .join("artifacts/fold-evidence");
3641 let names: Vec<_> = std::fs::read_dir(evidence)
3642 .unwrap()
3643 .map(|entry| entry.unwrap().file_name())
3644 .collect();
3645 assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
3646 assert_eq!(
3647 queue.get(&task.id).unwrap().status,
3648 crate::queue::TaskStatus::Held
3649 );
3650 }
3651
3652 #[tokio::test]
3653 async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
3654 for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
3655 let f = Fx::new().await;
3656 let mut state = f.run("cc06", true).await;
3657 let candidate = state.candidates[0].clone();
3658 let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3659 std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
3660 git::rescue_commit(&candidate.worktree, "second candidate change")
3661 .await
3662 .unwrap();
3663 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3664 state.tally = Some(crate::run::Tally {
3665 first_choice: Default::default(),
3666 borda: Default::default(),
3667 winner: 'A',
3668 rankings: 0,
3669 unanimous_initial: true,
3670 deliberated: false,
3671 changed_votes: 0,
3672 unanimous_final: true,
3673 tie_break: None,
3674 judges: 0,
3675 present: 0,
3676 quorum: 0,
3677 met_quorum: true,
3678 uncontested: Some("fixture".into()),
3679 });
3680 let reviewer = state.worktree_root().join("review-1");
3683 git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
3684 .await
3685 .unwrap();
3686 std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
3689 git::rescue_commit(&f.repo, "advance before squash")
3690 .await
3691 .unwrap();
3692 let published = if variant == "partial" {
3693 &intermediate
3694 } else {
3695 &head
3696 };
3697 git::git(&f.repo, &["merge", "--squash", published])
3698 .await
3699 .unwrap();
3700 git::git(&f.repo, &["commit", "-m", "squash candidate"])
3701 .await
3702 .unwrap();
3703 let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
3704 assert!(
3705 !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
3706 .await
3707 .unwrap()
3708 .ok()
3709 );
3710 std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
3711 git::rescue_commit(&f.repo, "advance after squash")
3712 .await
3713 .unwrap();
3714 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
3715 .await
3716 .unwrap();
3717 if matches!(variant, "checkpoint" | "reviewer") {
3718 std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
3720 git::rescue_commit(&reviewer, "unpublished checkpoint")
3721 .await
3722 .unwrap();
3723 if variant == "checkpoint" {
3724 git::git(&reviewer, &["reset", "--hard", &intermediate])
3725 .await
3726 .unwrap();
3727 }
3728 }
3729 if variant == "loser" {
3730 let mut loser = candidate.clone();
3731 loser.label = 'B';
3732 loser.branch = "magi/cc06/B".into();
3733 loser.worktree = state.worktree_root().join("cand-B");
3734 git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
3735 .await
3736 .unwrap();
3737 std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
3738 git::rescue_commit(&loser.worktree, "independent loser")
3739 .await
3740 .unwrap();
3741 state.candidates.push(loser);
3742 }
3743 state.status = RunStatus::Merged;
3744 state.merge = Some(crate::run::MergeOutcome {
3745 mode: crate::config::MergeMode::Pr,
3746 ok: true,
3747 detail: "candidate A squash merged".into(),
3748 empty: false,
3749 });
3750 state.save_under(&f.home).unwrap();
3751 let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
3752 let result = crate::graph::fold_run(&mut state, true, &f.home).await;
3753 result.unwrap();
3754 assert!(
3755 !candidate.worktree.exists() && !reviewer.exists(),
3756 "{variant}"
3757 );
3758 assert!(
3759 !git::branch_exists(&f.repo, &candidate.branch)
3760 .await
3761 .unwrap()
3762 );
3763 let artifacts = f
3764 .home
3765 .join("runs")
3766 .join(&state.id)
3767 .join("artifacts/fold-evidence");
3768 assert_eq!(
3769 artifacts.join(format!("{head}.patch")).exists(),
3770 variant == "partial",
3771 "only a partially published winner needs evidence: {variant}"
3772 );
3773 assert_eq!(
3774 artifacts.join(format!("{intermediate}.patch")).exists(),
3775 variant == "partial",
3776 "squash proof also covers intermediate reviewer tips: {variant}"
3777 );
3778 let patches: Vec<_> = std::fs::read_dir(&artifacts)
3779 .unwrap()
3780 .map(|entry| entry.unwrap().path())
3781 .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
3782 .collect();
3783 let expected = match variant {
3784 "partial" => 2,
3785 "loser" | "reviewer" => 1,
3786 _ => 0,
3787 };
3788 assert_eq!(patches.len(), expected, "{variant}");
3789 if variant == "loser" {
3790 assert!(
3791 std::fs::read_to_string(&patches[0])
3792 .unwrap()
3793 .contains("losing candidate")
3794 );
3795 }
3796 assert!(artifacts.join("manifest.json").exists());
3797 if variant == "reviewer" {
3798 assert!(
3799 std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
3800 .unwrap()
3801 .contains("reviewer checkpoint")
3802 );
3803 }
3804 crate::graph::fold_run(&mut state, true, &f.home)
3805 .await
3806 .unwrap();
3807 }
3808 }
3809
3810 #[tokio::test]
3811 async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
3812 for missing in [true, false] {
3813 let f = Fx::new().await;
3814 let mut state = f.run("cc05", true).await;
3815 git::git(
3818 &f.repo,
3819 &["update-ref", "refs/remotes/origin/main", &f.base],
3820 )
3821 .await
3822 .unwrap();
3823 let candidate = state.candidates[0].clone();
3824 if !missing {
3825 std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
3826 git::rescue_commit(&f.repo, "advance base before rebase")
3827 .await
3828 .unwrap();
3829 git::git(&candidate.worktree, &["rebase", "main"])
3830 .await
3831 .unwrap();
3832 git::sync_to_head(&candidate.worktree).await.unwrap();
3835 }
3836 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
3837 state.reviewed_commits = Some(Vec::new());
3839 state.candidates[0].worktree = state.worktree_root().join("under-review");
3840 git::git(
3841 &f.repo,
3842 &[
3843 "worktree",
3844 "move",
3845 candidate.worktree.to_str().unwrap(),
3846 state.candidates[0].worktree.to_str().unwrap(),
3847 ],
3848 )
3849 .await
3850 .unwrap();
3851 let reviewer = state.worktree_root().join("review-1");
3852 git::worktree_add_detached(&f.repo, &reviewer, &head)
3853 .await
3854 .unwrap();
3855 std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
3858 git::rescue_commit(&reviewer, "old review history")
3859 .await
3860 .unwrap();
3861 git::git(&reviewer, &["reset", "--hard", &head])
3862 .await
3863 .unwrap();
3864 if missing {
3865 std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
3866 }
3867 let run_dir = f.home.join("runs").join(&state.id);
3868 let artifacts = run_dir.join("artifacts");
3869 std::fs::create_dir_all(&artifacts).unwrap();
3870 std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
3871 if missing {
3872 std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
3873 }
3874 state.status = RunStatus::Merged;
3875 state.merge = Some(crate::run::MergeOutcome {
3876 mode: crate::config::MergeMode::Pr,
3877 ok: true,
3878 detail: "confirmed squash merge".into(),
3879 empty: false,
3880 });
3881 state.save_under(&f.home).unwrap();
3882 let owned = state.candidates[0].worktree.clone();
3884 state.candidates[0].worktree = f.home.join("missing-external");
3885 assert!(
3886 crate::graph::fold_run(&mut state, true, &f.home)
3887 .await
3888 .unwrap_err()
3889 .to_string()
3890 .contains("outside the inventoried")
3891 );
3892 state.candidates[0].worktree = owned;
3893 let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
3895 assert!(
3896 crate::graph::fold_run(&mut state, true, &f.home)
3897 .await
3898 .is_err()
3899 );
3900 assert!(reviewer.exists());
3901 drop(guard);
3902 let evidence = artifacts.join("fold-evidence");
3904 std::fs::write(&evidence, "blocked").unwrap();
3905 assert!(
3906 crate::graph::fold_run(&mut state, true, &f.home)
3907 .await
3908 .is_err()
3909 );
3910 assert!(reviewer.exists());
3911 std::fs::remove_file(&evidence).unwrap();
3912 std::fs::create_dir_all(&evidence).unwrap();
3913 let oversized = evidence.join("capacity-fixture");
3914 std::fs::File::create(&oversized)
3915 .unwrap()
3916 .set_len(64 * 1024 * 1024)
3917 .unwrap();
3918 assert!(
3919 crate::graph::fold_run(&mut state, true, &f.home)
3920 .await
3921 .is_err()
3922 );
3923 assert!(reviewer.exists());
3924 std::fs::remove_file(oversized).unwrap();
3925 std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
3928 std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
3929 assert!(
3930 crate::graph::fold_run(&mut state, true, &f.home)
3931 .await
3932 .is_err()
3933 );
3934 std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
3935 let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
3936 .await
3937 .unwrap();
3938 std::fs::write(&checkpoint, &head).unwrap();
3939 assert!(
3940 crate::graph::fold_run(&mut state, true, &f.home)
3941 .await
3942 .is_err()
3943 );
3944 std::fs::remove_file(checkpoint).unwrap();
3945 let (folded, unreadable) = crate::clean::fold_due(
3946 &f.home.join("runs"),
3947 &f.home,
3948 &f.bay,
3949 &Fx::disk(),
3950 Fx::now(),
3951 )
3952 .await
3953 .unwrap();
3954 if folded != 1 {
3955 panic!(
3956 "normal cleanup failed: {:#}",
3957 crate::graph::fold_run(&mut state, true, &f.home)
3958 .await
3959 .unwrap_err()
3960 );
3961 }
3962 assert_eq!((folded, unreadable), (1, 0));
3963 assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
3964 state = RunState::load_under(&state.id, &f.home).unwrap();
3965 assert!(state.candidates[0].folded);
3966 let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
3967 crate::graph::fold_run(&mut state, true, &f.home)
3968 .await
3969 .unwrap();
3970 assert_eq!(
3971 std::fs::read(evidence.join("manifest.json")).unwrap(),
3972 manifest
3973 );
3974 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
3975 .await
3976 .unwrap();
3977 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
3978 assert!(
3979 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
3980 .await
3981 .unwrap()
3982 .ok()
3983 );
3984 let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
3987 let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
3988 .as_str()
3989 .unwrap();
3990 let recovered = f.home.join("recovered");
3991 std::fs::create_dir_all(&recovered).unwrap();
3992 git::git(&recovered, &["init"]).await.unwrap();
3993 std::fs::write(
3994 recovered.join("source"),
3995 std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
3996 )
3997 .unwrap();
3998 git::git(
3999 &recovered,
4000 &[
4001 "apply",
4002 evidence.join(format!("{head}.patch")).to_str().unwrap(),
4003 ],
4004 )
4005 .await
4006 .unwrap();
4007 assert_eq!(
4008 std::fs::read_to_string(recovered.join("source")).unwrap(),
4009 "cc05"
4010 );
4011 assert!(
4012 !std::fs::read(evidence.join(format!("{head}.patch")))
4013 .unwrap()
4014 .is_empty()
4015 );
4016 assert_eq!(
4017 std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
4018 "review findings"
4019 );
4020 if missing {
4021 assert_eq!(
4022 std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
4023 "original snapshot"
4024 );
4025 } else {
4026 assert!(!artifacts.join("cand-A.patch").exists());
4027 }
4028 assert!(run_dir.join("run.json").exists());
4029 }
4030 }
4031
4032 #[tokio::test]
4033 async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
4034 use std::os::unix::fs::PermissionsExt as _;
4035 let f = Fx::new().await;
4036 let source = f.run("dd03", true).await;
4037 let empty = f.run("dd04", false).await;
4038 let task = f.task(&[source, empty.clone()]);
4039 let entered = f.repo.join("proof-entered");
4040 let release = f.repo.join("proof-release");
4041 let hook = f.repo.join("proof-monitor");
4042 std::fs::write(&hook, format!(
4043 "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
4044 entered.display(), release.display())).unwrap();
4045 std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
4046 git::git(
4047 &f.repo,
4048 &["config", "core.fsmonitor", hook.to_str().unwrap()],
4049 )
4050 .await
4051 .unwrap();
4052 struct ReleaseProof(PathBuf);
4053 impl Drop for ReleaseProof {
4054 fn drop(&mut self) {
4055 let _ = std::fs::write(&self.0, "continue");
4056 }
4057 }
4058 let _release_on_panic = ReleaseProof(release.clone());
4059 let home = f.home.clone();
4060 let sweep = tokio::spawn(async move {
4061 let mut report = Report::default();
4062 retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
4063 report
4064 });
4065 let watchdog = std::time::Duration::from_secs(60);
4068 tokio::time::timeout(watchdog, async {
4069 while !entered.exists() {
4070 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4071 }
4072 })
4073 .await
4074 .expect("Git retirement proof did not enter the blocking hook");
4075 let queue = crate::queue::Queue::at(f.home.join("queue"));
4076 let mut selected = queue.get(&task.id).unwrap();
4077 selected.resume_override = Some(crate::queue::OperatorResume {
4078 question_id: "during-slow-proof".into(),
4079 at: Fx::now(),
4080 conductor_rehold: None,
4081 forced: false,
4082 pinned_run: Some(empty.id.clone()),
4083 });
4084 let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
4085 let updated_while_blocked = match tokio::time::timeout(watchdog, &mut update).await {
4089 Ok(result) => {
4090 result.unwrap().unwrap();
4091 true
4092 }
4093 Err(_) => {
4094 std::fs::write(&release, "continue").unwrap();
4095 update.await.unwrap().unwrap();
4096 false
4097 }
4098 };
4099 std::fs::write(release, "continue").unwrap();
4100 let report = tokio::time::timeout(watchdog, sweep)
4101 .await
4102 .expect("Git retirement proof did not finish after release")
4103 .unwrap();
4104 assert!(
4105 updated_while_blocked,
4106 "task writers must not wait behind Git retirement proof"
4107 );
4108 assert_eq!(report.retired, 0);
4109 assert!(report.retained.iter().any(|r| r.contains("task changed")));
4110 assert!(empty.candidates[0].worktree.exists());
4111 assert!(
4112 RunState::load_under(&empty.id, &f.home)
4113 .unwrap()
4114 .retired_to
4115 .is_none()
4116 );
4117 }
4118
4119 #[tokio::test]
4120 async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
4121 let f = Fx::new().await;
4122 let source = f.run("dd01", true).await;
4123 let mut empty = f.run("dd02", false).await;
4124 let expected = f.task(&[source.clone(), empty.clone()]);
4125 let queue = crate::queue::Queue::at(f.home.join("queue"));
4126 let _claim = queue.claim(&expected.id).unwrap();
4127 let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
4129 let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
4130 let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
4131 let mut selected = queue.get(&expected.id).unwrap();
4132 selected.resume_override = Some(crate::queue::OperatorResume {
4133 question_id: "new-owner-decision".into(),
4134 at: Fx::now(),
4135 conductor_rehold: None,
4136 forced: false,
4137 pinned_run: Some(empty.id.clone()),
4138 });
4139 queue.put(&mut selected).unwrap();
4140 let error = commit_retirement(
4141 &f.home,
4142 &queue,
4143 &expected,
4144 &mut empty,
4145 &source.id,
4146 &expected.runs,
4147 Fx::now(),
4148 )
4149 .unwrap_err();
4150 assert!(error.to_string().contains("task changed"));
4151 assert!(
4152 RunState::load_under(&empty.id, &f.home)
4153 .unwrap()
4154 .retired_to
4155 .is_none()
4156 );
4157 assert!(empty.candidates[0].worktree.exists());
4158 assert_eq!(
4159 queue
4160 .get(&expected.id)
4161 .unwrap()
4162 .resume_override
4163 .unwrap()
4164 .pinned_run,
4165 Some(empty.id.clone())
4166 );
4167 assert!(
4168 !f.home
4169 .join("recovery-lineages")
4170 .join(format!("{}.json", expected.id))
4171 .exists()
4172 );
4173 assert!(
4175 commit_retirement(
4176 &f.home,
4177 &queue,
4178 &selected,
4179 &mut empty,
4180 &source.id,
4181 &selected.runs,
4182 Fx::now()
4183 )
4184 .unwrap_err()
4185 .to_string()
4186 .contains("selected this recovery source")
4187 );
4188 }
4189
4190 #[tokio::test]
4191 async fn cargo_owners_and_driver_resumes_are_protected_with_retained_root_data() {
4192 let f = Fx::new().await;
4193 let state = f.run("cccc", false).await;
4194 let wt = &state.candidates[0].worktree;
4195 Fx::target(wt);
4196 std::fs::write(wt.join("target/notes"), "user file").unwrap();
4197 std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
4198 assert!(
4199 !declared_outputs(&wt.join("target"))
4200 .unwrap()
4201 .contains(&wt.join("target/debug/link"))
4202 );
4203 std::fs::remove_file(wt.join("target/debug/link")).unwrap();
4204 let cargo = cargo_locks(&wt.join("target")).unwrap();
4205 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4206 assert_eq!(out.freed_bytes, 0);
4207 assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
4208 drop(cargo);
4209 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
4210 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4211 assert_eq!(out.freed_bytes, 0);
4212 drop(driver);
4213 let exclusive = OutputLease::take(wt, true).unwrap();
4214 assert!(
4215 OutputLease::borrow(wt).is_err(),
4216 "resume cannot start a writer during cleanup"
4217 );
4218 drop(exclusive);
4219 assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
4220 assert_eq!(
4221 std::fs::read(wt.join("target/notes")).unwrap(),
4222 b"user file"
4223 );
4224 }
4225
4226 #[tokio::test]
4227 async fn root_leftovers_allow_eviction_but_keep_empty_attempts_registered() {
4228 for leftover in ["tmp", "magi-tools", "lifecycle-client-check.log"] {
4229 let f = Fx::new().await;
4230 let source = f.run("aa10", true).await;
4231 let empty = f.run("aa11", false).await;
4232 f.task(&[source, empty.clone()]);
4233 let wt = &empty.candidates[0].worktree;
4234 Fx::target(wt);
4235 let path = wt.join("target").join(leftover);
4236 if leftover.ends_with(".log") {
4237 std::fs::write(&path, "keep log\n").unwrap();
4238 } else {
4239 std::fs::create_dir(&path).unwrap();
4241 }
4242 let mut disk = Fx::disk();
4243 disk.auto_fold = true;
4244 let report = sweep(&f.home, &disk, Fx::now()).await;
4245 assert_eq!(report.freed_bytes, 200, "{report:?}");
4246 assert_eq!(report.retired, 0, "{report:?}");
4247 assert!(!wt.join("target/debug/output").exists());
4248 assert!(!residue_proof(&f.home, &empty.id, wt).unwrap().exists());
4249 assert!(path.exists());
4250 if leftover.ends_with(".log") {
4251 assert_eq!(std::fs::read(&path).unwrap(), b"keep log\n");
4252 }
4253 assert_eq!(std::fs::read(wt.join("source")).unwrap(), b"base\n");
4254 assert!(worktrees(&empty).await.unwrap().contains(wt));
4255 assert!(
4256 git::branch_exists(&f.repo, &empty.candidates[0].branch)
4257 .await
4258 .unwrap()
4259 );
4260 assert!(
4261 RunState::load_under(&empty.id, &f.home)
4262 .unwrap()
4263 .retired_to
4264 .is_none()
4265 );
4266 }
4267 }
4268
4269 #[tokio::test]
4270 async fn residue_proof_rejects_new_unknown_root_entries_including_empty_directories() {
4271 let f = Fx::new().await;
4272 let state = f.run("aa12", false).await;
4273 let wt = &state.candidates[0].worktree;
4274 Fx::target(wt);
4275 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4276 assert_eq!(report.freed_bytes, 200, "{report:?}");
4277 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4278 for name in ["tmp", "magi-tools", "log"] {
4279 let path = wt.join("target").join(name);
4280 std::fs::create_dir(&path).unwrap();
4281 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
4282 std::fs::remove_dir(path).unwrap();
4283 }
4284 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4285 }
4286
4287 #[tokio::test]
4288 async fn target_identity_and_known_profiles_still_fail_closed_with_root_leftovers() {
4289 for invalid in [
4290 "tag",
4291 "identity",
4292 "tracked",
4293 "profile-file",
4294 "profile-symlink",
4295 "identity-symlink",
4296 ] {
4297 let f = Fx::new().await;
4298 let state = f.run("aa13", false).await;
4299 let wt = &state.candidates[0].worktree;
4300 Fx::target(wt);
4301 std::fs::create_dir(wt.join("target/tmp")).unwrap();
4302 let target = wt.join("target");
4303 match invalid {
4304 "tag" => std::fs::write(target.join("CACHEDIR.TAG"), "not Cargo").unwrap(),
4305 "identity" => std::fs::write(target.join(".rustc_info.json"), "{}").unwrap(),
4306 "tracked" => {
4307 git::git(wt, &["add", "-f", "target/debug/output"])
4308 .await
4309 .unwrap();
4310 }
4311 "profile-file" => {
4312 std::fs::write(target.join("release"), "keep").unwrap();
4313 }
4314 "profile-symlink" => {
4315 std::os::unix::fs::symlink(target.join("debug"), target.join("release"))
4316 .unwrap();
4317 }
4318 "identity-symlink" => {
4319 std::fs::rename(target.join(".rustc_info.json"), target.join("saved-info"))
4320 .unwrap();
4321 std::os::unix::fs::symlink(
4322 target.join("saved-info"),
4323 target.join(".rustc_info.json"),
4324 )
4325 .unwrap();
4326 }
4327 _ => unreachable!(),
4328 }
4329 let before = std::fs::read(target.join("debug/output")).unwrap();
4330 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4331 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
4332 assert_eq!(std::fs::read(target.join("debug/output")).unwrap(), before);
4333 assert!(target.join("tmp").is_dir());
4334 }
4335 }
4336
4337 #[tokio::test]
4338 async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
4339 let f = Fx::new().await;
4340 let source = f.run("0000", true).await;
4341 let empty = f.run("0001", false).await;
4342 let changed = f.run("0002", false).await;
4343 let unknown = f.run("0003", false).await;
4344 let unproven = f.run("0004", false).await;
4345 f.task(&[
4346 source.clone(),
4347 empty.clone(),
4348 changed.clone(),
4349 unknown.clone(),
4350 unproven.clone(),
4351 ]);
4352 for run in [&empty, &changed, &unknown] {
4353 Fx::target(&run.candidates[0].worktree);
4354 }
4355 let reviewer = empty.worktree_root().join("review-1");
4356 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4357 .await
4358 .unwrap();
4359 Fx::target(&reviewer);
4360 let mut disk = Fx::disk();
4361 disk.auto_fold = false;
4362 let report = sweep(&f.home, &disk, Fx::now()).await;
4363 assert_eq!(report.freed_bytes, 800, "{report:?}");
4364 let wt = &empty.candidates[0].worktree;
4365 assert!(wt.join("target/debug/output.d").exists());
4366 assert!(!wt.join("target/debug/output").exists());
4367 let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
4368 let proof: CargoResidue =
4369 serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
4370 assert_eq!(
4371 proof.files[Path::new("debug/output.d")],
4372 std::fs::read(wt.join("target/debug/output.d")).unwrap()
4373 );
4374 std::fs::write(
4375 changed.candidates[0].worktree.join("target/debug/output.d"),
4376 "owner notes",
4377 )
4378 .unwrap();
4379 std::fs::write(
4380 unknown.candidates[0].worktree.join("target/debug/notes"),
4381 "keep",
4382 )
4383 .unwrap();
4384 Fx::target(&unproven.candidates[0].worktree);
4385 std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
4386 disk.auto_fold = true;
4387 let locks = cargo_locks(&wt.join("target")).unwrap();
4388 let mut report = Report::default();
4389 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4390 assert_eq!(report.retired, 0, "{report:?}");
4391 assert!(
4392 report
4393 .retained
4394 .iter()
4395 .any(|reason| reason.contains("Cargo build"))
4396 );
4397 drop(locks);
4398 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
4399 assert_eq!(report.retired, 1, "{report:?}");
4400 assert!(!wt.exists());
4401 assert!(!reviewer.exists());
4402 assert!(
4403 residue_proof(&f.home, &empty.id, &reviewer)
4404 .unwrap()
4405 .exists()
4406 );
4407 assert!(
4408 proof_path.exists(),
4409 "metadata evidence outlives the worktree"
4410 );
4411 assert_eq!(
4412 RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
4413 Some(source.id)
4414 );
4415 for run in [&changed, &unknown, &unproven] {
4416 assert!(run.candidates[0].worktree.exists());
4417 assert!(
4418 RunState::load_under(&run.id, &f.home)
4419 .unwrap()
4420 .retired_to
4421 .is_none()
4422 );
4423 }
4424 }
4425
4426 #[tokio::test]
4427 async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
4428 let f = Fx::new().await;
4429 let source = f.run("0000", true).await;
4430 let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
4431 .await
4432 .unwrap();
4433 let mut runs = vec![source.clone()];
4434 for n in 1..=12 {
4435 runs.push(f.run(&format!("{n:04}"), false).await);
4436 }
4437 let dirty = runs[1].candidates[0].worktree.clone();
4438 std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
4439 let ignored = runs[2].candidates[0].worktree.clone();
4440 std::fs::create_dir(ignored.join("scratch")).unwrap();
4441 std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
4442 runs[1].candidates.clear();
4445 runs[1].save_under(&f.home).unwrap();
4446 let task = f.task(&runs);
4447 let mut total = 0;
4448 for _ in 0..4 {
4449 let mut report = Report::default();
4450 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4451 assert!(report.retired <= 5);
4452 total += report.retired;
4453 }
4454 assert_eq!(total, 10);
4455 assert!(dirty.exists());
4456 assert!(ignored.exists());
4457 assert_eq!(
4458 git::rev_parse(&f.repo, &source.candidates[0].branch)
4459 .await
4460 .unwrap(),
4461 source_head
4462 );
4463 let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
4464 assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
4465 assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
4466 let mut released = task.clone();
4467 released.release();
4468 assert_eq!(
4469 crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
4470 RunState::load_under(id, &f.home)
4471 }),
4472 Some(source.id.clone())
4473 );
4474
4475 assert!(
4476 f.home
4477 .join("runs")
4478 .join(&retired.id)
4479 .join("run.json")
4480 .exists()
4481 );
4482 assert!(
4483 f.home
4484 .join("recovery-lineages")
4485 .join(format!("{}.json", task.id))
4486 .exists()
4487 );
4488 }
4489
4490 #[tokio::test]
4491 async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
4492 let f = Fx::new().await;
4493 let a = f.run("dddd", true).await;
4494 let b = f.run("eeee", true).await;
4495 let empty = f.run("ffff", false).await;
4496 let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
4497 assert!(
4498 crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
4499 "unfinished queue history protects older source branches from ordinary folding"
4500 );
4501 let mut prior = a.clone();
4502 assert!(
4503 crate::graph::fold_run(&mut prior, true, &f.home)
4504 .await
4505 .unwrap_err()
4506 .to_string()
4507 .contains(&task.id)
4508 );
4509 let mut report = Report::default();
4510 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4511 assert_eq!(report.retired, 0);
4512 assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
4513 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4514 Fx::target(&a.candidates[0].worktree);
4515 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4516 assert!(
4517 report.freed_bytes > 0,
4518 "source retention does not retain regenerable outputs"
4519 );
4520 assert!(a.candidates[0].worktree.exists());
4521 assert!(b.candidates[0].worktree.exists());
4522 assert!(empty.candidates[0].worktree.exists());
4523 }
4524
4525 #[tokio::test]
4526 async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
4527 let f = Fx::new().await;
4528 let authority = f.run("p000", true).await;
4529 let question_run = f.run("p001", false).await;
4530 let mut checkpoint = f.run("p002", false).await;
4531 checkpoint.land_approval = Some(crate::run::LandApproval {
4532 question: "pending".into(),
4533 head: f.base.clone(),
4534 });
4535 checkpoint.save_under(&f.home).unwrap();
4536 let retained = f.run("p003", false).await;
4537 std::fs::write(
4538 f.home
4539 .join("runs")
4540 .join(&retained.id)
4541 .join("retain-recovery-sources"),
4542 "owner",
4543 )
4544 .unwrap();
4545 let mut partial = f.run("p004", false).await;
4546 let reviewer = partial.worktree_root().join("review-1");
4547 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
4548 .await
4549 .unwrap();
4550 partial.retired_to = Some(authority.id.clone());
4552 partial.save_under(&f.home).unwrap();
4553 assert!(
4554 git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
4555 .await
4556 .unwrap()
4557 );
4558 let git_checkpoint = f.run("p005", false).await;
4559 let location = git::git(
4560 &git_checkpoint.candidates[0].worktree,
4561 &[
4562 "rev-parse",
4563 "--path-format=absolute",
4564 "--git-path",
4565 "sequencer",
4566 ],
4567 )
4568 .await
4569 .unwrap();
4570 std::fs::create_dir(location.trim()).unwrap();
4571 let mut preservation = f.run("p006", false).await;
4572 preservation
4573 .candidate_preservation
4574 .insert('A', "pending preservation".into());
4575 preservation.save_under(&f.home).unwrap();
4576 let task = f.task(&[
4577 preservation.clone(),
4578 git_checkpoint.clone(),
4579 authority.clone(),
4580 question_run.clone(),
4581 checkpoint.clone(),
4582 retained.clone(),
4583 partial.clone(),
4584 ]);
4585 let questions = crate::ask::Questions::at(f.home.join("questions"));
4586 let mut question = crate::ask::Question::new(
4587 question_run.id.clone(),
4588 "implement".into(),
4589 "fixture".into(),
4590 "owner decision".into(),
4591 String::new(),
4592 vec!["keep".into()],
4593 );
4594 questions.put(&mut question).unwrap();
4595 let mut report = Report::default();
4596 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
4597 assert_eq!(report.retired, 1, "{report:?}");
4598 assert!(!reviewer.exists());
4599 for state in [
4600 &question_run,
4601 &checkpoint,
4602 &retained,
4603 &git_checkpoint,
4604 &preservation,
4605 ] {
4606 assert!(state.candidates[0].worktree.exists());
4607 assert!(
4608 RunState::load_under(&state.id, &f.home)
4609 .unwrap()
4610 .retired_to
4611 .is_none()
4612 );
4613 }
4614 assert!(questions.get(&question.id).unwrap().status.open());
4615 assert_eq!(
4616 crate::queue::Queue::at(f.home.join("queue"))
4617 .get(&task.id)
4618 .unwrap()
4619 .status,
4620 task.status
4621 );
4622 }
4623
4624 #[tokio::test]
4625 async fn preservation_schema_migration_remains_distinct_from_retirement() {
4626 let f = Fx::new().await;
4627 let mut state = f.run("p007", false).await;
4628 state.schema = 15;
4629 state.save_under(&f.home).unwrap();
4630 let native = RunState::load_under(&state.id, &f.home).unwrap();
4631 assert_eq!(native.schema, crate::run::SCHEMA);
4632 assert!(native.candidate_preservation.is_empty());
4633 assert!(native.retired_to.is_none());
4634 state.schema = 14;
4635 state.save_under(&f.home).unwrap();
4636 let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
4637 assert!(legacy.candidate_preservation.contains_key(&'A'));
4638 legacy.save_under(&f.home).unwrap();
4639 let restored = RunState::load_under(&state.id, &f.home).unwrap();
4640 assert_eq!(
4641 restored.candidate_preservation,
4642 legacy.candidate_preservation
4643 );
4644 }
4645
4646 #[tokio::test]
4647 async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
4648 let f = Fx::new().await;
4649 let cache = f.repo.join("cache");
4650 let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
4651 std::fs::create_dir_all(&cache).unwrap();
4652 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4653 assert!(
4654 crate::cache::maintenance_prune(&f.home, &cache, 1)
4655 .unwrap()
4656 .is_none()
4657 );
4658 drop(guard);
4659 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4660 }
4661
4662 #[tokio::test]
4663 async fn launcher_death_does_not_release_live_tool_output_reservations() {
4664 let f = Fx::new().await;
4665 let run = f.run("abcd", false).await;
4666 let wt = &run.candidates[0].worktree;
4667 let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
4668 helper
4669 .quiet()
4670 .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
4671 .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
4672 .env("MAGI_TEST_WORKTREE", wt)
4673 .stdin(std::process::Stdio::null())
4674 .stdout(std::process::Stdio::null())
4675 .stderr(std::process::Stdio::null())
4676 .kill_on_drop(true);
4677 assert!(
4678 tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
4679 .await
4680 .unwrap()
4681 .unwrap()
4682 .success()
4683 );
4684 let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
4685 .unwrap()
4686 .trim()
4687 .parse()
4688 .unwrap();
4689 struct KillTool(i32);
4690 impl Drop for KillTool {
4691 fn drop(&mut self) {
4692 unsafe { libc::kill(self.0, libc::SIGKILL) };
4694 }
4695 }
4696 let tool = KillTool(pid);
4697 assert!(OutputLease::take(wt, true).is_err());
4698 assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
4699 drop(tool);
4700 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4701 loop {
4702 if OutputLease::take(wt, true).is_ok()
4703 && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
4704 {
4705 break;
4706 }
4707 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4708 }
4709 })
4710 .await
4711 .unwrap();
4712 }
4713
4714 #[tokio::test]
4715 async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
4716 let f = Fx::new().await;
4717 let run = f.run("abcd", false).await;
4718 let wt = &run.candidates[0].worktree;
4719 let cache = f.repo.join("cache");
4720 std::fs::create_dir(&cache).unwrap();
4721 for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
4722 let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
4723 let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
4724 cmd.quiet()
4725 .args(["--exact", "seat_process::tests::orphan_tool_helper"])
4726 .env("MAGI_TEST_ORPHAN_PID", &pid_file)
4727 .stdin(std::process::Stdio::null())
4728 .stdout(std::process::Stdio::null())
4729 .stderr(std::process::Stdio::null())
4730 .kill_on_drop(true);
4731 if clear_owner {
4732 cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
4733 } else {
4734 cmd.env("MAGI_TEST_CLOSE_FDS", "1");
4735 }
4736 let gib = 1024 * 1024 * 1024;
4737 let mut leases = vec![
4738 OutputLease::borrow(wt).unwrap().unwrap(),
4739 OutputLease::cache(&f.home, &cache, false).unwrap(),
4740 ];
4741 leases.extend(
4742 OutputLease::admit_with(
4743 &f.home,
4744 std::slice::from_ref(wt),
4745 8 * gib,
4746 |_| Ok(9 * gib),
4747 )
4748 .unwrap(),
4749 );
4750 let mut running = crate::seat_process::Running::spawn(cmd, None, leases).unwrap();
4751 assert!(
4752 tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
4753 .await
4754 .unwrap()
4755 .unwrap()
4756 .success()
4757 );
4758 let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
4759 struct KillTool(Option<i32>);
4760 impl Drop for KillTool {
4761 fn drop(&mut self) {
4762 if let Some(pid) = self.0 {
4764 unsafe { libc::kill(pid, libc::SIGKILL) };
4765 }
4766 }
4767 }
4768 let mut tool = KillTool(Some(pid));
4769 assert!(OutputLease::take(wt, true).is_err());
4773 let finish = if cancel {
4774 drop(running);
4775 None
4776 } else {
4777 Some(tokio::spawn(async move { running.finish().await }))
4778 };
4779 if clear_owner {
4780 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
4781 assert!(crate::proc::pid_alive(pid as u32));
4782 assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
4783 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
4784 assert!(
4785 OutputLease::admit_with(&f.home, std::slice::from_ref(wt), 8 * gib, |_| Ok(
4786 9 * gib
4787 ))
4788 .is_err(),
4789 "orphan still owns its disk reservation"
4790 );
4791 assert!(
4792 crate::cache::maintenance_prune(&f.home, &cache, 0)
4793 .unwrap()
4794 .is_none()
4795 );
4796 if let Some(finish) = &finish {
4797 assert!(!finish.is_finished());
4798 }
4799 unsafe { libc::kill(pid, libc::SIGKILL) };
4801 }
4802 if let Some(finish) = finish {
4803 tokio::time::timeout(std::time::Duration::from_secs(5), finish)
4804 .await
4805 .unwrap()
4806 .unwrap();
4807 }
4808 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4809 loop {
4810 if OutputLease::take(wt, true).is_ok()
4811 && OutputLease::cache(&f.home, &cache, true).is_ok()
4812 && OutputLease::admit_with(
4813 &f.home,
4814 std::slice::from_ref(wt),
4815 8 * gib,
4816 |_| Ok(9 * gib),
4817 )
4818 .is_ok()
4819 {
4820 if !clear_owner {
4822 assert!(!crate::proc::pid_alive(pid as u32));
4823 }
4824 break;
4825 }
4826 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4827 }
4828 })
4829 .await
4830 .unwrap();
4831 tool.0 = None;
4832 }
4833 }
4834
4835 #[tokio::test]
4836 async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
4837 let f = Fx::new().await;
4838 let run = f.run("abcd", false).await;
4839 let wt = &run.candidates[0].worktree;
4840 let cache = f.repo.join("cache");
4841 std::fs::create_dir(&cache).unwrap();
4842 let pid_file = f.repo.join("child-pid");
4843 let mut cmd = tokio::process::Command::new("sh");
4844 cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
4845 .quiet()
4846 .env("CHILD_PID_FILE", &pid_file)
4847 .process_group(0)
4848 .kill_on_drop(true);
4849 let running = crate::seat_process::Running::spawn(
4850 cmd,
4851 None,
4852 vec![
4853 OutputLease::borrow(wt).unwrap().unwrap(),
4854 OutputLease::cache(&f.home, &cache, false).unwrap(),
4855 ],
4856 )
4857 .unwrap();
4858 tokio::time::timeout(std::time::Duration::from_secs(5), async {
4859 while !pid_file.exists() {
4860 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4861 }
4862 })
4863 .await
4864 .unwrap();
4865 assert!(OutputLease::take(wt, true).is_err());
4866 assert!(
4867 crate::cache::maintenance_prune(&f.home, &cache, 0)
4868 .unwrap()
4869 .is_none()
4870 );
4871 let pid: u32 = std::fs::read_to_string(pid_file)
4872 .unwrap()
4873 .trim()
4874 .parse()
4875 .unwrap();
4876 drop(running);
4877 tokio::time::timeout(std::time::Duration::from_secs(10), async {
4878 loop {
4879 if let Ok(guard) = OutputLease::take(wt, true) {
4880 assert!(
4881 !crate::proc::pid_alive(pid),
4882 "output lock released before tool death"
4883 );
4884 drop(guard);
4885 break;
4886 }
4887 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4888 }
4889 })
4890 .await
4891 .unwrap();
4892 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
4893 }
4894
4895 #[tokio::test]
4896 async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
4897 let f = Fx::new().await;
4898 let reason = tokio::time::timeout(
4899 std::time::Duration::from_secs(2),
4900 emergency(vec![f.repo.clone()], u64::MAX),
4901 )
4902 .await
4903 .unwrap();
4904 assert!(reason.contains("disk pressure"));
4905 assert!(
4906 tokio::time::timeout(
4907 std::time::Duration::from_millis(10),
4908 emergency(vec![f.repo.clone()], 0)
4909 )
4910 .await
4911 .is_err()
4912 );
4913 assert_eq!(
4914 std::fs::read_to_string(f.repo.join("source")).unwrap(),
4915 "base\n"
4916 );
4917 }
4918
4919 #[tokio::test]
4920 async fn automatic_requeues_cannot_refund_failed_input_generation() {
4921 let f = Fx::new().await;
4922 let mut run = f.run("1234", false).await;
4923 run.prep_failure = Some("cherry-pick conflict".into());
4924 let mut task = f.task(std::slice::from_ref(&run));
4925 let inputs = crate::queue::recovery_inputs(&run.sources);
4926 for _ in 0..108 {
4927 task.recovery_failed(&run);
4928 task.requeue();
4929 assert_eq!(task.attempts, 0);
4930 }
4931 let bytes = serde_json::to_vec(&task).unwrap();
4932 let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
4933 assert!(
4934 restored
4935 .recovery_exhausted(&run.base_commit, &inputs, 2)
4936 .is_some()
4937 );
4938 assert!(
4939 restored
4940 .recovery_exhausted("different-base", &inputs, 2)
4941 .is_none()
4942 );
4943 let original_base = run.base_commit.clone();
4944 run.base_commit = "different-base".into();
4945 restored.recovery_failed(&run);
4946 assert!(
4947 restored
4948 .recovery_exhausted(&original_base, &inputs, 2)
4949 .is_some(),
4950 "returning to old inputs does not replenish their budget"
4951 );
4952 assert!(
4953 restored
4954 .recovery_exhausted(&run.base_commit, &inputs, 2)
4955 .is_none()
4956 );
4957 restored.release_operator("test-owner", None);
4958 assert!(
4959 restored
4960 .recovery_exhausted(&run.base_commit, &inputs, 2)
4961 .is_none()
4962 );
4963 }
4964 async fn inactive_source(f: &Fx, tag: &str, useful: bool) -> RunState {
4965 let mut state = f.run(tag, useful).await;
4966 state.status = RunStatus::Superseded;
4967 state.save_under(&f.home).unwrap();
4968 state
4969 }
4970
4971 fn historical_review(state: &mut RunState, head: &str) {
4972 state.reviews.push(
4973 serde_json::from_value(serde_json::json!({
4974 "round": state.reviews.len() + 1, "head": head, "reviews": []
4975 }))
4976 .unwrap(),
4977 );
4978 }
4979
4980 async fn reviewer(f: &Fx, state: &RunState, head: &str) -> PathBuf {
4981 let path = state.worktree_root().join("review-1");
4982 git::worktree_add_detached(&f.repo, &path, head)
4983 .await
4984 .unwrap();
4985 path
4986 }
4987
4988 #[tokio::test]
4989 async fn source_eviction_preserves_history_and_restores_binary_modes_after_gc() {
4990 use std::os::unix::fs::PermissionsExt;
4991 let mut f = Fx::new().await;
4992 std::fs::write(f.repo.join("binary"), b"old\0binary\xff").unwrap();
4993 git::git(&f.repo, &["add", "."]).await.unwrap();
4994 git::git(&f.repo, &["commit", "-m", "binary base"])
4995 .await
4996 .unwrap();
4997 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
4998 let mut state = inactive_source(&f, "ec01", true).await;
4999 let candidate = state.candidates[0].worktree.clone();
5000 let earlier = git::rev_parse(&candidate, "HEAD").await.unwrap();
5001 historical_review(&mut state, &earlier);
5002 let review = reviewer(&f, &state, &earlier).await;
5003 std::fs::remove_file(candidate.join("source")).unwrap();
5004 std::fs::write(candidate.join("binary"), b"new\0binary\xfe").unwrap();
5005 std::fs::write(candidate.join("script"), "run\n").unwrap();
5006 std::fs::set_permissions(
5007 candidate.join("script"),
5008 std::fs::Permissions::from_mode(0o755),
5009 )
5010 .unwrap();
5011 git::git(&candidate, &["add", "."]).await.unwrap();
5012 git::git(&candidate, &["commit", "-m", "binary deletion and mode"])
5013 .await
5014 .unwrap();
5015 let head = git::rev_parse(&candidate, "HEAD").await.unwrap();
5016 let tree = git::rev_parse(&candidate, "HEAD^{tree}").await.unwrap();
5017 let base_tree = git::rev_parse(&candidate, &format!("{}^{{tree}}", f.base))
5018 .await
5019 .unwrap();
5020 historical_review(&mut state, &head);
5021 historical_review(&mut state, &head);
5022 state.status = RunStatus::Superseded;
5023 state.save_under(&f.home).unwrap();
5024 let task = f.task(std::slice::from_ref(&state));
5025 let before = serde_json::to_value(&state).unwrap();
5026 git::git(&review, &["reset", "--hard", &head])
5027 .await
5028 .unwrap();
5029 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5030 assert_eq!(report.evicted_copies, 1, "{report:?}");
5031 assert!(!review.exists());
5032 assert!(candidate.exists(), "selected implementation survives");
5033 assert!(report.source_unlinked_logical_bytes > 0);
5034 assert!(report.source_unlinked_allocated_bytes.is_some());
5035 assert!(report.source_evidence_added_allocated_bytes.is_some());
5036 assert_eq!(
5037 serde_json::to_value(RunState::load_under(&state.id, &f.home).unwrap()).unwrap(),
5038 before
5039 );
5040 assert_eq!(
5041 crate::queue::Queue::at(f.home.join("queue"))
5042 .get(&task.id)
5043 .unwrap()
5044 .runs,
5045 task.runs
5046 );
5047 let archive = f
5048 .home
5049 .join("runs")
5050 .join(&state.id)
5051 .join("artifacts/source-evidence")
5052 .join(format!("{head}-{}", f.base));
5053 let patch = source_diff(&archive).await.unwrap();
5054 assert!(String::from_utf8_lossy(&patch).contains("GIT binary patch"));
5055 assert!(String::from_utf8_lossy(&patch).contains("100755"));
5056 assert!(
5057 git::worktree_remove_clean(&f.repo, &candidate)
5058 .await
5059 .unwrap()
5060 );
5061 git::git(&f.repo, &["branch", "-D", &state.candidates[0].branch])
5062 .await
5063 .unwrap();
5064 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5065 .await
5066 .unwrap();
5067 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5068 assert!(
5069 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
5070 .await
5071 .unwrap()
5072 .ok()
5073 );
5074 std::fs::remove_dir_all(&f.repo).unwrap();
5076 assert_eq!(source_diff(&archive).await.unwrap(), patch);
5077 let restored = f._temp.path().join("restored");
5078 restore_source(&archive, &restored).await.unwrap();
5079 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5080 assert_eq!(
5081 std::fs::read(restored.join("binary")).unwrap(),
5082 b"new\0binary\xfe"
5083 );
5084 assert!(!restored.join("source").exists());
5085 assert_ne!(
5086 std::fs::metadata(restored.join("script"))
5087 .unwrap()
5088 .permissions()
5089 .mode()
5090 & 0o111,
5091 0
5092 );
5093 assert!(restore_source(&archive, &restored).await.is_err());
5094 git::git(&restored, &["read-tree", "--reset", "-u", &base_tree])
5097 .await
5098 .unwrap();
5099 let patch_file = f._temp.path().join("saved.patch");
5100 std::fs::write(&patch_file, &patch).unwrap();
5101 git::git(
5102 &restored,
5103 &["apply", "--index", "--binary", patch_file.to_str().unwrap()],
5104 )
5105 .await
5106 .unwrap();
5107 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
5108 assert_eq!(
5109 std::fs::read(restored.join("binary")).unwrap(),
5110 b"new\0binary\xfe"
5111 );
5112 }
5113
5114 #[tokio::test]
5115 async fn source_eviction_archives_unrelated_tips_and_allows_lineage_references() {
5116 let f = Fx::new().await;
5117 let state = inactive_source(&f, "ec10", true).await;
5118 let review = reviewer(&f, &state, &f.base).await;
5119 let mut successor = inactive_source(&f, "ec11", true).await;
5120 successor.retired_to = Some(state.id.clone());
5121 successor.save_under(&f.home).unwrap();
5122 f.task(&[state.clone(), successor]);
5123
5124 let mut tips = Vec::new();
5125 for contents in ["reflog-only evidence", "ORIG_HEAD-only evidence"] {
5126 std::fs::write(review.join("source"), contents).unwrap();
5127 git::git(&review, &["add", "."]).await.unwrap();
5128 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5129 let tip = git::git(
5130 &review,
5131 &["commit-tree", &tree, "-p", &f.base, "-m", contents],
5132 )
5133 .await
5134 .unwrap();
5135 git::git(&review, &["reset", "--hard", &tip]).await.unwrap();
5136 tips.push(tip);
5137 git::git(&review, &["reset", "--hard", &f.base])
5138 .await
5139 .unwrap();
5140 }
5141 git::git(&review, &["reflog", "delete", "--rewrite", "HEAD@{1}"])
5143 .await
5144 .unwrap();
5145 git::git(&review, &["update-ref", "ORIG_HEAD", &tips[1]])
5146 .await
5147 .unwrap();
5148 let mut report = Report::default();
5149 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5150 assert_eq!(report.evicted_copies, 1, "{report:?}");
5151 assert!(!review.exists());
5152 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5153 .await
5154 .unwrap();
5155 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5156 for tip in &tips {
5157 assert!(
5158 !git::git_raw(&f.repo, &["cat-file", "-e", tip])
5159 .await
5160 .unwrap()
5161 .ok()
5162 );
5163 }
5164 std::fs::remove_dir_all(&f.repo).unwrap();
5166 let shown = seat_evidence(&f.home, &state.id).await.unwrap();
5167 for (i, tip) in tips.iter().enumerate() {
5168 assert!(
5169 shown
5170 .iter()
5171 .any(|row| row.contains(tip) && row.contains("patch_bytes="))
5172 );
5173 let destination = f._temp.path().join(format!("restore-history-{i}"));
5174 restore_seat(&f.home, &state.id, tip, &destination)
5175 .await
5176 .unwrap();
5177 assert_eq!(
5178 std::fs::read_to_string(destination.join("source")).unwrap(),
5179 ["reflog-only evidence", "ORIG_HEAD-only evidence"][i]
5180 );
5181 }
5182 assert!(
5183 restore_seat(
5184 &f.home,
5185 &state.id,
5186 "unknown",
5187 &f._temp.path().join("missing")
5188 )
5189 .await
5190 .is_err()
5191 );
5192 }
5193
5194 #[tokio::test]
5195 async fn source_eviction_revalidates_archive_immediately_before_unlink() {
5196 fn corrupt(path: &Path) -> Result<()> {
5197 let root = path.parent().unwrap();
5198 let marker = root.join("archive-to-corrupt");
5201 let archive = std::fs::read_to_string(&marker)?;
5202 std::fs::write(Path::new(&archive).join("change.patch"), "corrupt")?;
5203 Ok(())
5204 }
5205 let f = Fx::new().await;
5206 let state = inactive_source(&f, "ec12", true).await;
5207 let review = reviewer(&f, &state, &f.base).await;
5208 let archive = f
5209 .home
5210 .join("runs")
5211 .join(&state.id)
5212 .join("artifacts/source-evidence")
5213 .join(format!("{}-{}", f.base, f.base));
5214 std::fs::write(
5215 state.worktree_root().join("archive-to-corrupt"),
5216 archive.to_str().unwrap(),
5217 )
5218 .unwrap();
5219 let mut report = Report::default();
5220 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut report, corrupt).await;
5221 assert_eq!(report.evicted_copies, 0, "{report:?}");
5222 assert!(review.exists());
5223 assert!(
5224 report
5225 .retained
5226 .iter()
5227 .any(|r| r.contains("corrupt snapshot")),
5228 "{report:?}"
5229 );
5230 }
5231
5232 #[tokio::test]
5233 async fn source_eviction_fails_closed_for_dirty_ignored_borrowed_and_authoritative_copies() {
5234 for kind in [
5235 "dirty",
5236 "hidden",
5237 "ignored",
5238 "empty-directory",
5239 "untracked",
5240 "borrowed",
5241 "lease",
5242 "owner",
5243 "driver",
5244 "task-pin",
5245 "task-replacement",
5246 "malformed-orig-head",
5247 "grace",
5248 "resumable",
5249 "archive-symlink",
5250 "root-symlink",
5251 "public",
5252 "checkpoint",
5253 "symlink",
5254 "retained",
5255 "unreadable",
5256 "unknown-field",
5257 "unknown-task-field",
5258 ] {
5259 let f = Fx::new().await;
5260 let mut state = inactive_source(&f, "ec02", true).await;
5261 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5262 .await
5263 .unwrap();
5264 historical_review(&mut state, &head);
5265 let review = reviewer(&f, &state, &head).await;
5266 let mut output = None;
5267 let mut owner = None;
5268 let mut driver = None;
5269 let mut disk = Fx::disk();
5270 match kind {
5271 "dirty" => std::fs::write(review.join("source"), "notes").unwrap(),
5272 "hidden" => {
5273 git::git(&review, &["update-index", "--assume-unchanged", "source"])
5274 .await
5275 .unwrap();
5276 std::fs::write(review.join("source"), "notes").unwrap();
5277 }
5278 "ignored" => {
5279 std::fs::create_dir(review.join("scratch")).unwrap();
5280 std::fs::write(review.join("scratch/notes"), "notes").unwrap();
5281 }
5282 "empty-directory" => std::fs::create_dir(review.join("owner-notes")).unwrap(),
5283 "untracked" => std::fs::write(review.join("notes"), "notes").unwrap(),
5284 "borrowed" => {
5285 let mut other = f.run("ec03", true).await;
5286 other.candidates[0].worktree = review.clone();
5287 other.save_under(&f.home).unwrap();
5288 }
5289 "lease" => output = OutputLease::borrow(&review).unwrap(),
5290 "owner" => owner = Some(crate::pr_head::owner_lock(&f.home, &state.id).unwrap()),
5291 "driver" => driver = Some(crate::driver::maintenance(&f.home, &state.id).unwrap()),
5292 "task-pin" => {
5293 let mut task = f.task(std::slice::from_ref(&state));
5294 task.resume_override = Some(crate::queue::OperatorResume {
5295 question_id: "fixture-pin".into(),
5296 at: Fx::now(),
5297 conductor_rehold: None,
5298 forced: true,
5299 pinned_run: Some(state.id.clone()),
5300 });
5301 crate::queue::Queue::at(f.home.join("queue"))
5302 .put(&mut task)
5303 .unwrap();
5304 }
5305 "task-replacement" => {
5306 let mut task = f.task(std::slice::from_ref(&state));
5307 task.replacement = Some(crate::merge_decision::Replacement {
5308 question: "fixture-replacement".into(),
5309 run: state.id.clone(),
5310 pr: "https://example.invalid/pull/1".into(),
5311 agent: "worker".into(),
5312 context: String::new(),
5313 });
5314 crate::queue::Queue::at(f.home.join("queue"))
5315 .put(&mut task)
5316 .unwrap();
5317 }
5318 "malformed-orig-head" => {
5319 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
5320 .await
5321 .unwrap();
5322 std::fs::write(Path::new(&admin).join("ORIG_HEAD"), "not-an-object-id")
5323 .unwrap();
5324 }
5325 "grace" => disk.fold_grace_secs = 3600,
5326 "resumable" => state.status = RunStatus::Blocked,
5327 "archive-symlink" => {
5328 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
5329 std::fs::create_dir_all(&artifacts).unwrap();
5330 let outside = f._temp.path().join("outside");
5331 std::fs::create_dir(&outside).unwrap();
5332 std::os::unix::fs::symlink(&outside, artifacts.join("source-evidence"))
5333 .unwrap();
5334 }
5335 "root-symlink" => {
5336 let root = state.worktree_root();
5337 let outside = f._temp.path().join("outside-root");
5338 std::fs::rename(&root, &outside).unwrap();
5339 std::os::unix::fs::symlink(&outside, &root).unwrap();
5340 }
5341 "public" => state.pr_head = Some(state.candidates[0].branch.clone()),
5342 "checkpoint" => {
5343 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
5344 .await
5345 .unwrap();
5346 std::fs::write(Path::new(&admin).join("MERGE_MSG"), "checkpoint").unwrap();
5347 }
5348 "symlink" => {
5349 std::fs::remove_file(review.join("source")).unwrap();
5350 std::os::unix::fs::symlink(".gitignore", review.join("source")).unwrap();
5351 }
5352 "retained" => std::fs::write(
5353 f.home
5354 .join("runs")
5355 .join(&state.id)
5356 .join("retain-recovery-sources"),
5357 "owner",
5358 )
5359 .unwrap(),
5360 "unreadable" => {
5361 std::fs::create_dir_all(f.home.join("runs/20260101-000000-ffff")).unwrap();
5362 }
5363 "unknown-field" | "unknown-task-field" => {}
5364 _ => unreachable!(),
5365 }
5366 state.save_under(&f.home).unwrap();
5367 if kind == "unknown-field" {
5368 let path = f.home.join("runs").join(&state.id).join("run.json");
5369 let mut raw: serde_json::Value =
5370 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5371 raw["future_source_owner"] = serde_json::json!({"path": review});
5372 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
5373 }
5374 if kind == "unknown-task-field" {
5375 let task = f.task(std::slice::from_ref(&state));
5376 let path = crate::queue::Queue::at(f.home.join("queue")).path_of(&task.id);
5377 let mut raw: serde_json::Value =
5378 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5379 raw["future_source_owner"] = serde_json::json!({"path": review});
5380 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
5381 }
5382 let mut report = Report::default();
5383 copies::evict(&f.home, &disk, Fx::now(), &mut report).await;
5384 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5385 assert!(review.exists(), "{kind}: {report:?}");
5386 assert!(!report.retained.is_empty(), "{kind}: {report:?}");
5387 drop(output);
5388 drop(owner);
5389 drop(driver);
5390 }
5391 }
5392
5393 #[tokio::test]
5394 async fn source_eviction_reconciles_interruption_and_refuses_corrupt_archives() {
5395 let f = Fx::new().await;
5396 let state = inactive_source(&f, "ec04", true).await;
5397 let review = reviewer(&f, &state, &f.base).await;
5398 let mut report = Report::default();
5399 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5400 assert_eq!(report.evicted_copies, 1, "{report:?}");
5401 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
5402 let ledger = artifacts.join("eviction-review-1.json");
5403 let mut record: serde_json::Value =
5404 serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
5405 record["phase"] = "pending".into();
5406 std::fs::write(&ledger, serde_json::to_vec(&record).unwrap()).unwrap();
5407 let archive = PathBuf::from(record["evidence"].as_str().unwrap());
5408 let pack = std::fs::read(archive.join("objects.pack")).unwrap();
5409 std::fs::write(archive.join("objects.pack"), b"corrupt").unwrap();
5410 let mut blocked = Report::default();
5411 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
5412 assert!(
5413 blocked
5414 .retained
5415 .iter()
5416 .any(|r| r.contains("corrupt snapshot")),
5417 "{blocked:?}"
5418 );
5419 assert_eq!(
5420 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
5421 ["phase"],
5422 "pending"
5423 );
5424 assert!(
5425 restore_source(&archive, &f._temp.path().join("bad-restore"))
5426 .await
5427 .is_err()
5428 );
5429 std::fs::write(archive.join("objects.pack"), &pack).unwrap();
5430 let mut retried = Report::default();
5431 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
5432 assert_eq!(
5433 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
5434 ["phase"],
5435 "evicted"
5436 );
5437 assert_eq!(retried.evicted_copies, 0);
5438 assert!(!review.exists());
5439 let pending = archive.with_extension("pending");
5441 std::fs::rename(&archive, &pending).unwrap();
5442 assert_eq!(
5443 crate::fold_evidence::snapshot(&state, &f.home, &f.base, 64 * 1024 * 1024)
5444 .await
5445 .unwrap(),
5446 archive
5447 );
5448 assert!(!pending.exists());
5449 }
5450
5451 #[tokio::test]
5452 async fn source_eviction_reclaims_superseded_empty_attempt_without_retiring_lineage() {
5453 let f = Fx::new().await;
5454 let mut empty = inactive_source(&f, "ec05", false).await;
5455 historical_review(&mut empty, &f.base);
5456 empty.status = RunStatus::Superseded;
5457 empty.save_under(&f.home).unwrap();
5458 let source = inactive_source(&f, "ec06", true).await;
5459 let task = f.task(&[empty.clone(), source.clone()]);
5460 let before = serde_json::to_value(&empty).unwrap();
5461 let mut report = Report::default();
5462 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5463 assert_eq!(report.evicted_copies, 1, "{report:?}");
5464 assert!(!empty.candidates[0].worktree.exists());
5465 assert!(source.candidates[0].worktree.exists());
5466 assert!(
5467 git::branch_exists(&f.repo, &empty.candidates[0].branch)
5468 .await
5469 .unwrap()
5470 );
5471 assert_eq!(
5472 serde_json::to_value(RunState::load_under(&empty.id, &f.home).unwrap()).unwrap(),
5473 before
5474 );
5475 let mut retry = Report::default();
5476 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5477 assert_eq!(retry.removed_empty_roots, 1, "{retry:?}");
5478 assert!(!empty.worktree_root().exists());
5479 assert_eq!(
5480 crate::queue::Queue::at(f.home.join("queue"))
5481 .get(&task.id)
5482 .unwrap()
5483 .runs,
5484 task.runs
5485 );
5486 }
5487 #[tokio::test]
5488 async fn source_eviction_retries_pending_intent_and_exact_missing_registration() {
5489 fn interrupt(_: &Path) -> Result<()> {
5490 anyhow::bail!("fixture interruption before unlink")
5491 }
5492 fn vanish(path: &Path) -> Result<()> {
5493 std::fs::remove_dir_all(path)?;
5494 anyhow::bail!("fixture interruption after checkout unlink")
5495 }
5496 for stage in [
5497 "intent",
5498 "missing-registration",
5499 "changed-admin",
5500 "missing-archive",
5501 "wrong-identity",
5502 ] {
5503 let f = Fx::new().await;
5504 let state = inactive_source(&f, "ec07", true).await;
5505 let review = reviewer(&f, &state, &f.base).await;
5506 let admin = PathBuf::from(
5507 git::git(&review, &["rev-parse", "--absolute-git-dir"])
5508 .await
5509 .unwrap(),
5510 );
5511 let mut interrupted = Report::default();
5512 copies::evict_with(
5513 &f.home,
5514 &Fx::disk(),
5515 Fx::now(),
5516 &mut interrupted,
5517 if stage == "missing-registration" || stage == "changed-admin" {
5518 vanish
5519 } else {
5520 interrupt
5521 },
5522 )
5523 .await;
5524 assert_eq!(interrupted.evicted_copies, 0, "{stage}: {interrupted:?}");
5525 let ledger = f
5526 .home
5527 .join("runs")
5528 .join(&state.id)
5529 .join("artifacts/eviction-review-1.json");
5530 let record: serde_json::Value =
5531 serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
5532 assert_eq!(record["phase"], "pending");
5533 if stage == "changed-admin" {
5534 std::fs::write(admin.join("owner-note"), "keep").unwrap();
5535 }
5536 if stage == "missing-archive" {
5537 std::fs::remove_file(
5538 Path::new(record["evidence"].as_str().unwrap()).join("objects.pack"),
5539 )
5540 .unwrap();
5541 }
5542 if stage == "wrong-identity" {
5543 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
5544 .await
5545 .unwrap();
5546 let other =
5547 crate::fold_evidence::snapshot(&state, &f.home, &head, 64 * 1024 * 1024)
5548 .await
5549 .unwrap();
5550 let original = Path::new(record["evidence"].as_str().unwrap());
5551 for name in [
5552 "manifest.json",
5553 "objects.pack",
5554 "change.patch",
5555 "change.index",
5556 ] {
5557 std::fs::copy(other.join(name), original.join(name)).unwrap();
5558 }
5559 }
5560 let mut retried = Report::default();
5561 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
5562 let phase =
5563 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap())
5564 .unwrap()["phase"]
5565 .clone();
5566 if stage == "changed-admin" || stage == "missing-archive" || stage == "wrong-identity" {
5567 assert_eq!(phase, "pending", "{stage}: {retried:?}");
5568 assert!(!retried.retained.is_empty());
5569 if stage == "changed-admin" {
5570 assert!(admin.join("owner-note").exists());
5571 } else {
5572 assert!(review.exists());
5573 }
5574 } else {
5575 assert_eq!(phase, "evicted", "{stage}: {retried:?}");
5576 assert!(!review.exists());
5577 assert!(!admin.exists());
5578 }
5579 }
5580 }
5581 #[tokio::test]
5582 async fn source_eviction_batches_reproducible_seats_without_dropping_task_history() {
5583 let f = Fx::new().await;
5584 let mut runs = Vec::new();
5585 let mut paths = Vec::new();
5586 for tag in ["ec10", "ec11", "ec12", "ec13"] {
5587 let state = inactive_source(&f, tag, true).await;
5588 for name in ["judge-1", "advisor-1"] {
5589 let path = state.worktree_root().join(name);
5590 git::worktree_add_detached(&f.repo, &path, &f.base)
5591 .await
5592 .unwrap();
5593 paths.push(path);
5594 }
5595 runs.push(state);
5596 }
5597 let task = f.task(&runs);
5598 let mut report = Report::default();
5599 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5600 assert_eq!(report.evicted_copies, 5, "{report:?}");
5601 let mut next = Report::default();
5602 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
5603 assert_eq!(next.evicted_copies, 3, "{next:?}");
5604 assert!(paths.iter().all(|p| !p.exists()));
5605 assert!(runs.iter().all(|r| r.candidates[0].worktree.exists()));
5606 assert_eq!(
5607 crate::queue::Queue::at(f.home.join("queue"))
5608 .get(&task.id)
5609 .unwrap()
5610 .runs,
5611 task.runs
5612 );
5613 let limited = Fx::new().await;
5614 let state = inactive_source(&limited, "ec15", true).await;
5615 let copy = reviewer(&limited, &state, &limited.base).await;
5616 let evidence = limited
5617 .home
5618 .join("runs")
5619 .join(&state.id)
5620 .join("artifacts/source-evidence");
5621 std::fs::create_dir_all(&evidence).unwrap();
5622 let retained = evidence.join("reserved-evidence");
5623 std::fs::File::create(&retained)
5624 .unwrap()
5625 .set_len(64 * 1024 * 1024)
5626 .unwrap();
5627 let mut blocked = Report::default();
5628 copies::evict(&limited.home, &Fx::disk(), Fx::now(), &mut blocked).await;
5629 assert_eq!(blocked.evicted_copies, 0, "{blocked:?}");
5630 assert!(copy.exists());
5631 assert_eq!(
5632 std::fs::metadata(&retained).unwrap().len(),
5633 64 * 1024 * 1024
5634 );
5635 assert!(
5636 blocked
5637 .retained
5638 .iter()
5639 .any(|r| r.contains("per-run storage limit")),
5640 "{blocked:?}"
5641 );
5642 }
5643 #[tokio::test]
5644 async fn source_eviction_migrates_historical_records_without_rewriting_them() {
5645 let f = Fx::new().await;
5646 let mut state = inactive_source(&f, "ec14", true).await;
5647 historical_review(&mut state, &f.base);
5648 historical_review(&mut state, &f.base);
5649 state.status = RunStatus::Superseded;
5650 state.save_under(&f.home).unwrap();
5651 let review = reviewer(&f, &state, &f.base).await;
5652 let task = f.task(std::slice::from_ref(&state));
5653 let path = f.home.join("runs").join(&state.id).join("run.json");
5654 let mut raw: serde_json::Value =
5655 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5656 raw["schema"] = 10.into();
5657 raw.as_object_mut().unwrap().remove("output_activity_at");
5658 raw["resource_wait"] = serde_json::Value::Null;
5661 raw["resource_block"] = serde_json::Value::Null;
5662 raw["driver_claimed"] = serde_json::Value::Null;
5663 raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
5664 let graph = raw["config"]["graph"].as_object_mut().unwrap();
5666 let count = graph.remove("implementers").unwrap();
5667 graph.insert("candidates".into(), count);
5668 let queue = crate::queue::Queue::at(f.home.join("queue"));
5669 let task_path = queue.path_of(&task.id);
5670 let mut task_raw: serde_json::Value =
5671 serde_json::from_slice(&std::fs::read(&task_path).unwrap()).unwrap();
5672 task_raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
5673 let task_historical = serde_json::to_vec_pretty(&task_raw).unwrap();
5674 std::fs::write(&task_path, &task_historical).unwrap();
5675 let historical = serde_json::to_vec_pretty(&raw).unwrap();
5676 std::fs::write(&path, &historical).unwrap();
5677 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5678 assert_eq!(report.evicted_copies, 1, "{report:?}");
5679 assert!(!review.exists());
5680 assert_eq!(std::fs::read(&path).unwrap(), historical);
5681 assert_eq!(std::fs::read(&task_path).unwrap(), task_historical);
5682 assert!(state.candidates[0].worktree.exists());
5683 assert_eq!(
5684 crate::queue::Queue::at(f.home.join("queue"))
5685 .get(&task.id)
5686 .unwrap()
5687 .runs,
5688 task.runs
5689 );
5690 }
5691 #[tokio::test]
5692 async fn source_eviction_archives_old_only_reflog_oid_after_unrewritten_deletion() {
5693 let f = Fx::new().await;
5694 let state = inactive_source(&f, "ed01", true).await;
5695 let review = reviewer(&f, &state, &f.base).await;
5696 std::fs::write(review.join("old-only"), b"old-only\0binary\xff").unwrap();
5697 git::git(&review, &["add", "."]).await.unwrap();
5698 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5699 let tip = git::git(
5700 &review,
5701 &[
5702 "commit-tree",
5703 &tree,
5704 "-p",
5705 &f.base,
5706 "-m",
5707 "temporary detached source",
5708 ],
5709 )
5710 .await
5711 .unwrap();
5712 git::git(&review, &["checkout", "--detach", &tip])
5713 .await
5714 .unwrap();
5715 git::git(&review, &["checkout", "--detach", &f.base])
5716 .await
5717 .unwrap();
5718 git::git(&review, &["reflog", "delete", "HEAD@{1}"])
5719 .await
5720 .unwrap();
5721 git::git(&review, &["update-ref", "-d", "ORIG_HEAD"])
5722 .await
5723 .unwrap();
5724 let shown = git::git(&review, &["reflog", "show", "--format=%H", "HEAD"])
5725 .await
5726 .unwrap();
5727 assert!(!shown.lines().any(|oid| oid == tip));
5728 let admin = PathBuf::from(
5729 git::git(&review, &["rev-parse", "--absolute-git-dir"])
5730 .await
5731 .unwrap(),
5732 );
5733 let raw = std::fs::read_to_string(admin.join("logs/HEAD")).unwrap();
5734 assert!(raw.lines().any(|line| line.starts_with(&format!("{tip} "))));
5735 fn interrupt(_: &Path) -> Result<()> {
5736 anyhow::bail!("fixture interruption before unlink")
5737 }
5738 let mut pending = Report::default();
5739 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut pending, interrupt).await;
5740 assert!(review.exists());
5741 let ledger = f
5742 .home
5743 .join("runs")
5744 .join(&state.id)
5745 .join("artifacts/eviction-review-1.json");
5746 let complete = std::fs::read(&ledger).unwrap();
5747 let mut incomplete: serde_json::Value = serde_json::from_slice(&complete).unwrap();
5748 incomplete["history"]
5749 .as_object_mut()
5750 .unwrap()
5751 .remove(&tip)
5752 .unwrap();
5753 std::fs::write(&ledger, serde_json::to_vec(&incomplete).unwrap()).unwrap();
5754 let mut blocked = Report::default();
5755 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
5756 assert!(review.exists());
5757 assert!(
5758 blocked
5759 .retained
5760 .iter()
5761 .any(|reason| reason.contains("incomplete reflog checkpoint")),
5762 "{blocked:?}"
5763 );
5764 std::fs::write(&ledger, complete).unwrap();
5765 let mut report = Report::default();
5766 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5767 assert_eq!(report.evicted_copies, 1, "{report:?}");
5768 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
5769 .await
5770 .unwrap();
5771 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
5772 assert!(
5773 !git::git_raw(&f.repo, &["cat-file", "-e", &tip])
5774 .await
5775 .unwrap()
5776 .ok()
5777 );
5778 std::fs::remove_dir_all(&f.repo).unwrap();
5779 let destination = f._temp.path().join("restore-old-only");
5780 restore_seat(&f.home, &state.id, &tip, &destination)
5781 .await
5782 .unwrap();
5783 assert_eq!(
5784 std::fs::read(destination.join("old-only")).unwrap(),
5785 b"old-only\0binary\xff"
5786 );
5787 }
5788
5789 #[tokio::test]
5790 async fn source_eviction_crashed_driver_lease_does_not_block_cleanup_or_retry() {
5791 fn interrupt(_: &Path) -> Result<()> {
5792 anyhow::bail!("fixture interruption before unlink")
5793 }
5794 fn vanish(path: &Path) -> Result<()> {
5795 std::fs::remove_dir_all(path)?;
5796 anyhow::bail!("fixture interruption before unregistration")
5797 }
5798 for stage in ["ordinary", "intent", "missing-registration", "live-driver"] {
5799 let f = Fx::new().await;
5800 let mut state = inactive_source(&f, "ed02", true).await;
5801 state.driver_exited = false;
5802 state.driver_claimed = Some(true);
5803 state.driver_pid = Some(std::process::id());
5804 state.driver_started_at = crate::proc::process_started_at(std::process::id());
5805 state.begin_pass();
5806 state.status = RunStatus::Superseded;
5807 state.save_under(&f.home).unwrap();
5808 let review = reviewer(&f, &state, &f.base).await;
5809 let identity = serde_json::json!({"pid":state.driver_pid,"started_at":state.driver_started_at,
5812 "pass":state.passes.last().unwrap().started_at,"pass_index":state.passes.last().unwrap().index});
5813 std::fs::write(
5814 f.home.join("runs").join(&state.id).join("driver.json"),
5815 serde_json::to_vec(&identity).unwrap(),
5816 )
5817 .unwrap();
5818 let live = if stage == "live-driver" {
5819 Some(crate::driver::maintenance(&f.home, &state.id).unwrap())
5820 } else {
5821 None
5822 };
5823 assert_eq!(
5824 state.liveness_under(&f.home, false),
5825 if live.is_some() {
5826 Liveness::Live
5827 } else {
5828 Liveness::Dead
5829 }
5830 );
5831 let mut report = Report::default();
5832 if stage == "intent" || stage == "missing-registration" {
5833 copies::evict_with(
5834 &f.home,
5835 &Fx::disk(),
5836 Fx::now(),
5837 &mut report,
5838 if stage == "intent" { interrupt } else { vanish },
5839 )
5840 .await;
5841 assert_eq!(report.evicted_copies, 0, "{stage}: {report:?}");
5842 let mut retry = Report::default();
5843 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5844 assert!(!review.exists(), "{stage}: {retry:?}");
5845 let ledger = f
5846 .home
5847 .join("runs")
5848 .join(&state.id)
5849 .join("artifacts/eviction-review-1.json");
5850 let receipt: serde_json::Value =
5851 serde_json::from_slice(&std::fs::read(ledger).unwrap()).unwrap();
5852 assert_eq!(receipt["phase"], "evicted", "{stage}: {retry:?}");
5853 } else {
5854 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5855 assert_eq!(
5856 report.evicted_copies,
5857 usize::from(live.is_none()),
5858 "{stage}: {report:?}"
5859 );
5860 assert_eq!(review.exists(), live.is_some());
5861 }
5862 drop(live);
5863 assert!(
5864 !RunState::load_under(&state.id, &f.home)
5865 .unwrap()
5866 .driver_exited
5867 );
5868 }
5869 }
5870}