1mod copies;
4pub mod reconcile;
5
6pub(crate) fn has_archived_seats(home: &Path, state: &RunState) -> Result<bool> {
7 let artifacts = home.join("runs").join(&state.id).join("artifacts");
8 if !artifacts.try_exists()? {
9 return Ok(false);
10 }
11 ensure!(
12 plain_dir(&artifacts) && artifacts.canonicalize()? == artifacts,
13 "noncanonical artifacts"
14 );
15 for (i, entry) in std::fs::read_dir(artifacts)?.enumerate() {
16 ensure!(i < 4096, "eviction record inspection limit exceeded");
17 let name = entry?.file_name();
18 let name = name.to_string_lossy();
19 if name.starts_with("eviction-") && name.ends_with(".json") {
20 return Ok(true);
21 }
22 }
23 Ok(false)
24}
25
26pub(crate) async fn restore_execution_seats(
29 home: &Path,
30 state: &RunState,
31 coordination: Option<&crate::repo_lock::SeatLease>,
32) -> Result<()> {
33 copies::restore_execution_seats(home, state, coordination).await
34}
35
36pub async fn source_diff(evidence: &Path) -> Result<Vec<u8>> {
38 crate::fold_evidence::validate_snapshot(evidence).await?;
39 Ok(std::fs::read(evidence.join("change.patch"))?)
40}
41
42pub async fn restore_source(evidence: &Path, destination: &Path) -> Result<()> {
44 crate::fold_evidence::restore_snapshot(evidence, destination).await
45}
46
47pub fn seat_inspections(home: &Path, id: &str) -> Result<Vec<String>> {
49 ensure!(crate::run::is_run_id(id), "invalid run id");
50 copies::seat_inspections(home, id)
51}
52
53pub async fn seat_evidence(home: &Path, id: &str) -> Result<Vec<String>> {
55 copies::seat_evidence(home, id).await
56}
57
58pub async fn seat_heads(
60 home: &Path,
61 id: &str,
62) -> Result<std::collections::BTreeMap<String, String>> {
63 copies::seat_heads(home, id).await
64}
65
66pub async fn seat_diff(home: &Path, id: &str, head: &str) -> Result<Vec<u8>> {
68 copies::seat_diff(home, id, head).await
69}
70
71pub async fn restore_seat(home: &Path, id: &str, head: &str, destination: &Path) -> Result<()> {
73 copies::restore_seat(home, id, head, destination).await
74}
75
76use crate::{
77 git,
78 proc::Quiet as _,
79 run::{Liveness, RunState},
80};
81use anyhow::{Context, Result, ensure};
82use jiff::Timestamp;
83use serde::{Deserialize, Serialize};
84use std::{
85 fs::{File, OpenOptions},
86 path::{Path, PathBuf},
87};
88
89#[derive(Debug)]
93pub(crate) struct OutputLease {
94 _file: std::sync::Arc<OutputFile>,
95 _activity: bool,
96}
97
98#[derive(Debug)]
99struct OutputFile(File);
100
101impl std::ops::Deref for OutputFile {
102 type Target = File;
103 fn deref(&self) -> &File {
104 &self.0
105 }
106}
107
108#[cfg(unix)]
109impl Drop for OutputFile {
110 fn drop(&mut self) {
111 use std::os::fd::AsRawFd;
112 unsafe {
114 libc::flock(self.0.as_raw_fd(), libc::LOCK_UN);
115 }
116 }
117}
118
119impl OutputLease {
120 #[cfg(unix)]
121 pub(crate) fn descriptor(&self) -> std::os::fd::RawFd {
122 use std::os::fd::AsRawFd;
123 self._file.as_raw_fd()
124 }
125
126 pub(crate) fn pre_clean(dir: &Path) -> Result<Self> {
128 let file = OpenOptions::new()
129 .create(true)
130 .truncate(false)
131 .read(true)
132 .write(true)
133 .open(dir.join("magi-pre-clean.lock"))?;
134 crate::pr_head::try_lock_file(&file)
135 .context("pre-clean already running; retry after descendants terminate")?;
136 Ok(Self {
137 _file: std::sync::Arc::new(OutputFile(file)),
138 _activity: false,
139 })
140 }
141
142 pub(crate) fn duplicate(&self) -> Self {
143 Self {
144 _file: std::sync::Arc::clone(&self._file),
145 _activity: false,
146 }
147 }
148
149 pub(crate) fn borrow(worktree: &Path) -> Result<Option<Self>> {
150 if !worktree.join(".git").is_file() {
151 return Ok(None);
152 }
153 Self::take(worktree, false).map(Some)
154 }
155 pub(crate) fn cache(home: &Path, cache: &Path, exclusive: bool) -> Result<Self> {
156 use std::hash::{Hash, Hasher};
157 let mut hash = std::collections::hash_map::DefaultHasher::new();
158 let mut normalized = cache.to_owned();
159 let mut missing = Vec::new();
160 while !normalized.exists() {
161 let Some(name) = normalized.file_name() else {
162 break;
163 };
164 missing.push(name.to_owned());
165 if !normalized.pop() {
166 break;
167 }
168 }
169 normalized = normalized.canonicalize().unwrap_or(normalized);
170 for part in missing.into_iter().rev() {
171 normalized.push(part);
172 }
173 normalized.hash(&mut hash);
174 let dir = home.join("cache-leases");
175 std::fs::create_dir_all(&dir)?;
176 Self::lock(
177 &dir.join(format!("outputs-{:x}.lock", hash.finish())),
178 exclusive,
179 )
180 }
181 pub(crate) fn admit(home: &Path, paths: &[PathBuf], min_free: u64) -> Result<Vec<Self>> {
185 #[cfg(unix)]
186 return Self::admit_isolated(home, paths, min_free);
187 #[cfg(not(unix))]
188 {
189 let _ = (home, paths, min_free);
190 Ok(Vec::new())
191 }
192 }
193
194 #[cfg(any(unix, windows))]
195 pub(crate) fn admit_isolated(
196 home: &Path,
197 paths: &[PathBuf],
198 min_free: u64,
199 ) -> Result<Vec<Self>> {
200 Self::admit_with(home, paths, min_free, crate::disk::managed_free_bytes)
201 }
202
203 #[cfg(any(unix, windows))]
204 pub(crate) fn admit_with(
205 home: &Path,
206 paths: &[PathBuf],
207 min_free: u64,
208 free_bytes: impl Fn(&Path) -> Result<u64>,
209 ) -> Result<Vec<Self>> {
210 #[cfg(unix)]
211 use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
212 #[cfg(windows)]
213 use std::os::windows::fs::OpenOptionsExt;
214 if min_free == 0 {
215 return Ok(Vec::new());
216 }
217 const RESERVE: u64 = 1024 * 1024 * 1024;
218 std::fs::create_dir_all(home)?;
219 let dir = home.canonicalize()?.join("disk-admission");
220 std::fs::create_dir_all(&dir)?;
221 ensure!(
222 plain_dir(&dir) && dir.canonicalize()? == dir,
223 "disk admission directory is not canonical"
224 );
225 let mut devices = std::collections::BTreeSet::new();
226 let mut reservations = Vec::new();
227 for path in paths {
228 let probe = path
229 .ancestors()
230 .find(|p| p.exists())
231 .context("disk admission path missing")?;
232 #[cfg(unix)]
233 let device = std::fs::metadata(probe)?.dev();
234 #[cfg(windows)]
235 let device = windows_volume(probe)?;
236 if !devices.insert(device) {
237 continue;
238 }
239 let open = |path: &Path| -> Result<File> {
240 let mut opts = OpenOptions::new();
241 opts.read(true).write(true).create(true).truncate(false);
242 #[cfg(unix)]
243 opts.custom_flags(libc::O_NOFOLLOW);
244 #[cfg(windows)]
245 opts.custom_flags(0x0020_0000); let file = opts.open(path)?;
247 #[cfg(windows)]
248 {
249 use std::os::windows::fs::MetadataExt as _;
250 ensure!(
251 file.metadata()?.file_attributes() & 0x400 == 0,
252 "disk admission lock is a reparse point"
253 );
254 }
255 Ok(file)
256 };
257 let admission = Self {
261 _file: std::sync::Arc::new(OutputFile(open(&dir.join(format!("{device}.lock")))?)),
262 _activity: false,
263 };
264 let started = std::time::Instant::now();
265 loop {
266 match crate::pr_head::try_lock_file(&admission._file) {
267 Ok(()) => break,
268 Err(error)
269 if lock_contended(&error)
270 && started.elapsed() < std::time::Duration::from_secs(1) =>
271 {
272 std::thread::sleep(std::time::Duration::from_millis(5));
273 }
274 Err(error) => {
275 return Err(error)
276 .context("disk admission busy or unavailable; retry without dispatch");
277 }
278 }
279 }
280 let mut active = 0u64;
281 let mut available = None;
282 for slot in 0..1024 {
283 let file = dir.join(format!("{device}-{slot}.reserve"));
284 if !file.exists() {
285 if available.is_none() {
286 available = Some(file);
287 }
288 continue;
289 }
290 let file = open(&file)?;
291 match crate::pr_head::try_lock_file(&file) {
292 Ok(()) if available.is_none() => {
293 available = Some(dir.join(format!("{device}-{slot}.reserve")))
294 }
295 Ok(()) => {}
296 Err(_) => active = active.saturating_add(RESERVE),
297 }
298 }
299 let free = free_bytes(probe)?;
300 let required = min_free.saturating_add(active).saturating_add(RESERVE);
301 ensure!(
302 free >= required,
303 "disk admission blocked: {free} bytes free; floor {min_free}, active writer reservations {active}, next writer reservation {RESERVE} bytes (estimates, not quotas)"
304 );
305 let file = open(&available.context("disk admission reservation slots exhausted")?)?;
306 crate::pr_head::try_lock_file(&file)?;
307 reservations.push(Self {
308 _file: std::sync::Arc::new(OutputFile(file)),
309 _activity: false,
310 });
311 }
312 Ok(reservations)
313 }
314
315 #[cfg(not(any(unix, windows)))]
316 pub(crate) fn admit_isolated(
317 _home: &Path,
318 _paths: &[PathBuf],
319 _min_free: u64,
320 ) -> Result<Vec<Self>> {
321 anyhow::bail!("disk admission is unavailable on this platform")
322 }
323
324 pub(crate) fn take(worktree: &Path, exclusive: bool) -> Result<Self> {
325 let out = std::process::Command::new("git")
326 .args(["rev-parse", "--absolute-git-dir"])
327 .current_dir(worktree)
328 .quiet()
329 .output()?;
330 ensure!(
331 out.status.success(),
332 "cannot identify output lease directory"
333 );
334 let dir = PathBuf::from(String::from_utf8(out.stdout)?.trim());
335 Self::lock(&dir.join("magi-output.lock"), exclusive)
336 }
337 pub(crate) fn lock(path: &Path, exclusive: bool) -> Result<Self> {
338 let file = OpenOptions::new()
339 .create(true)
340 .truncate(false)
341 .read(true)
342 .write(true)
343 .open(path)
344 .with_context(|| format!("output lease: open {}", path.display()))?;
345 #[cfg(unix)]
346 {
347 use std::os::fd::AsRawFd;
348 let mode = if exclusive {
349 libc::LOCK_EX
350 } else {
351 libc::LOCK_SH
352 };
353 ensure!(
355 unsafe { libc::flock(file.as_raw_fd(), mode | libc::LOCK_NB) } == 0,
356 "output lease: outputs are in use or being reclaimed: {}",
357 path.display()
358 );
359 }
360 #[cfg(windows)]
361 {
362 use std::os::windows::io::AsRawHandle as _;
363 ensure!(
364 crate::chat_watch::windows::lock_file_mode(file.as_raw_handle(), true, exclusive)?,
365 "output lease: outputs are in use or being reclaimed: {}",
366 path.display()
367 );
368 }
369 #[cfg(not(any(unix, windows)))]
370 let _ = exclusive;
371 if !exclusive {
372 file.set_len(1)?;
374 file.set_modified(std::time::SystemTime::now())
375 .with_context(|| format!("record output activity {}", path.display()))?;
376 }
377 Ok(Self {
378 _file: std::sync::Arc::new(OutputFile(file)),
379 _activity: !exclusive,
380 })
381 }
382}
383
384#[cfg(any(unix, windows))]
385fn lock_contended(error: &std::io::Error) -> bool {
386 matches!(
387 error.kind(),
388 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted
389 ) || cfg!(windows) && error.raw_os_error() == Some(33)
390}
391
392#[cfg(windows)]
393fn windows_volume(path: &Path) -> Result<u128> {
394 use std::os::windows::ffi::OsStrExt as _;
395 #[link(name = "kernel32")]
396 unsafe extern "system" {
397 fn GetVolumePathNameW(path: *const u16, mount: *mut u16, length: u32) -> i32;
398 fn GetVolumeNameForVolumeMountPointW(mount: *const u16, name: *mut u16, length: u32)
399 -> i32;
400 }
401 let path: Vec<_> = path
402 .canonicalize()?
403 .as_os_str()
404 .encode_wide()
405 .chain(Some(0))
406 .collect();
407 let mut mount = vec![0u16; 32_768];
408 let mut name = [0u16; 64];
409 if unsafe { GetVolumePathNameW(path.as_ptr(), mount.as_mut_ptr(), mount.len() as u32) } == 0
411 || unsafe {
412 GetVolumeNameForVolumeMountPointW(mount.as_ptr(), name.as_mut_ptr(), name.len() as u32)
413 } == 0
414 {
415 return Err(std::io::Error::last_os_error()).context("identify disk admission volume");
416 }
417 let end = name
418 .iter()
419 .position(|&c| c == 0)
420 .context("invalid volume name")?;
421 let name = String::from_utf16(&name[..end])?;
422 let id = name
423 .strip_prefix(r"\\?\Volume{")
424 .and_then(|s| s.strip_suffix("}\\"))
425 .context("volume has no stable GUID")?;
426 ensure!(
427 id.bytes().all(|b| b.is_ascii_hexdigit() || b == b'-'),
428 "invalid volume GUID"
429 );
430 Ok(u128::from_str_radix(&id.replace('-', ""), 16)?)
431}
432
433#[cfg(all(test, windows))]
434#[test]
435fn isolated_windows_reservations_share_the_actual_volume_budget() {
436 let home = tempfile::tempdir().unwrap();
437 let first = home.path().join("first checkout");
438 let second = home.path().join("second 日本語 checkout");
439 std::fs::create_dir(&first).unwrap();
440 std::fs::create_dir(&second).unwrap();
441 let reserve =
442 |paths: &[PathBuf]| OutputLease::admit_with(home.path(), paths, 1, |_| Ok(1 + (1 << 30)));
443 let writer = reserve(&[first, second.clone()]).unwrap();
444 let err = reserve(std::slice::from_ref(&second)).unwrap_err();
445 assert!(
446 err.to_string()
447 .contains("active writer reservations 1073741824")
448 );
449 drop(writer);
450 assert!(reserve(&[second]).is_ok());
451}
452
453#[cfg(unix)]
454impl Drop for OutputLease {
455 fn drop(&mut self) {
456 if self._activity
457 && let Err(error) = self._file.set_modified(std::time::SystemTime::now())
458 {
459 let _ = self._file.set_len(2);
462 tracing::warn!("could not record output lease release activity: {error}");
463 }
464 }
465}
466
467#[derive(Default, Debug, Serialize, Deserialize)]
468#[serde(default)]
469pub(crate) struct Report {
470 pub checked_at: Option<Timestamp>,
471 pub limit_bytes: u64,
472 pub reclaim_budget_bytes: u64,
473 pub measured_bytes: u64,
475 pub freed_bytes: u64,
477 pub measured_allocated_file_bytes: Option<u64>,
478 pub proven_reclaimable_bytes: u64,
480 pub proven_reclaimable_allocated_bytes: Option<u64>,
483 pub protected_or_unproven_bytes: u64,
484 pub estimated_released_allocated_bytes: Option<u64>,
487 pub observed_home_free_change_bytes: Option<i64>,
490 pub retired: usize,
491 pub evicted_copies: usize,
492 pub source_scan: Option<serde_json::Value>,
493 pub removed_empty_roots: usize,
494 pub source_unlinked_logical_bytes: u64,
495 pub source_unlinked_allocated_bytes: Option<u64>,
496 pub source_evidence_added_logical_bytes: u64,
497 pub source_evidence_added_allocated_bytes: Option<u64>,
498 pub retained: Vec<String>,
499 pub failures: Vec<String>,
500}
501
502#[cfg(unix)]
503fn allocated_inventory(
504 root: &Path,
505 seen: &mut std::collections::BTreeSet<(u64, u64)>,
506) -> Result<u64> {
507 use std::os::unix::fs::MetadataExt;
508 ensure!(
509 plain_dir(root) && root.canonicalize()? == root,
510 "allocated inventory root is not canonical"
511 );
512 let mut dirs = vec![root.to_owned()];
513 let mut bytes = 0u64;
514 while let Some(dir) = dirs.pop() {
515 for entry in std::fs::read_dir(dir)? {
516 let path = entry?.path();
517 let metadata = std::fs::symlink_metadata(&path)?;
518 if metadata.is_dir() {
519 dirs.push(path);
520 } else if metadata.is_file() && seen.insert((metadata.dev(), metadata.ino())) {
521 bytes = bytes.saturating_add(metadata.blocks().saturating_mul(512));
522 }
523 }
524 }
525 Ok(bytes)
526}
527
528#[derive(Default)]
529struct ReleasedAllocation {
530 #[cfg(unix)]
531 files: std::collections::BTreeMap<PathBuf, (u64, u64, u64, u64)>,
532 #[cfg(unix)]
533 removed: std::collections::BTreeMap<(u64, u64), u64>,
534}
535
536impl ReleasedAllocation {
537 fn of(paths: &[PathBuf]) -> Result<Self> {
538 #[cfg(unix)]
539 {
540 use std::os::unix::fs::MetadataExt;
541 let mut files = std::collections::BTreeMap::new();
542 for path in paths {
543 let m = std::fs::symlink_metadata(path)?;
544 ensure!(m.is_file(), "allocation candidate is not regular");
545 files.insert(
546 path.clone(),
547 (m.dev(), m.ino(), m.nlink(), m.blocks().saturating_mul(512)),
548 );
549 }
550 Ok(Self {
551 files,
552 removed: Default::default(),
553 })
554 }
555 #[cfg(not(unix))]
556 {
557 let _ = paths;
558 Ok(Self::default())
559 }
560 }
561 fn reclaimable_allocated_bytes(&self) -> Option<u64> {
562 #[cfg(unix)]
563 {
564 let mut inodes = std::collections::BTreeMap::new();
565 for &(device, inode, links, blocks) in self.files.values() {
566 let entry = inodes
567 .entry((device, inode))
568 .or_insert((0u64, links, blocks));
569 entry.0 += 1;
570 }
571 Some(
572 inodes
573 .values()
574 .filter(|(count, links, _)| count == links)
575 .fold(0u64, |sum, (_, _, blocks)| sum.saturating_add(*blocks)),
576 )
577 }
578 #[cfg(not(unix))]
579 {
580 None
581 }
582 }
583
584 fn unlinked(&mut self, path: &Path) -> Option<u64> {
585 #[cfg(unix)]
586 {
587 let (device, inode, links, blocks) = self.files[path];
588 let removed = self.removed.entry((device, inode)).or_default();
589 *removed += 1;
590 Some(if *removed == links { blocks } else { 0 })
591 }
592 #[cfg(not(unix))]
593 {
594 let _ = path;
595 None
596 }
597 }
598}
599
600fn plain_dir(path: &Path) -> bool {
601 std::fs::symlink_metadata(path).is_ok_and(|m| m.is_dir() && !m.file_type().is_symlink())
602}
603
604pub(crate) fn read_seat(path: &Path) -> bool {
605 path.file_name()
606 .and_then(|s| s.to_str())
607 .is_some_and(|name| {
608 ["review-", "judge-", "advisor-"].iter().any(|prefix| {
609 name.strip_prefix(prefix)
610 .is_some_and(|n| n.parse::<usize>().is_ok_and(|n| n > 0))
611 })
612 })
613}
614
615fn run_paths_missing(state: &RunState) -> Result<bool> {
618 for path in [state.worktree_root(), state.repo.clone()] {
619 match std::fs::metadata(&path) {
620 Ok(_) => {}
621 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(true),
622 Err(e) => return Err(e).with_context(|| format!("inspect {}", path.display())),
623 }
624 }
625 Ok(false)
626}
627
628pub(crate) async fn worktrees(state: &RunState) -> Result<Vec<PathBuf>> {
630 let root = state.worktree_root();
631 if !plain_dir(&root) {
632 return Ok(Vec::new());
633 }
634 let list = git::git(&state.repo, &["worktree", "list", "--porcelain"]).await?;
635 registered_worktrees(state, &list)
636}
637
638fn registered_worktrees(state: &RunState, list: &str) -> Result<Vec<PathBuf>> {
639 let root = state.worktree_root();
640 if !plain_dir(&root) {
641 return Ok(Vec::new());
642 }
643 let root = root.canonicalize()?;
644 let mut paths = Vec::new();
645 for path in list.lines().filter_map(|l| l.strip_prefix("worktree ")) {
646 let path = PathBuf::from(path);
647 #[cfg(windows)]
648 let path = if plain_dir(&path) {
649 git::canonical_output_path(path.to_str().context("non UTF-8 registration")?)?
650 } else {
651 continue;
652 };
653 if plain_dir(&path)
654 && path.parent() == Some(root.as_path())
655 && path.canonicalize()? == path
656 && path.join(".git").is_file()
657 {
658 paths.push(path);
659 }
660 }
661 Ok(paths)
662}
663
664async fn cargo_target(worktree: &Path) -> Result<PathBuf> {
667 validate_cargo_target(worktree, false).await
668}
669
670async fn validate_cargo_target(worktree: &Path, allow_missing_tag: bool) -> Result<PathBuf> {
671 let target = worktree.join("target");
672 ensure!(plain_dir(&target), "target absent or not a plain directory");
673 ensure!(target.canonicalize()? == target, "target path changed");
674 ensure!(
675 git::git(worktree, &["ls-files", "--", "target"])
676 .await?
677 .is_empty(),
678 "tracked target contents"
679 );
680 ensure!(
681 git::git_raw(worktree, &["check-ignore", "--quiet", "target/"])
682 .await?
683 .ok(),
684 "target is not ignored"
685 );
686 cache_tag(&target, allow_missing_tag)?;
687 let path = target.join(".rustc_info.json");
688 plain_file(&path, "rustc identity")?;
689 let info: serde_json::Value = serde_json::from_slice(
690 &std::fs::read(&path).with_context(|| format!("read rustc identity {}", path.display()))?,
691 )
692 .with_context(|| format!("parse rustc identity {}", path.display()))?;
693 ensure!(
694 info.get("rustc_fingerprint").is_some(),
695 "unknown Cargo identity {}",
696 path.display()
697 );
698 if allow_missing_tag && !target.join("CACHEDIR.TAG").exists() {
699 ensure!(
700 info.get("rustc_fingerprint")
701 .is_some_and(serde_json::Value::is_u64)
702 && info
703 .get("outputs")
704 .and_then(serde_json::Value::as_object)
705 .is_some_and(|outputs| outputs.values().any(|output| {
706 output.get("success").and_then(serde_json::Value::as_bool) == Some(true)
707 && output
708 .get("stdout")
709 .and_then(serde_json::Value::as_str)
710 .is_some_and(|stdout| stdout.starts_with("rustc "))
711 && output
712 .get("stderr")
713 .is_some_and(serde_json::Value::is_string)
714 }))
715 && info
716 .get("successes")
717 .is_some_and(serde_json::Value::is_object),
718 "missing-tag rustc identity structure invalid: {}",
719 path.display()
720 );
721 let mut profiles = 0;
722 for profile in ["debug", "release"] {
723 let dir = target.join(profile);
724 if !dir.exists() {
725 continue;
726 }
727 profile_output_dirs(&dir, true)?;
728 plain_file(&dir.join(".cargo-lock"), "missing-tag Cargo lock")?;
729 profiles += 1;
730 }
731 ensure!(
732 profiles > 0,
733 "missing-tag proof: no Cargo profile in {}",
734 target.display()
735 );
736 }
737 for name in ["debug", "release"] {
738 let path = target.join(name);
739 match std::fs::symlink_metadata(&path) {
740 Ok(metadata) => ensure!(
741 metadata.is_dir() && path.canonicalize()? == path,
742 "profile validation: not a plain directory: {}",
743 path.display()
744 ),
745 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
746 Err(error) => {
747 return Err(error)
748 .with_context(|| format!("profile validation: inspect {}", path.display()));
749 }
750 }
751 }
752 Ok(target)
753}
754
755fn plain_file(path: &Path, stage: &str) -> Result<()> {
756 let meta = std::fs::symlink_metadata(path)
757 .with_context(|| format!("{stage}: inspect {}", path.display()))?;
758 ensure!(
759 meta.is_file() && path.canonicalize()? == path,
760 "{stage}: not a plain file: {}",
761 path.display()
762 );
763 Ok(())
764}
765
766fn cache_tag(target: &Path, allow_missing: bool) -> Result<()> {
767 let path = target.join("CACHEDIR.TAG");
768 match std::fs::symlink_metadata(&path) {
769 Err(error) if allow_missing && error.kind() == std::io::ErrorKind::NotFound => {
770 return Ok(());
771 }
772 result => {
773 result.with_context(|| format!("cache tag: inspect {}", path.display()))?;
774 }
775 }
776 plain_file(&path, "cache tag")?;
777 let tag = std::fs::read_to_string(&path)
778 .with_context(|| format!("cache tag: read {}", path.display()))?;
779 ensure!(
780 tag.starts_with("Signature: 8a477f597d28d172789f06886806bc55"),
781 "unknown Cargo cache tag: {}",
782 path.display()
783 );
784 Ok(())
785}
786
787fn output_age(target: &Path, now: Timestamp, grace: u64) -> Result<()> {
791 let mut dirs = vec![target.to_owned()];
792 let mut entries = 0;
793 while let Some(dir) = dirs.pop() {
794 for entry in std::fs::read_dir(&dir)
795 .with_context(|| format!("output activity: list {}", dir.display()))?
796 {
797 let path = entry?.path();
798 entries += 1;
799 ensure!(
800 entries <= 100_000,
801 "output activity inventory too large: {}",
802 target.display()
803 );
804 let metadata = std::fs::symlink_metadata(&path)
805 .with_context(|| format!("output activity: inspect {}", path.display()))?;
806 if metadata.is_dir() {
807 if dir != target
809 || matches!(
810 path.file_name().and_then(|s| s.to_str()),
811 Some("debug" | "release")
812 )
813 {
814 dirs.push(path);
815 }
816 } else if metadata.is_file()
817 && (dir != target || path.file_name().is_some_and(|n| n == ".rustc_info.json"))
818 {
819 let at =
820 Timestamp::try_from(metadata.modified().with_context(|| {
821 format!("output activity: modified {}", path.display())
822 })?)?;
823 ensure!(
824 crate::clean::due(now, at, grace),
825 "output activity within grace period: {} modified {at} (now {now}, grace {grace}s)",
826 path.display()
827 );
828 }
829 }
830 }
831 Ok(())
832}
833
834pub(crate) fn artifact_signature(path: &Path) -> Result<bool> {
835 use std::io::Read as _;
836 let mut header = [0; 8];
837 let n = File::open(path)?.read(&mut header)?;
838 let header = &header[..n];
839 Ok(header.starts_with(b"\x7fELF")
840 || header.starts_with(b"!<arch>\n")
841 || header.starts_with(b"rust\0\0\0")
842 || header.starts_with(b"MZ\x90\0")
843 || [
844 [0xfe, 0xed, 0xfa, 0xce],
845 [0xce, 0xfa, 0xed, 0xfe],
846 [0xfe, 0xed, 0xfa, 0xcf],
847 [0xcf, 0xfa, 0xed, 0xfe],
848 [0xca, 0xfe, 0xba, 0xbe],
849 [0xca, 0xfe, 0xba, 0xbf],
850 ]
851 .iter()
852 .any(|magic| header.starts_with(magic)))
853}
854
855fn profile_output_dirs(profile: &Path, require_proof: bool) -> Result<Vec<PathBuf>> {
859 let mut present = std::collections::BTreeSet::new();
860 for path in [
861 profile.to_owned(),
862 profile.join("deps"),
863 profile.join(".fingerprint"),
864 profile.join("build"),
865 ] {
866 match std::fs::symlink_metadata(&path) {
867 Ok(_) => {
868 ensure!(
869 plain_dir(&path) && path.canonicalize()? == path,
870 "Cargo profile proof: {} is not a plain directory",
871 path.display()
872 );
873 present.insert(path);
874 }
875 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
876 Err(e) => return Err(e.into()),
877 }
878 }
879 if !present.contains(profile) {
880 return Ok(Vec::new());
881 }
882 let deps = present.contains(&profile.join("deps"));
883 let fingerprint = present.contains(&profile.join(".fingerprint"));
884 let mut dirs = vec![profile.to_owned()];
885 if deps || fingerprint {
886 ensure!(
887 !require_proof || (deps && fingerprint),
888 "incomplete Cargo v1 profile proof"
889 );
890 if deps {
891 dirs.push(profile.join("deps"));
892 }
893 if present.contains(&profile.join("build")) {
896 for entry in std::fs::read_dir(profile.join("build"))? {
897 let path = entry?.path();
898 if plain_dir(&path) {
899 for entry in std::fs::read_dir(&path)? {
900 ensure!(
901 !entry?.file_name().to_str().is_some_and(unit_hash),
902 "mixed Cargo profile layouts"
903 );
904 }
905 }
906 }
907 }
908 } else if present.contains(&profile.join("build")) {
909 let mut units = 0;
910 for package in std::fs::read_dir(profile.join("build"))? {
911 let package = package?.path();
912 let meta = std::fs::symlink_metadata(&package)?;
913 ensure!(
914 !meta.file_type().is_symlink(),
915 "Cargo v2 package is a symlink: {}",
916 package.display()
917 );
918 if !meta.is_dir() {
919 ensure!(!require_proof, "Cargo v2 package is not a directory");
920 continue;
921 }
922 ensure!(
923 package.canonicalize()? == package,
924 "Cargo v2 package is not a plain directory"
925 );
926 let mut package_units = 0;
927 for unit in std::fs::read_dir(&package)? {
928 let unit = unit?.path();
929 let meta = std::fs::symlink_metadata(&unit)?;
930 ensure!(
931 !meta.file_type().is_symlink(),
932 "Cargo v2 unit is a symlink: {}",
933 unit.display()
934 );
935 let structured = meta.is_dir()
939 && unit
940 .file_name()
941 .and_then(|n| n.to_str())
942 .is_some_and(unit_hash)
943 && unit.canonicalize()? == unit
944 && ["fingerprint", "out"].iter().all(|name| {
945 let path = unit.join(name);
946 plain_dir(&path) && path.canonicalize().is_ok_and(|p| p == path)
947 });
948 if structured && v2_run_script_unit(&unit) {
951 continue;
952 }
953 let complete = structured
954 && plain_file(
955 &unit.join("fingerprint/invoked.timestamp"),
956 "Cargo unit fingerprint",
957 )
958 .is_ok();
959 if !complete {
960 ensure!(!require_proof, "invalid or incomplete Cargo v2 unit");
961 continue;
962 }
963 dirs.push(unit.join("out"));
964 units += 1;
965 package_units += 1;
966 ensure!(units <= 4096, "Cargo unit inventory too large");
967 }
968 ensure!(
969 package_units > 0 || !require_proof,
970 "empty Cargo v2 package"
971 );
972 }
973 ensure!(units > 0 || !require_proof, "missing Cargo v2 units");
974 } else {
975 ensure!(!require_proof, "missing Cargo profile proof");
976 }
977 Ok(dirs)
978}
979
980fn v2_run_script_unit(unit: &Path) -> bool {
983 let fp = unit.join("fingerprint");
984 if std::fs::symlink_metadata(fp.join("invoked.timestamp")).is_ok() {
985 return false;
986 }
987 let Ok(entries) = std::fs::read_dir(&fp) else {
988 return false;
989 };
990 for entry in entries.flatten() {
991 let path = entry.path();
992 let name = path.file_name().unwrap_or_default().to_string_lossy();
993 if name.starts_with("run-build-script-")
994 && path.extension().is_none()
995 && plain_file(&path, "Cargo run fingerprint").is_ok()
996 && plain_file(&path.with_extension("json"), "Cargo run fingerprint json").is_ok()
997 {
998 return true;
999 }
1000 }
1001 false
1002}
1003
1004fn v2_compiler_dep_info(dir: &Path, dep_info: &Path) -> Result<bool> {
1007 let unit = dir.parent().context("Cargo unit parent")?;
1008 let hash = unit.file_name().unwrap_or_default().to_string_lossy();
1009 let stem = dep_info.file_stem().unwrap_or_default().to_string_lossy();
1010 for entry in std::fs::read_dir(unit.join("fingerprint"))? {
1011 let path = entry?.path();
1012 let name = path.file_name().unwrap_or_default().to_string_lossy();
1013 let library = name
1016 .strip_prefix("lib-")
1017 .is_some_and(|name| stem == format!("{}-{hash}", name.replace('-', "_")));
1018 let binary = name
1019 .strip_prefix("bin-")
1020 .is_some_and(|name| stem == name.replace('-', "_"));
1021 if library || binary {
1022 plain_file(&path, "Cargo compiler fingerprint")?;
1023 plain_file(
1024 &path.with_extension("json"),
1025 "Cargo compiler fingerprint json",
1026 )?;
1027 plain_file(
1028 &path.with_file_name(format!(
1029 "dep-{}",
1030 path.file_name().unwrap().to_string_lossy()
1031 )),
1032 "Cargo compiler fingerprint dep-info",
1033 )?;
1034 return Ok(true);
1035 }
1036 }
1037 Ok(false)
1038}
1039
1040fn unit_hash(name: &str) -> bool {
1041 name.len() == 16 && name.bytes().all(|b| b.is_ascii_hexdigit())
1042}
1043
1044fn declared_outputs(target: &Path) -> Result<Vec<PathBuf>> {
1048 let mut outputs = std::collections::BTreeSet::new();
1049 for profile in ["debug", "release"] {
1050 for dir in profile_output_dirs(&target.join(profile), false)? {
1051 if !plain_dir(&dir) || dir.canonicalize()? != dir {
1052 continue;
1053 }
1054 for entry in std::fs::read_dir(&dir)? {
1055 let path = entry?.path();
1056 if path.extension().is_none_or(|ext| ext != "d")
1057 || !std::fs::symlink_metadata(&path)?.is_file()
1058 || std::fs::metadata(&path)?.len() > 1024 * 1024
1059 || path.canonicalize()? != path
1060 {
1061 continue;
1062 }
1063 if dir.file_name().is_some_and(|name| name == "out")
1064 && !v2_compiler_dep_info(&dir, &path)?
1065 {
1066 continue;
1067 }
1068 let body = std::fs::read_to_string(&path)?;
1069 for line in body.lines() {
1070 let Some((names, dependencies)) = line.split_once(": ") else {
1071 continue;
1072 };
1073 if dependencies.trim().is_empty() {
1074 continue;
1075 }
1076 let mut words = Vec::new();
1079 let mut word = String::new();
1080 let mut escaped = false;
1081 for c in names.chars() {
1082 if escaped {
1083 word.push(c);
1084 escaped = false;
1085 } else if c == '\\' {
1086 escaped = true;
1087 } else if c.is_whitespace() {
1088 if !word.is_empty() {
1089 words.push(std::mem::take(&mut word));
1090 }
1091 } else {
1092 word.push(c);
1093 }
1094 }
1095 if escaped || word.contains(['$', '%']) {
1096 continue;
1097 }
1098 if !word.is_empty() {
1099 words.push(word);
1100 }
1101 for name in words {
1102 let output = PathBuf::from(name);
1103 let stem = path.file_stem().unwrap_or_default().to_string_lossy();
1104 let output_stem = output.file_stem().unwrap_or_default().to_string_lossy();
1105 let artifact_name =
1106 output_stem == stem || output_stem == format!("lib{stem}");
1107 let artifact_kind = output.extension().is_none_or(|ext| {
1108 matches!(
1109 ext.to_str(),
1110 Some(
1111 "rlib"
1112 | "rmeta"
1113 | "so"
1114 | "dylib"
1115 | "a"
1116 | "lib"
1117 | "dll"
1118 | "exe"
1119 | "pdb"
1120 )
1121 )
1122 });
1123 if artifact_name
1124 && artifact_kind
1125 && output.parent() == Some(dir.as_path())
1126 && output.extension().is_none_or(|ext| ext != "d")
1127 && std::fs::symlink_metadata(&output).is_ok_and(|m| m.is_file())
1128 && output.canonicalize().is_ok_and(|p| p == output)
1129 && artifact_signature(&output)?
1130 {
1131 outputs.insert(output);
1132 }
1133 }
1134 }
1135 }
1136 }
1137 }
1138 Ok(outputs.into_iter().collect())
1139}
1140
1141#[derive(Serialize, Deserialize)]
1142struct CargoResidue {
1143 target: PathBuf,
1144 files: std::collections::BTreeMap<PathBuf, Vec<u8>>,
1145}
1146
1147fn residue_files(
1150 target: &Path,
1151 outputs: &[PathBuf],
1152) -> Result<std::collections::BTreeMap<PathBuf, Vec<u8>>> {
1153 let mut files = std::collections::BTreeMap::new();
1154 let mut dirs = vec![target.to_owned()];
1155 let mut bytes = 0usize;
1156 let mut entries = 0usize;
1157 while let Some(dir) = dirs.pop() {
1158 for entry in std::fs::read_dir(dir)? {
1159 let path = entry?.path();
1160 entries += 1;
1161 ensure!(entries <= 4096, "residue inventory too large");
1162 let metadata = std::fs::symlink_metadata(&path)?;
1163 ensure!(
1164 !metadata.file_type().is_symlink(),
1165 "symlink in Cargo residue"
1166 );
1167 if metadata.is_dir() {
1168 dirs.push(path);
1169 } else {
1170 ensure!(metadata.is_file(), "non-file in Cargo residue");
1171 if outputs.contains(&path) {
1172 continue;
1173 }
1174 ensure!(metadata.len() <= 1024 * 1024, "residue file too large");
1175 let body = std::fs::read(&path)?;
1176 bytes = bytes.saturating_add(body.len());
1177 ensure!(bytes <= 8 * 1024 * 1024, "residue inventory too large");
1178 files.insert(path.strip_prefix(target)?.to_owned(), body);
1179 }
1180 }
1181 }
1182 Ok(files)
1183}
1184
1185fn residue_proof(home: &Path, id: &str, worktree: &Path) -> Result<PathBuf> {
1186 Ok(home
1187 .join("runs")
1188 .join(id)
1189 .join("build-residue")
1190 .join(worktree.file_name().context("missing worktree name")?))
1191}
1192
1193fn residue_root(target: &Path) -> Result<()> {
1196 for entry in std::fs::read_dir(target)? {
1197 let name = entry?.file_name();
1198 ensure!(
1199 matches!(
1200 name.to_str(),
1201 Some("debug" | "release" | "CACHEDIR.TAG" | ".rustc_info.json")
1202 ),
1203 "unrecognized Cargo residue root entry"
1204 );
1205 }
1206 Ok(())
1207}
1208
1209fn record_residue(home: &Path, id: &str, target: &Path, outputs: &[PathBuf]) -> Result<()> {
1210 cache_tag(target, false)?;
1211 residue_root(target)?;
1212 let files = residue_files(target, outputs)?;
1213 for (relative, body) in &files {
1214 let path = target.join(relative);
1215 let identity = matches!(relative.to_str(), Some("CACHEDIR.TAG" | ".rustc_info.json"));
1216 let lock = matches!(
1217 relative.to_str(),
1218 Some("debug/.cargo-lock" | "release/.cargo-lock")
1219 ) && body.is_empty();
1220 let dep_info = path.extension().is_some_and(|ext| ext == "d")
1221 && outputs.iter().any(|output| {
1222 output.parent() == path.parent()
1223 && (output.file_stem() == path.file_stem()
1224 || output.file_stem().is_some_and(|stem| {
1225 stem.to_string_lossy()
1226 == format!(
1227 "lib{}",
1228 path.file_stem().unwrap_or_default().to_string_lossy()
1229 )
1230 }))
1231 });
1232 ensure!(identity || lock || dep_info, "unproven Cargo residue");
1233 }
1234 let proof = residue_proof(home, id, target.parent().context("missing worktree")?)?;
1235 std::fs::create_dir_all(proof.parent().context("missing proof directory")?)?;
1236 let tmp = proof.with_extension("tmp");
1237 std::fs::write(
1238 &tmp,
1239 serde_json::to_vec(&CargoResidue {
1240 target: target.to_owned(),
1241 files,
1242 })?,
1243 )?;
1244 std::fs::rename(tmp, proof)?;
1245 Ok(())
1246}
1247
1248pub(crate) async fn verify_residue(home: &Path, id: &str, worktree: &Path) -> Result<Vec<File>> {
1249 let target = cargo_target(worktree).await?;
1250 residue_root(&target)?;
1251 let locks = cargo_locks(&target)?;
1252 let proof: CargoResidue =
1253 serde_json::from_slice(&std::fs::read(residue_proof(home, id, worktree)?)?)?;
1254 ensure!(
1255 proof.target == target && proof.files == residue_files(&target, &[])?,
1256 "Cargo residue changed or contains unproven files"
1257 );
1258 Ok(locks)
1259}
1260
1261pub(crate) fn cargo_locks(target: &Path) -> Result<Vec<File>> {
1263 let mut held = Vec::new();
1264 for (profile, name) in ["debug", "release"].into_iter().flat_map(|profile| {
1265 [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"].map(|name| (profile, name))
1266 }) {
1267 let lock = target.join(profile).join(name);
1268 match std::fs::symlink_metadata(&lock) {
1269 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
1270 result => {
1271 result.with_context(|| format!("Cargo lock: inspect {}", lock.display()))?;
1272 }
1273 }
1274 {
1275 plain_file(&lock, "Cargo lock")?;
1276 let file = OpenOptions::new()
1277 .read(true)
1278 .write(true)
1279 .open(&lock)
1280 .with_context(|| format!("Cargo lock: open {}", lock.display()))?;
1281 crate::pr_head::try_lock_file(&file)
1282 .with_context(|| format!("Cargo build is active: {}", lock.display()))?;
1283 held.push(file);
1284 }
1285 }
1286 Ok(held)
1287}
1288
1289pub(crate) async fn sweep(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
1292 let home = home.to_owned();
1293 let disk = disk.clone();
1294 let runtime = tokio::runtime::Handle::current();
1295 tokio::task::spawn_blocking(move || runtime.block_on(sweep_inner(&home, &disk, now)))
1296 .await
1297 .unwrap_or_else(|e| Report {
1298 failures: vec![format!("retention worker failed: {e}")],
1299 ..Report::default()
1300 })
1301}
1302
1303#[derive(Default, Serialize, Deserialize)]
1304struct Measurement {
1305 at: Option<Timestamp>,
1306 bytes: u64,
1307}
1308
1309async fn sweep_inner(home: &Path, disk: &crate::config::Disk, now: Timestamp) -> Report {
1310 let mut report = Report {
1311 checked_at: Some(now),
1312 limit_bytes: disk.worktree_limit_bytes,
1313 reclaim_budget_bytes: disk.worktree_reclaim_bytes,
1314 ..Report::default()
1315 };
1316 if !cfg!(unix) {
1317 report
1318 .failures
1319 .push("Automatic output reclamation requires Unix process containment".into());
1320 return report;
1321 }
1322 let Ok(_janitor) = OutputLease::lock(&home.join("retention.lock"), true) else {
1323 return report;
1324 };
1325 let mut targets = Vec::new();
1326 let mut under_pressure = false;
1327 let mut registrations = std::collections::HashMap::new();
1328 let measurements_path = home.join("output-measurements.json");
1329 let mut measurements: std::collections::BTreeMap<PathBuf, Measurement> =
1330 std::fs::read(&measurements_path)
1331 .ok()
1332 .and_then(|b| serde_json::from_slice(&b).ok())
1333 .unwrap_or_default();
1334 let mut observed = std::collections::BTreeSet::new();
1335 let mut ids: Vec<_> = std::fs::read_dir(home.join("runs"))
1336 .into_iter()
1337 .flatten()
1338 .flatten()
1339 .map(|e| e.file_name().to_string_lossy().into_owned())
1340 .collect();
1341 ids.sort();
1342 for id in ids {
1343 if !crate::run::is_run_id(&id) {
1344 continue;
1345 }
1346 let state = match RunState::load_under(&id, home) {
1347 Ok(state) => state,
1348 Err(e) => {
1349 report
1350 .retained
1351 .push(format!("{id}: unreadable run ownership: {e:#}"));
1352 continue;
1353 }
1354 };
1355 if disk.worktree_limit_bytes == 0 {
1356 continue;
1357 }
1358 match run_paths_missing(&state) {
1359 Ok(true) => continue,
1360 Ok(false) => {}
1361 Err(e) => {
1362 report.failures.push(format!("{id}: {e:#}"));
1363 continue;
1364 }
1365 }
1366 let list = match registrations.entry(state.repo.clone()) {
1367 std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(),
1368 std::collections::hash_map::Entry::Vacant(entry) => {
1369 match git::git(&state.repo, &["worktree", "list", "--porcelain"]).await {
1370 Ok(list) => entry.insert(list),
1371 Err(e) => {
1372 if matches!(run_paths_missing(&state), Ok(true)) {
1375 continue;
1376 }
1377 report.failures.push(format!("{id}: {e:#}"));
1378 continue;
1379 }
1380 }
1381 }
1382 };
1383 let paths = match registered_worktrees(&state, list) {
1384 Ok(paths) => paths,
1385 Err(e) => {
1386 report.failures.push(format!("{id}: {e:#}"));
1387 continue;
1388 }
1389 };
1390 for path in paths {
1391 if !path.join("target").exists() {
1392 continue;
1393 }
1394 let target = path.join("target");
1395 observed.insert(target.clone());
1396 let pressure =
1397 crate::disk::free_bytes(&path).map_or(true, |free| free < disk.min_free_bytes);
1398 under_pressure |= pressure;
1399 let measurement = measurements.entry(target).or_default();
1400 if pressure
1401 || measurement
1402 .at
1403 .is_none_or(|at| crate::clean::due(now, at, 300))
1404 {
1405 measurement.bytes = crate::disk::dir_size(&path.join("target"));
1406 measurement.at = Some(now);
1407 }
1408 let size = measurement.bytes;
1409 report.measured_bytes = report.measured_bytes.saturating_add(size);
1410 targets.push((state.output_activity(), id.clone(), path, size));
1411 }
1412 }
1413 if under_pressure {
1414 targets.sort_by_key(|t| (std::cmp::Reverse(t.3), t.0));
1417 } else {
1418 targets.sort_by_key(|t| t.0);
1419 }
1420 let free_before = crate::disk::free_bytes(home).ok();
1421 #[cfg(unix)]
1422 {
1423 let mut seen = std::collections::BTreeSet::new();
1424 report.measured_allocated_file_bytes = Some(0);
1425 report.estimated_released_allocated_bytes = Some(0);
1426 report.proven_reclaimable_allocated_bytes = Some(0);
1427 for (_, _, path, _) in &targets {
1428 report.measured_allocated_file_bytes = report
1429 .measured_allocated_file_bytes
1430 .zip(allocated_inventory(&path.join("target"), &mut seen).ok())
1431 .map(|(a, b)| a.saturating_add(b));
1432 }
1433 }
1434 let mut remaining = report.measured_bytes;
1435 let pressure_paths: Vec<_> = targets.iter().map(|t| t.2.clone()).collect();
1436 let mut deletion_started = false;
1437 let mut oversized_started = false;
1438 for (updated, id, path, _) in targets {
1439 if disk.worktree_limit_bytes == 0
1440 || remaining <= disk.worktree_limit_bytes
1441 && crate::disk::free_bytes(&path).is_ok_and(|free| free >= disk.min_free_bytes)
1442 {
1443 report.retained.push(format!("{id}: output reclamation disabled or aggregate usage and free space are within policy"));
1444 continue;
1445 }
1446 if oversized_started || report.freed_bytes >= disk.worktree_reclaim_bytes {
1447 report
1448 .retained
1449 .push(format!("{id}: per-pass byte budget spent"));
1450 continue;
1451 }
1452 let result = async {
1453 ensure!(
1454 crate::clean::due(now, updated, disk.fold_grace_secs),
1455 "execution activity within grace period: {updated} (now {now}, grace {}s)", disk.fold_grace_secs
1456 );
1457 let _owner = crate::pr_head::owner_lock(home, &id)?;
1458 let state = RunState::load_under(&id, home)?;
1459 ensure!(
1460 state.liveness_under(home, crate::daemon::is_working_on(home, &id, now))
1461 == Liveness::Dead,
1462 "run live or ownership unknown"
1463 );
1464 ensure!(crate::clean::due(now, state.output_activity(), disk.fold_grace_secs), "execution activity after inventory: {}", state.output_activity());
1465 let _driver = crate::driver::maintenance(home, &id)?;
1466 let _outputs = OutputLease::take(&path, true)?;
1467 ensure!(
1468 worktrees(&state).await?.contains(&path),
1469 "worktree registration changed"
1470 );
1471 let target = validate_cargo_target(&path, true).await
1472 .with_context(|| format!("stage validate-target: {}", path.join("target").display()))?;
1473 let _cargo = cargo_locks(&target)
1474 .with_context(|| format!("stage cargo-locks: {}", target.display()))?;
1475 validate_cargo_target(&path, true).await
1478 .with_context(|| format!("stage validate-target (locked): {}", target.display()))?;
1479 let lease_meta = _outputs._file.metadata().context("output lease activity: metadata")?;
1480 if lease_meta.len() > 0 {
1481 ensure!(lease_meta.len() == 1, "output lease activity witness is unreadable or incomplete for {}", path.display());
1482 let lease_at = Timestamp::try_from(lease_meta.modified().context("output lease activity: modified")?)?;
1483 ensure!(crate::clean::due(now, lease_at, disk.fold_grace_secs),
1484 "output lease activity within grace period: {lease_at} (now {now}, grace {}s)", disk.fold_grace_secs);
1485 }
1486 output_age(&target, now, disk.fold_grace_secs)
1487 .with_context(|| format!("stage output-age: {}", target.display()))?;
1488 let mut outputs = declared_outputs(&target)
1489 .with_context(|| format!("stage declared-outputs: {}", target.display()))?;
1490 let incremental = match crate::incremental::candidates(&path) {
1491 Ok(candidates) => candidates,
1492 Err(error) => {
1493 report.retained.push(format!("{id}: incremental cache retained: {error:#}"));
1494 crate::incremental::Reclaim::default()
1495 }
1496 };
1497 let incremental_files = incremental.files.clone();
1498 outputs.extend(incremental_files.iter().cloned());
1499 outputs.sort();
1500 outputs.dedup();
1501 ensure!(!outputs.is_empty(), "no dep-info-proven outputs or provenance-proven incremental caches; legacy incremental/build and unknown files retained");
1502 let sizes = outputs.iter().map(|p| std::fs::metadata(p).map(|m| m.len())
1503 .with_context(|| format!("stage measure: {}", p.display())))
1504 .collect::<Result<Vec<_>>>()?;
1505 let current_size = sizes.iter().fold(0u64, |bytes, size| bytes.saturating_add(*size));
1506 report.proven_reclaimable_bytes = report.proven_reclaimable_bytes.saturating_add(current_size);
1507 let mut allocation = ReleasedAllocation::of(&outputs)?;
1508 report.proven_reclaimable_allocated_bytes = report.proven_reclaimable_allocated_bytes
1509 .zip(allocation.reclaimable_allocated_bytes()).map(|(a, b)| a.saturating_add(b));
1510 let exceeds_budget = current_size > disk.worktree_reclaim_bytes.saturating_sub(report.freed_bytes);
1511 ensure!(!deletion_started || !exceeds_budget,
1515 "declared outputs exceed remaining per-pass byte budget");
1516 let ledger = home.join("runs").join(&id).join("build-outputs.json");
1519 std::fs::write(
1520 &ledger,
1521 serde_json::to_vec(
1522 &serde_json::json!({"target":target,"bytes":current_size,"at":now,"action":"reclaim-declared-outputs","identity":if target.join("CACHEDIR.TAG").exists() { "cache-tag" } else { "rustc-profile-dep-info" },"files":outputs}),
1523 )?,
1524 ).with_context(|| format!("stage ledger: {}", ledger.display()))?;
1525 if !incremental_files.is_empty() {
1528 report.retained.push(format!("{id}: incremental proof does not authorize retirement"));
1529 } else if let Err(error) = record_residue(home, &id, &target, &outputs) {
1530 report.retained.push(format!("{id}: residue not adopted for retirement: {error:#}"));
1531 }
1532 ensure!(worktrees(&state).await?.contains(&path),
1533 "worktree registration changed before deletion");
1534 deletion_started = true;
1535 oversized_started = exceeds_budget;
1536 measurements.remove(&target);
1537 for (output, size) in outputs.iter().zip(sizes) {
1538 std::fs::remove_file(output).with_context(|| format!("stage remove: {}", output.display()))?;
1539 report.estimated_released_allocated_bytes = report.estimated_released_allocated_bytes
1540 .zip(allocation.unlinked(output)).map(|(a, b)| a.saturating_add(b));
1541 report.freed_bytes = report.freed_bytes.saturating_add(size);
1543 remaining = remaining.saturating_sub(size);
1544 }
1545 incremental.finish().context("finish incremental reclamation")?;
1546 report.retained.push(format!("{id}: unmanifested files and Cargo metadata retained"));
1547 Ok::<_, anyhow::Error>(())
1548 }
1549 .await;
1550 if let Err(e) = result {
1551 report.retained.push(format!(
1552 "{id}/{}: {e:#}",
1553 path.file_name().unwrap_or_default().to_string_lossy()
1554 ));
1555 }
1556 }
1557 measurements.retain(|path, _| observed.contains(path));
1558 if let Ok(body) = serde_json::to_vec(&measurements)
1559 && let Err(e) = std::fs::write(&measurements_path, body)
1560 {
1561 report.failures.push(format!("output measurements: {e}"));
1562 }
1563 copies::evict(home, disk, now, &mut report).await;
1564 retire_empty(home, disk, now, &mut report).await;
1565 report.protected_or_unproven_bytes = report
1566 .measured_bytes
1567 .saturating_sub(report.proven_reclaimable_bytes);
1568 let pressure_remaining = pressure_paths
1569 .iter()
1570 .any(|path| crate::disk::free_bytes(path).map_or(true, |free| free < disk.min_free_bytes));
1571 if !report.failures.is_empty()
1572 || disk.worktree_limit_bytes > 0
1573 && (remaining > disk.worktree_limit_bytes || pressure_remaining)
1574 {
1575 let details = report
1576 .retained
1577 .iter()
1578 .filter(|reason| {
1579 !reason.contains("within policy")
1580 && !reason.ends_with("unmanifested files and Cargo metadata retained")
1581 })
1582 .take(4)
1583 .chain(report.failures.iter().take(2))
1584 .cloned()
1585 .collect::<Vec<_>>()
1586 .join("; ");
1587 crate::notices::raise_in(
1588 home,
1589 crate::notices::Notice::warn(
1590 "disk:retention",
1591 format!(
1592 "Retention remains blocked by the disk cap, free-space floor or inspection failures (logical inventory {}B, proven reclaimable {}B, protected or unproven {}B, unlinked {}B). {} Retained/held targets: {details}. Inspect the cleanup report for active owners, retained recovery sources and failures.",
1593 report.measured_bytes,
1594 report.proven_reclaimable_bytes,
1595 report.protected_or_unproven_bytes,
1596 report.freed_bytes,
1597 if report.proven_reclaimable_bytes == 0 {
1598 "No inspected contents have deletion authority."
1599 } else {
1600 "Authorized reclamation was insufficient; remaining protections and budgets still apply."
1601 }
1602 ),
1603 ),
1604 );
1605 }
1606 report.observed_home_free_change_bytes = free_before
1607 .zip(crate::disk::free_bytes(home).ok())
1608 .map(|(before, after)| {
1609 (i128::from(after) - i128::from(before))
1610 .clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64
1611 });
1612 let store = home.join("retention.json");
1613 if let Ok(body) = serde_json::to_vec_pretty(&report) {
1614 let tmp = home.join("retention.json.tmp");
1615 if let Err(e) = std::fs::write(&tmp, body).and_then(|()| std::fs::rename(tmp, store)) {
1616 tracing::warn!("retention report could not be saved: {e}");
1617 }
1618 }
1619 tracing::info!(
1620 measured_bytes = report.measured_bytes,
1621 freed_bytes = report.freed_bytes,
1622 retired = report.retired,
1623 retained = report.retained.len(),
1624 failures = report.failures.len(),
1625 "retained worktree cleanup"
1626 );
1627 for reason in report.retained.iter().chain(&report.failures) {
1628 tracing::debug!(%reason, "worktree retained");
1629 }
1630 report
1631}
1632
1633async fn retire_empty(
1634 home: &Path,
1635 disk: &crate::config::Disk,
1636 now: Timestamp,
1637 report: &mut Report,
1638) {
1639 if !disk.auto_fold {
1640 return;
1641 }
1642 let queue = crate::queue::Queue::at(home.join("queue"));
1643 let questions = crate::ask::Questions::at(home.join("questions"));
1644 let open = questions.list_for_effects();
1645 for listed in queue.list() {
1646 let Ok(_claim) = queue.claim(&listed.id) else {
1647 continue;
1648 };
1649 let Ok(task) = queue.get(&listed.id) else {
1650 continue;
1651 };
1652 if open
1653 .iter()
1654 .any(|q| (q.run == task.id || task.blocked_by.contains(&q.id)) && q.status.open())
1655 {
1656 report.retained.push(format!(
1657 "{}: task has an unanswered owner decision",
1658 task.id
1659 ));
1660 continue;
1661 }
1662 let sources: Vec<_> = task
1663 .runs
1664 .iter()
1665 .collect::<std::collections::BTreeSet<_>>()
1666 .into_iter()
1667 .cloned()
1668 .collect();
1669 let mut useful = Vec::new();
1670 let mut selected = Vec::new();
1671 let mut public = Vec::new();
1672 let mut unreadable = false;
1673 for id in &sources {
1674 let Ok(state) = RunState::load_under(id, home) else {
1675 unreadable = true;
1676 continue;
1677 };
1678 if state.released() || state.retired_to.is_some() {
1679 continue;
1680 }
1681 if task.pr_recovery.is_some() && state.pr_recovery == task.pr_recovery {
1682 selected.push(state.clone());
1683 }
1684 if state.pr.is_some() && state.pr_taken_over_by.is_none() {
1685 public.push(state.clone());
1686 }
1687 if state.candidates.iter().any(|c| c.commits > 0 && !c.empty) {
1688 useful.push(state);
1689 }
1690 }
1691 let pool = if !selected.is_empty() {
1695 selected
1696 } else if !public.is_empty() {
1697 public
1698 } else {
1699 useful
1700 };
1701 if unreadable || pool.len() != 1 {
1702 report.retained.push(format!(
1703 "{}: recovery authority is ambiguous or history is unreadable; {} sources retained",
1704 task.id,
1705 sources.len()
1706 ));
1707 continue;
1708 }
1709 let authority = &pool[0];
1710 if authority.candidates.is_empty() {
1711 continue;
1712 }
1713 let latest = &authority.id;
1714 for id in &sources {
1715 if id == latest || report.retired >= 5 {
1716 continue;
1717 }
1718 if let Ok(state) = RunState::load_under(id, home)
1719 && state.retired_to.as_ref() == Some(latest)
1720 && worktrees(&state).await.is_ok_and(|paths| paths.is_empty())
1721 {
1722 continue;
1723 }
1724 let result = async {
1725 ensure!(
1726 !task
1727 .resume_override
1728 .as_ref()
1729 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(id)),
1730 "operator-pinned recovery run"
1731 );
1732 ensure!(
1733 !task.replacement.as_ref().is_some_and(|pin| pin.run == *id),
1734 "selected replacement recovery source"
1735 );
1736 let _owner = crate::pr_head::owner_lock(home, id)?;
1737 let mut state = RunState::load_under(id, home)?;
1738 ensure!(
1739 state.repo == authority.repo,
1740 "repository differs from recovery authority"
1741 );
1742 ensure!(
1743 !crate::pr_recovery::explicit_sources_retained(home, id),
1744 "explicitly retained recovery source"
1745 );
1746 ensure!(
1747 state.liveness_under(home, crate::daemon::is_working_on(home, id, now))
1748 == Liveness::Dead,
1749 "run live or ownership unknown"
1750 );
1751 let _driver = crate::driver::maintenance(home, id)?;
1752 ensure!(
1753 crate::clean::due(now, state.updated_at, disk.fold_grace_secs),
1754 "within grace period"
1755 );
1756 ensure!(
1757 !open.iter().any(|q| q.run == *id && q.status.open()),
1758 "unanswered owner decision"
1759 );
1760 ensure!(
1761 state.pr.is_none()
1762 && state.pr_head.is_none()
1763 && state.pr_recovery.is_none()
1764 && state.land_approval.is_none()
1765 && state.merge_decision.is_none()
1766 && state.reviews.is_empty()
1767 && state.recovery.is_none()
1768 && state.candidate_preservation.is_empty()
1769 && state.interrupted_implementations.is_empty()
1770 && !state.released(),
1771 "PR ownership or recovery checkpoint"
1772 );
1773 let proof = prove_empty(home, &state, latest).await?;
1774 commit_retirement(home, &queue, &task, &mut state, latest, &sources, now)?;
1775 for path in &proof.paths {
1778 ensure!(
1779 git::worktree_remove_clean(&state.repo, path).await?,
1780 "Git refused clean worktree removal"
1781 );
1782 }
1783 Ok::<_, anyhow::Error>(())
1784 }
1785 .await;
1786 match result {
1787 Ok(()) => report.retired += 1,
1788 Err(e) => report.retained.push(format!("{id}: {e:#}")),
1789 }
1790 }
1791 }
1792}
1793
1794struct EmptyProof {
1795 paths: Vec<PathBuf>,
1796 _outputs: Vec<OutputLease>,
1797 _cargo: Vec<File>,
1798}
1799
1800pub(crate) async fn tracked_base(worktree: &Path, base: &str) -> Result<()> {
1803 let tree = git::git(worktree, &["ls-tree", "-r", "--full-tree", "-z", base]).await?;
1804 let mut files = Vec::new();
1805 let mut blobs = Vec::new();
1806 for record in tree.split('\0').filter(|s| !s.is_empty()) {
1807 let (header, name) = record.split_once('\t').context("unknown tree record")?;
1808 let parts: Vec<_> = header.split_whitespace().collect();
1809 ensure!(
1810 parts.len() == 3 && matches!(parts[0], "100644" | "100755") && parts[1] == "blob",
1811 "unsupported tracked source kind"
1812 );
1813 let path = worktree.join(name);
1814 let meta = std::fs::symlink_metadata(&path)?;
1815 ensure!(
1816 meta.is_file() && !meta.file_type().is_symlink() && path.canonicalize()? == path,
1817 "noncanonical tracked source"
1818 );
1819 #[cfg(unix)]
1820 {
1821 use std::os::unix::fs::PermissionsExt as _;
1822 ensure!(
1823 (meta.permissions().mode() & 0o111 != 0) == (parts[0] == "100755"),
1824 "tracked executable mode differs from original base"
1825 );
1826 }
1827 path.to_str().context("non UTF-8 source")?;
1828 files.push(git::path_arg(&path));
1829 blobs.push(parts[2].to_owned());
1830 }
1831 for (paths, expected) in files.chunks(64).zip(blobs.chunks(64)) {
1832 let mut args = vec!["hash-object", "--no-filters", "--"];
1833 args.extend(paths.iter().map(String::as_str));
1834 let actual = git::git(worktree, &args).await?;
1835 ensure!(
1836 actual.lines().eq(expected.iter().map(String::as_str)),
1837 "tracked bytes differ from original base"
1838 );
1839 }
1840 Ok(())
1841}
1842
1843async fn prove_empty(home: &Path, state: &RunState, authority: &str) -> Result<EmptyProof> {
1844 let id = &state.id;
1845 let paths = worktrees(state).await?;
1846 let mut guards = Vec::new();
1847 let mut cargo_guards = Vec::new();
1848 for path in &paths {
1849 guards.push(OutputLease::take(path, true)?);
1850 ensure!(
1851 git::rev_parse(path, "HEAD").await? == state.base_commit,
1852 "unique or unrecorded commits"
1853 );
1854 ensure!(
1855 git::git(
1856 path,
1857 &[
1858 "--no-optional-locks",
1859 "status",
1860 "--porcelain",
1861 "--untracked-files=all"
1862 ]
1863 )
1864 .await?
1865 .is_empty(),
1866 "tracked or untracked source changes"
1867 );
1868 ensure!(
1869 git::git(
1870 path,
1871 &[
1872 "--no-optional-locks",
1873 "status",
1874 "--porcelain",
1875 "--untracked-files=all",
1876 "--ignored",
1877 "--",
1878 ".",
1879 ":(exclude)target"
1880 ]
1881 )
1882 .await?
1883 .is_empty(),
1884 "source changes, ignored files or build outputs"
1885 );
1886 tracked_base(path, &state.base_commit).await?;
1887 match std::fs::symlink_metadata(path.join("target")) {
1888 Ok(_) => cargo_guards.extend(verify_residue(home, id, path).await?),
1889 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
1890 Err(error) => return Err(error.into()),
1891 }
1892 for checkpoint in [
1893 "rebase-merge",
1894 "rebase-apply",
1895 "sequencer",
1896 "BISECT_LOG",
1897 "REVERT_HEAD",
1898 "AUTO_MERGE",
1899 "MERGE_MSG",
1900 "CHERRY_PICK_HEAD",
1901 "MERGE_HEAD",
1902 "REBASE_HEAD",
1903 ] {
1904 let location = git::git(
1905 path,
1906 &[
1907 "rev-parse",
1908 "--path-format=absolute",
1909 "--git-path",
1910 checkpoint,
1911 ],
1912 )
1913 .await?;
1914 ensure!(
1915 matches!(std::fs::symlink_metadata(location.trim()), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
1916 "retained Git recovery metadata: {checkpoint}"
1917 );
1918 }
1919 let origin_path = git::git(
1922 path,
1923 &[
1924 "rev-parse",
1925 "--path-format=absolute",
1926 "--git-path",
1927 "ORIG_HEAD",
1928 ],
1929 )
1930 .await?;
1931 match std::fs::symlink_metadata(origin_path.trim()) {
1932 Ok(meta) => {
1933 ensure!(
1934 meta.is_file() && !meta.file_type().is_symlink(),
1935 "unknown ORIG_HEAD artifact"
1936 );
1937 git::rev_parse(path, "ORIG_HEAD")
1938 .await
1939 .context("unreadable ORIG_HEAD")?;
1940 }
1941 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1942 Err(e) => return Err(e.into()),
1943 }
1944 if let Ok(original) = git::rev_parse(path, "ORIG_HEAD").await {
1945 ensure!(
1946 git::git_raw(
1947 path,
1948 &["merge-base", "--is-ancestor", &original, &state.base_commit]
1949 )
1950 .await?
1951 .ok(),
1952 "ORIG_HEAD retains commits outside the original base"
1953 );
1954 }
1955 let heads = git::git(path, &["reflog", "show", "--format=%H", "HEAD"]).await?;
1956 for head in heads.lines().collect::<std::collections::BTreeSet<_>>() {
1957 ensure!(
1958 git::git_raw(
1959 path,
1960 &["merge-base", "--is-ancestor", head, &state.base_commit]
1961 )
1962 .await?
1963 .ok(),
1964 "worktree reflog retains commits outside the original base"
1965 );
1966 }
1967 for checkpoint in ["CHERRY_PICK_HEAD", "MERGE_HEAD", "REBASE_HEAD"] {
1968 ensure!(
1969 !git::git_raw(path, &["rev-parse", "--verify", checkpoint])
1970 .await?
1971 .ok(),
1972 "unfinished Git operation"
1973 );
1974 }
1975 }
1976 if paths.is_empty() && state.retired_to.as_deref() == Some(authority) {
1979 return Ok(EmptyProof {
1980 paths,
1981 _outputs: guards,
1982 _cargo: cargo_guards,
1983 });
1984 }
1985 ensure!(!paths.is_empty(), "no worktrees to prove empty");
1986 for candidate in &state.candidates {
1987 ensure!(
1988 (paths.contains(&candidate.worktree)
1989 || state.retired_to.as_deref() == Some(authority) && !candidate.worktree.exists())
1990 && git::rev_parse(&state.repo, &candidate.branch).await? == state.base_commit,
1991 "candidate recovery source"
1992 );
1993 }
1994 let pattern = format!("refs/heads/magi/{}/", state.short());
1995 let branches = git::git(
1996 &state.repo,
1997 &["for-each-ref", "--format=%(objectname)", &pattern],
1998 )
1999 .await?;
2000 ensure!(
2001 branches.lines().all(|head| head == state.base_commit),
2002 "unrecorded candidate branch changes"
2003 );
2004 Ok(EmptyProof {
2005 paths,
2006 _outputs: guards,
2007 _cargo: cargo_guards,
2008 })
2009}
2010
2011fn commit_retirement(
2016 home: &Path,
2017 queue: &crate::queue::Queue,
2018 expected: &crate::queue::Task,
2019 state: &mut RunState,
2020 authority: &str,
2021 sources: &[String],
2022 now: Timestamp,
2023) -> Result<()> {
2024 let _write = queue.lock_task(&expected.id)?;
2025 let current = queue.get(&expected.id)?;
2026 ensure!(
2027 serde_json::to_value(¤t)? == serde_json::to_value(expected)?,
2028 "task changed during retirement proof; retaining worktrees for the current owner decision"
2029 );
2030 ensure!(
2031 !current
2032 .resume_override
2033 .as_ref()
2034 .is_some_and(|pin| pin.pinned_run.as_ref() == Some(&state.id))
2035 && !current
2036 .replacement
2037 .as_ref()
2038 .is_some_and(|pin| pin.run == state.id),
2039 "current owner selected this recovery source"
2040 );
2041 ensure!(
2042 !crate::pr_recovery::explicit_sources_retained(home, &state.id),
2043 "explicitly retained recovery source"
2044 );
2045 ensure!(
2046 !crate::ask::Questions::at(home.join("questions"))
2047 .list_for_effects()
2048 .iter()
2049 .any(|q| q.status.open()
2050 && (q.run == state.id
2051 || q.run == current.id
2052 || current.blocked_by.contains(&q.id))),
2053 "owner decision arrived during retirement proof"
2054 );
2055 let dir = home.join("recovery-lineages");
2056 std::fs::create_dir_all(&dir)?;
2057 let tmp = dir.join(format!("{}.tmp", current.id));
2058 let lineage = serde_json::json!({"task":current.id,"authority":authority,"sources":sources,
2059 "selection":current.pr_recovery,"task_updated_at":current.updated_at,"at":now});
2060 std::fs::write(&tmp, serde_json::to_vec_pretty(&lineage)?)?;
2061 std::fs::rename(tmp, dir.join(format!("{}.json", current.id)))?;
2062 state.retired_to = Some(authority.to_owned());
2063 state.event(
2064 "retention",
2065 format!(
2066 "empty attempt retired to {authority}; recovery sources: {}",
2067 sources.join(", ")
2068 ),
2069 );
2070 state.save_under(home)?;
2071 Ok(())
2073}
2074
2075pub(crate) async fn emergency(paths: Vec<PathBuf>, min_free: u64, isolated: bool) -> String {
2079 let measure = if isolated {
2080 crate::disk::managed_free_bytes
2081 } else {
2082 crate::disk::free_bytes
2083 };
2084 if min_free == 0 {
2085 return std::future::pending().await;
2086 }
2087 loop {
2088 for path in &paths {
2089 let existing = path.ancestors().find(|p| p.exists()).unwrap_or(path);
2090 match measure(existing) {
2091 Ok(free) if free >= min_free / 2 => {}
2092 Ok(free) => {
2093 return format!(
2094 "disk pressure: {free} bytes free; stopping owned writers to preserve the emergency reserve"
2095 );
2096 }
2097 Err(e) => return format!("disk pressure: free space measurement failed: {e:#}"),
2098 }
2099 }
2100 tokio::time::sleep(std::time::Duration::from_secs(5)).await;
2101 }
2102}
2103
2104#[cfg(all(test, unix))]
2105mod tests {
2106 use super::*;
2107 use crate::{
2108 config::{Config, Disk},
2109 queue::{Source, Task},
2110 run::{Candidate, RunStatus},
2111 };
2112
2113 struct Fx {
2114 _temp: tempfile::TempDir,
2115 home: PathBuf,
2116 repo: PathBuf,
2117 bay: PathBuf,
2118 base: String,
2119 }
2120 impl Fx {
2121 async fn new() -> Self {
2122 let temp = tempfile::tempdir().unwrap();
2123 let root = temp.path().canonicalize().unwrap();
2124 let home = root.join("home");
2125 let repo = root.join("repo");
2126 let bay = root.join("worktrees");
2127 std::fs::create_dir_all(&repo).unwrap();
2128 std::fs::create_dir_all(home.join("runs")).unwrap();
2129 git::git(&repo, &["init", "-b", "main"]).await.unwrap();
2130 git::git(&repo, &["config", "user.name", "Fixture"])
2131 .await
2132 .unwrap();
2133 git::git(&repo, &["config", "user.email", "fixture@example.invalid"])
2134 .await
2135 .unwrap();
2136 git::git(&repo, &["config", "commit.gpgsign", "false"])
2137 .await
2138 .unwrap();
2139 std::fs::write(repo.join(".gitignore"), "target/\nscratch/\n").unwrap();
2140 std::fs::write(repo.join("source"), "base\n").unwrap();
2141 git::git(&repo, &["add", "."]).await.unwrap();
2142 git::git(&repo, &["commit", "-m", "fixture"]).await.unwrap();
2143 let base = git::rev_parse(&repo, "HEAD").await.unwrap();
2144 Self {
2145 _temp: temp,
2146 home,
2147 repo,
2148 bay,
2149 base,
2150 }
2151 }
2152 async fn run(&self, tag: &str, useful: bool) -> RunState {
2153 let mut cfg = Config::default();
2154 cfg.graph.worktree_root = Some(self.bay.clone());
2155 let mut state = RunState::new(
2156 self.repo.clone(),
2157 "main".into(),
2158 self.base.clone(),
2159 "task".into(),
2160 cfg,
2161 );
2162 state.id = format!("20260101-000000-{tag}");
2163 state.status = RunStatus::Blocked;
2164 state.driver_exited = true;
2165 let wt = state.worktree_root().join("cand-A");
2166 let branch = format!("magi/{tag}/A");
2167 git::worktree_add_branch(&self.repo, &wt, &branch, &self.base)
2168 .await
2169 .unwrap();
2170 if useful {
2171 std::fs::write(wt.join("source"), tag).unwrap();
2172 git::git(&wt, &["add", "."]).await.unwrap();
2173 git::git(&wt, &["commit", "-m", "candidate change"])
2174 .await
2175 .unwrap();
2176 }
2177 state.candidates.push(Candidate {
2178 index: 0,
2179 label: 'A',
2180 agent: "fixture".into(),
2181 branch,
2182 worktree: wt,
2183 summary: String::new(),
2184 stat: String::new(),
2185 files: usize::from(useful),
2186 commits: usize::from(useful),
2187 empty: !useful,
2188 failed: None,
2189 verified_noop: None,
2190 duration_ms: 0,
2191 folded: false,
2192 });
2193 state.save_under(&self.home).unwrap();
2194 state
2195 }
2196 fn disk() -> Disk {
2197 Disk {
2198 worktree_limit_bytes: 1,
2199 worktree_reclaim_bytes: 1024 * 1024,
2200 fold_grace_secs: 0,
2201 min_free_bytes: 0,
2202 ..Disk::default()
2203 }
2204 }
2205 fn target(wt: &Path) {
2206 let target = wt.join("target");
2207 std::fs::create_dir_all(target.join("debug")).unwrap();
2208 let mut artifact = [1; 200];
2209 artifact[..4].copy_from_slice(b"\x7fELF");
2210 std::fs::write(target.join("debug/output"), artifact).unwrap();
2211 let output = target
2212 .join("debug/output")
2213 .display()
2214 .to_string()
2215 .replace('\\', "\\\\")
2216 .replace(' ', "\\ ");
2217 std::fs::write(target.join("debug/output.d"), format!("{output}: source\n")).unwrap();
2218 std::fs::write(target.join("debug/.cargo-lock"), "").unwrap();
2219 std::fs::write(
2220 target.join("CACHEDIR.TAG"),
2221 "Signature: 8a477f597d28d172789f06886806bc55\n",
2222 )
2223 .unwrap();
2224 std::fs::write(
2225 target.join(".rustc_info.json"),
2226 "{\"rustc_fingerprint\":123}",
2227 )
2228 .unwrap();
2229 }
2230 fn now() -> Timestamp {
2231 Timestamp::now() + jiff::SignedDuration::from_secs(2)
2232 }
2233 fn task(&self, runs: &[RunState]) -> Task {
2234 let mut task = Task::new(
2235 "fixture".into(),
2236 "task".into(),
2237 self.repo.clone(),
2238 Source::Human,
2239 );
2240 task.runs = runs.iter().map(|s| s.id.clone()).collect();
2241 crate::queue::Queue::at(self.home.join("queue"))
2242 .put(&mut task)
2243 .unwrap();
2244 task
2245 }
2246 }
2247
2248 #[cfg(unix)]
2249 #[tokio::test]
2250 async fn incremental_only_reclamation_preserves_legacy_and_unknown_data() {
2251 let f = Fx::new().await;
2252 let state = f.run("ca01", true).await;
2253 let wt = &state.candidates[0].worktree;
2254 let generation = crate::incremental::tests::generate(wt);
2255 assert!(!declared_outputs(&wt.join("target")).unwrap().is_empty());
2257 for dir in profile_output_dirs(&wt.join("target/debug"), true).unwrap() {
2258 for entry in std::fs::read_dir(dir).unwrap() {
2259 let path = entry.unwrap().path();
2260 if path.extension().is_some_and(|x| x == "d") {
2261 std::fs::remove_file(path).unwrap();
2262 }
2263 }
2264 }
2265 assert!(declared_outputs(&wt.join("target")).unwrap().is_empty());
2266 assert_eq!(
2267 sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes,
2268 0,
2269 "live generation is protected"
2270 );
2271 drop(generation);
2272 let candidates = crate::incremental::candidates(wt).unwrap();
2273 let logical: u64 = candidates
2274 .files
2275 .iter()
2276 .map(|p| std::fs::metadata(p).unwrap().len())
2277 .sum();
2278 let unknown = candidates.files[0].parent().unwrap().join("operator-note");
2279 std::fs::write(&unknown, "retain unknown data").unwrap();
2280 drop(candidates);
2281 let refused = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2282 assert_eq!(refused.freed_bytes, 0, "{refused:?}");
2283 assert!(
2284 refused
2285 .retained
2286 .iter()
2287 .any(|s| s.contains("provenance mismatch"))
2288 );
2289 std::fs::remove_file(&unknown).unwrap();
2290 let legacy = wt.join("target/debug/incremental/legacy-session");
2291 std::fs::create_dir(&legacy).unwrap();
2292 let candidates = crate::incremental::candidates(wt).unwrap();
2294 let bin = candidates
2295 .files
2296 .iter()
2297 .find(|p| p.extension().is_some_and(|x| x == "bin"))
2298 .unwrap();
2299 std::fs::copy(bin, legacy.join("query-cache.bin")).unwrap();
2300 drop(candidates);
2301 let cargo = cargo_locks(&wt.join("target")).unwrap();
2302 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2303 drop(cargo);
2304 let outputs = OutputLease::borrow(wt).unwrap().unwrap();
2305 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
2306 drop(outputs);
2307 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2308 assert_eq!(report.freed_bytes, logical, "{report:?}");
2309 assert_eq!(report.proven_reclaimable_bytes, logical);
2310 assert!(report.measured_bytes > logical);
2311 assert_eq!(
2312 report.protected_or_unproven_bytes,
2313 report.measured_bytes - logical
2314 );
2315 #[cfg(unix)]
2316 {
2317 assert!(report.estimated_released_allocated_bytes.unwrap() > 0);
2318 assert!(
2319 report.measured_allocated_file_bytes.unwrap()
2320 >= report.estimated_released_allocated_bytes.unwrap()
2321 );
2322 }
2323 assert!(report.observed_home_free_change_bytes.is_some());
2324 assert!(legacy.join("query-cache.bin").is_file());
2325 assert!(wt.join("src/lib.rs").is_file());
2326 assert_eq!(report.retired, 0);
2327 }
2328
2329 #[cfg(unix)]
2330 #[tokio::test]
2331 async fn incremental_and_deps_links_account_for_retained_third_link() {
2332 use std::os::unix::fs::MetadataExt;
2333 let f = Fx::new().await;
2334 let state = f.run("ca02", true).await;
2335 let wt = &state.candidates[0].worktree;
2336 let generation = crate::incremental::tests::generate(wt);
2337 drop(generation);
2338 let candidates = crate::incremental::candidates(wt).unwrap();
2339 let objects = candidates
2340 .files
2341 .iter()
2342 .filter(|p| p.extension().is_some_and(|x| x == "o"))
2343 .take(2)
2344 .cloned()
2345 .collect::<Vec<_>>();
2346 assert_eq!(
2347 objects.len(),
2348 2,
2349 "fixture must produce real incremental objects"
2350 );
2351 let mut outputs = candidates.files.clone();
2352 drop(candidates);
2353 let target = wt.join("target");
2354 assert!(!declared_outputs(&target).unwrap().is_empty());
2355 let generated_dirs = profile_output_dirs(&target.join("debug"), true).unwrap();
2356 let mut output_dir = None;
2359 for dir in &generated_dirs {
2360 for entry in std::fs::read_dir(dir).unwrap() {
2361 let path = entry.unwrap().path();
2362 if path.extension().is_some_and(|x| x == "d") {
2363 std::fs::remove_file(path).unwrap();
2364 if dir != &generated_dirs[0] {
2365 output_dir = Some(dir.clone());
2366 }
2367 }
2368 }
2369 }
2370 let output_dir = output_dir.expect("Cargo generated no recognised output directory");
2371 for entry in std::fs::read_dir(&output_dir).unwrap() {
2375 let path = entry.unwrap().path();
2376 if path.is_file() {
2377 std::fs::remove_file(path).unwrap();
2378 }
2379 }
2380 let stem = if output_dir.file_name().is_some_and(|n| n == "out") {
2383 let unit = output_dir.parent().unwrap();
2384 let hash = unit.file_name().unwrap().to_string_lossy().into_owned();
2385 let name = std::fs::read_dir(unit.join("fingerprint"))
2386 .unwrap()
2387 .filter_map(|e| e.unwrap().file_name().into_string().ok())
2388 .find_map(|n| {
2389 n.strip_prefix("lib-")
2390 .filter(|n| !n.contains('.'))
2391 .map(str::to_owned)
2392 })
2393 .expect("Cargo unit has a lib fingerprint");
2394 format!("{}-{hash}", name.replace('-', "_"))
2395 } else {
2396 "link".to_owned()
2397 };
2398 let mut declared = Vec::new();
2401 for (index, object) in objects.iter().enumerate() {
2402 assert_eq!(std::fs::metadata(object).unwrap().nlink(), 1);
2403 let dep = output_dir.join(if index == 0 {
2404 stem.clone()
2405 } else {
2406 format!("lib{stem}")
2407 });
2408 std::fs::hard_link(object, &dep).unwrap();
2409 declared.push(dep.display().to_string());
2410 outputs.push(dep);
2411 }
2412 std::fs::write(
2413 output_dir.join(format!("{stem}.d")),
2414 format!("{}: src/lib.rs\n", declared.join(" ")),
2415 )
2416 .unwrap();
2417 let retained = wt.join("retained-input");
2418 std::fs::hard_link(&objects[1], &retained).unwrap();
2419 let sentinel = std::fs::read(&retained).unwrap();
2420 let logical: u64 = outputs
2421 .iter()
2422 .map(|p| std::fs::metadata(p).unwrap().len())
2423 .sum();
2424 let mut seen = std::collections::BTreeSet::new();
2425 let retained_meta = std::fs::metadata(&retained).unwrap();
2426 let retained_inode = (retained_meta.dev(), retained_meta.ino());
2427 let mut expected = 0u64;
2428 for output in &outputs {
2429 let m = std::fs::metadata(output).unwrap();
2430 let inode = (m.dev(), m.ino());
2431 if inode != retained_inode && seen.insert(inode) {
2432 expected += m.blocks() * 512;
2433 }
2434 }
2435 assert!(expected > 0);
2436 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2437 assert_eq!(report.proven_reclaimable_bytes, logical, "{report:?}");
2438 assert_eq!(report.freed_bytes, logical, "{report:?}");
2439 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(expected));
2440 assert_eq!(report.estimated_released_allocated_bytes, Some(expected));
2441 assert!(outputs.iter().all(|p| !p.exists()));
2442 assert_eq!(std::fs::read(retained).unwrap(), sentinel);
2443 assert!(wt.join("src/lib.rs").is_file());
2444 }
2445
2446 #[tokio::test]
2447 async fn retention_notice_explains_unproven_inventory_and_live_owner() {
2448 let f = Fx::new().await;
2449 let state = f.run("ca03", true).await;
2450 let wt = &state.candidates[0].worktree;
2451 Fx::target(wt);
2452 std::fs::remove_file(wt.join("target/debug/output")).unwrap();
2453 let unknown = wt.join("target/debug/operator-note");
2454 std::fs::write(&unknown, vec![1; 4096]).unwrap();
2455 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2456 assert_eq!(report.proven_reclaimable_bytes, 0);
2457 #[cfg(unix)]
2458 assert_eq!(report.proven_reclaimable_allocated_bytes, Some(0));
2459 assert!(report.protected_or_unproven_bytes >= 4096);
2460 let message = || {
2461 crate::notices::Notices::at(f.home.join("notifications"))
2462 .list()
2463 .into_iter()
2464 .find(|n| n.key == "disk:retention")
2465 .unwrap()
2466 .message
2467 };
2468 let notice = message();
2469 assert!(notice.contains("proven reclaimable 0B"), "{notice}");
2470 assert!(notice.contains("protected or unproven"), "{notice}");
2471 assert!(
2472 notice.contains("legacy incremental/build and unknown files retained"),
2473 "{notice}"
2474 );
2475 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
2476 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2477 assert_eq!(report.freed_bytes, 0);
2478 let notice = message();
2479 assert!(notice.contains(&state.id), "{notice}");
2480 assert!(
2481 notice.contains("lock") || notice.contains("owner"),
2482 "{notice}"
2483 );
2484 drop(owner);
2485 assert_eq!(std::fs::read(unknown).unwrap(), vec![1; 4096]);
2486 }
2487
2488 #[tokio::test]
2489 async fn real_cargo_build_holds_the_target_against_reclamation() {
2490 let f = Fx::new().await;
2491 let state = f.run("ca02", true).await;
2492 let wt = &state.candidates[0].worktree;
2493 Fx::target(wt);
2494 std::fs::create_dir(wt.join("src")).unwrap();
2495 std::fs::write(
2496 wt.join("Cargo.toml"),
2497 "[package]\nname='live-cache-fixture'\nversion='0.1.0'\nedition='2021'\n[workspace]\n",
2498 )
2499 .unwrap();
2500 std::fs::write(wt.join("src/lib.rs"), "pub fn answer() -> usize { 42 }\n").unwrap();
2501 std::fs::write(
2502 wt.join("build.rs"),
2503 r#"fn main() {
2504 println!("cargo:rerun-if-env-changed=FIXTURE_READY");
2505 let Ok(ready) = std::env::var("FIXTURE_READY") else { return };
2506 let release = std::env::var("FIXTURE_RELEASE").unwrap();
2507 std::fs::write(ready, "ready").unwrap();
2508 let start = std::time::Instant::now();
2509 while !std::path::Path::new(&release).exists() {
2510 assert!(start.elapsed().as_secs() < 120, "fixture was not released");
2511 std::thread::sleep(std::time::Duration::from_millis(10));
2512 }
2513 }"#,
2514 )
2515 .unwrap();
2516 let ready = wt.join("ready");
2517 let release = wt.join("release");
2518 let mut cargo = tokio::process::Command::new("cargo");
2519 cargo.env_clear();
2520 cargo.env(
2521 "PATH",
2522 crate::build_target::command::standalone_fixture_path().unwrap(),
2523 );
2524 for key in ["HOME", "RUSTUP_HOME"] {
2525 if let Some(value) = std::env::var_os(key) {
2526 cargo.env(key, value);
2527 }
2528 }
2529 cargo
2530 .current_dir(wt)
2531 .env("RUSTUP_TOOLCHAIN", "stable")
2532 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2533 .env("CARGO_INCREMENTAL", "1")
2534 .args(["build", "--offline", "--jobs", "1", "--target-dir"])
2535 .arg(wt.join("target"));
2536 let warm = cargo.output().await.unwrap();
2540 assert!(
2541 warm.status.success(),
2542 "fixture compilation failed: {}",
2543 String::from_utf8_lossy(&warm.stderr)
2544 );
2545 let log_path = f._temp.path().join("live-cargo.stderr");
2546 let mut child = cargo
2547 .env("FIXTURE_READY", &ready)
2548 .env("FIXTURE_RELEASE", &release)
2549 .stdout(std::process::Stdio::null())
2550 .stderr(std::fs::File::create(&log_path).unwrap())
2551 .kill_on_drop(true)
2552 .spawn()
2553 .unwrap();
2554 let readiness = tokio::time::timeout(std::time::Duration::from_secs(60), async {
2555 while !ready.exists() {
2556 if let Some(status) = child.try_wait().unwrap() {
2557 return Err(format!("fixture exited before readiness: {status}"));
2558 }
2559 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
2560 }
2561 Ok(())
2562 })
2563 .await;
2564 let arrived = matches!(readiness, Ok(Ok(())));
2565 let report = if arrived {
2566 Some(sweep(&f.home, &Fx::disk(), Fx::now()).await)
2567 } else {
2568 None
2569 };
2570 std::fs::write(release, "release").unwrap();
2572 let status = tokio::time::timeout(std::time::Duration::from_secs(60), child.wait()).await;
2573 if status.is_err() {
2574 child.kill().await.unwrap();
2575 child.wait().await.unwrap();
2576 }
2577 assert!(
2578 arrived,
2579 "fixture build never reached its active build script: {readiness:?}\n{}",
2580 std::fs::read_to_string(&log_path).unwrap()
2581 );
2582 assert!(
2583 status.unwrap().unwrap().success(),
2584 "fixture build failed: {}",
2585 std::fs::read_to_string(&log_path).unwrap()
2586 );
2587 let report = report.unwrap();
2588 assert_eq!(report.freed_bytes, 0, "{report:?}");
2589 assert!(
2590 report
2591 .retained
2592 .iter()
2593 .any(|s| s.contains("Cargo build is active")),
2594 "{report:?}"
2595 );
2596 assert!(wt.join("target/debug/output").is_file());
2597 assert!(crate::disk::dir_size(&wt.join("target/debug/incremental")) > 0);
2598 }
2599
2600 #[cfg(unix)]
2601 #[test]
2602 fn allocated_accounting_deduplicates_links_and_requires_the_last_link() {
2603 let temp = tempfile::tempdir().unwrap();
2604 let a = temp.path().join("a");
2605 let b = temp.path().join("b");
2606 let outside = temp.path().join("retained-input");
2607 std::fs::write(&a, vec![1; 65536]).unwrap();
2608 std::fs::hard_link(&a, &b).unwrap();
2609 std::fs::hard_link(&a, &outside).unwrap();
2610 let mut seen = Default::default();
2611 let blocks = allocated_inventory(&temp.path().canonicalize().unwrap(), &mut seen).unwrap();
2612 assert!(blocks > 0);
2613 let mut accounting = ReleasedAllocation::of(&[a.clone(), b.clone()]).unwrap();
2614 std::fs::remove_file(&a).unwrap();
2615 assert_eq!(accounting.unlinked(&a), Some(0));
2616 std::fs::remove_file(&b).unwrap();
2617 assert_eq!(accounting.unlinked(&b), Some(0));
2618 assert_eq!(std::fs::read(&outside).unwrap().len(), 65536);
2619 let mut accounting = ReleasedAllocation::of(std::slice::from_ref(&outside)).unwrap();
2620 std::fs::remove_file(&outside).unwrap();
2621 assert_eq!(accounting.unlinked(&outside), Some(blocks));
2622 }
2623
2624 #[cfg(unix)]
2625 #[test]
2626 fn relative_admission_home_matches_absolute_home_and_retains_symlink_checks() {
2627 let temp = tempfile::tempdir_in(".").unwrap();
2628 let home = PathBuf::from(temp.path().file_name().unwrap()).join("state");
2629 assert!(home.is_relative());
2630 let paths = vec![std::env::current_dir().unwrap()];
2631 let gib = 1024 * 1024 * 1024;
2632 let reservation = OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).unwrap();
2633 assert!(
2634 OutputLease::admit_with(&home.canonicalize().unwrap(), &paths, 8 * gib, |_| Ok(
2635 9 * gib
2636 ))
2637 .is_err()
2638 );
2639 drop(reservation);
2640 assert!(OutputLease::admit_with(&home, &paths, 8 * gib, |_| Ok(9 * gib)).is_ok());
2641 let bad = temp.path().join("symlink-state");
2642 std::fs::create_dir(&bad).unwrap();
2643 std::os::unix::fs::symlink(
2644 home.canonicalize().unwrap().join("disk-admission"),
2645 bad.join("disk-admission"),
2646 )
2647 .unwrap();
2648 assert!(OutputLease::admit_with(&bad, &paths, 8 * gib, |_| Ok(9 * gib)).is_err());
2649 }
2650
2651 #[cfg(unix)]
2652 #[test]
2653 fn concurrent_disk_admission_reserves_once_per_volume_until_release() {
2654 let temp = tempfile::tempdir().unwrap();
2655 let home = temp.path().canonicalize().unwrap();
2656 let paths = vec![home.clone(), home.join("not-created-yet")];
2657 let gib = 1024 * 1024 * 1024;
2658 let floor = 8 * gib;
2659 let first = OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap();
2660 assert_eq!(first.len(), 1);
2661 let blocked =
2662 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).unwrap_err();
2663 assert!(blocked.to_string().contains("active writer reservations"));
2664 let second =
2665 OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + 2 * gib)).unwrap();
2666 drop(first);
2667 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_err());
2668 drop(second);
2669 assert!(OutputLease::admit_with(&home, &paths, floor, |_| Ok(floor + gib)).is_ok());
2670 assert!(
2671 OutputLease::admit_with(&home, &paths, floor, |_| anyhow::bail!("unmeasured")).is_err()
2672 );
2673 }
2674
2675 #[cfg(unix)]
2676 #[test]
2677 fn simultaneous_writers_cannot_spend_the_same_volume_headroom() {
2678 let temp = tempfile::tempdir().unwrap();
2679 let home = temp.path().canonicalize().unwrap();
2680 let barrier = std::sync::Barrier::new(2);
2681 let gib = 1024 * 1024 * 1024;
2682 let results = std::thread::scope(|scope| {
2683 let threads: Vec<_> = (0..2)
2684 .map(|_| {
2685 scope.spawn(|| {
2686 barrier.wait();
2687 OutputLease::admit_with(&home, std::slice::from_ref(&home), 100, |_| {
2688 Ok(100 + gib)
2689 })
2690 })
2691 })
2692 .collect();
2693 threads
2694 .into_iter()
2695 .map(|thread| thread.join().unwrap())
2696 .collect::<Vec<_>>()
2697 });
2698 assert_eq!(results.iter().filter(|r| r.is_ok()).count(), 1);
2699 assert!(
2700 results
2701 .iter()
2702 .filter_map(|r| r.as_ref().err())
2703 .all(|e| e.to_string().contains("active writer reservations"))
2704 );
2705 }
2706
2707 #[tokio::test]
2708 async fn sweep_reclaims_real_cargo_outputs_and_preserves_source_and_unknown_files() {
2709 real_cargo_fixture(false).await;
2710 }
2711
2712 #[tokio::test]
2713 async fn missing_tag_from_precreated_target_reclaims_only_real_cargo_artifacts() {
2714 real_cargo_fixture(true).await;
2715 }
2716
2717 async fn real_cargo_fixture(precreate: bool) {
2718 let f = Fx::new().await;
2719 let state = f.run("ca00", true).await;
2720 let wt = &state.candidates[0].worktree;
2721 let source = "fn main() { println!(\"retained fixture\"); }\n";
2722 std::fs::create_dir(wt.join("src")).unwrap();
2723 std::fs::write(wt.join("src/main.rs"), source).unwrap();
2724 std::fs::write(
2725 wt.join("Cargo.toml"),
2726 "[package]\nname = \"retained-fixture\"\nversion = \"0.1.0\"\nedition = \"2021\"\n[workspace]\n",
2727 )
2728 .unwrap();
2729 if precreate {
2730 std::fs::create_dir_all(wt.join("target/magi-tools")).unwrap();
2731 }
2732 let mut cargo = std::process::Command::new("cargo");
2733 cargo.env_clear();
2734 cargo.env(
2735 "PATH",
2736 crate::build_target::command::standalone_fixture_path().unwrap(),
2737 );
2738 for key in ["HOME", "RUSTUP_HOME"] {
2739 if let Some(value) = std::env::var_os(key) {
2740 cargo.env(key, value);
2741 }
2742 }
2743 let build = cargo
2744 .current_dir(wt)
2745 .env("RUSTUP_TOOLCHAIN", "stable")
2746 .env("CARGO_HOME", f._temp.path().join("cargo-home"))
2747 .env("CARGO_INCREMENTAL", "0")
2748 .arg("--config")
2749 .arg(format!(
2750 "build.build-dir={}",
2751 serde_json::to_string(&wt.join("target")).unwrap()
2752 ))
2753 .args([
2754 "build",
2755 "--offline",
2756 "--release",
2757 "--jobs",
2758 "1",
2759 "--target-dir",
2760 ])
2761 .arg(wt.join("target"))
2762 .output()
2763 .unwrap();
2764 assert!(
2765 build.status.success(),
2766 "{}",
2767 String::from_utf8_lossy(&build.stderr)
2768 );
2769 assert_eq!(wt.join("target/CACHEDIR.TAG").exists(), !precreate);
2770 if precreate {
2771 let again = cargo.output().unwrap();
2773 assert!(
2774 again.status.success(),
2775 "{}",
2776 String::from_utf8_lossy(&again.stderr)
2777 );
2778 assert!(!wt.join("target/CACHEDIR.TAG").exists());
2779 assert!(cargo_target(wt).await.is_err());
2780 }
2781 git::git(wt, &["add", "Cargo.toml", "Cargo.lock", "src"])
2782 .await
2783 .unwrap();
2784 git::git(wt, &["commit", "-m", "real Cargo fixture"])
2785 .await
2786 .unwrap();
2787 let artifact = wt.join("target/release/retained-fixture");
2788 assert!(artifact.is_file());
2789 let unknown = wt.join("target/release/operator-note");
2790 std::fs::write(&unknown, "keep this note\n").unwrap();
2791
2792 let target = wt.join("target");
2793 let mut preserved = std::collections::BTreeMap::new();
2794 for relative in [
2795 "tmp/sentinel",
2796 "magi-tools/sentinel",
2797 "lifecycle-client-check.log",
2798 ] {
2799 let path = target.join(relative);
2800 std::fs::create_dir_all(path.parent().unwrap()).unwrap();
2801 std::fs::write(&path, format!("retain {relative}\n")).unwrap();
2802 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2803 }
2804 let decoy = target.join("magi-tools/retained-fixture");
2807 std::fs::copy(&artifact, &decoy).unwrap();
2808 let dep_info = decoy.with_extension("d");
2809 std::fs::write(&dep_info, format!("{}: src/main.rs\n", decoy.display())).unwrap();
2810 for path in [
2811 decoy,
2812 dep_info,
2813 wt.join("Cargo.toml"),
2814 wt.join("Cargo.lock"),
2815 ] {
2816 preserved.insert(path.clone(), std::fs::read(path).unwrap());
2817 }
2818 let supported_dirs = profile_output_dirs(&target.join("release"), true).unwrap();
2819 let proven = declared_outputs(&target).unwrap();
2820 assert!(
2821 proven
2822 .iter()
2823 .any(|path| path.parent() != Some(target.join("release").as_path())),
2824 "both uplifted and unit/deps binary references must be proven: {proven:?}"
2825 );
2826
2827 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2828 assert!(out.freed_bytes > 0, "{out:?}");
2829 for (path, bytes) in preserved {
2830 assert_eq!(std::fs::read(&path).unwrap(), bytes, "{}", path.display());
2831 }
2832 assert!(!artifact.exists(), "{out:?}");
2833 assert_eq!(
2834 std::fs::read_to_string(wt.join("src/main.rs")).unwrap(),
2835 source
2836 );
2837 assert_eq!(
2838 std::fs::read_to_string(&unknown).unwrap(),
2839 "keep this note\n"
2840 );
2841 assert_eq!(std::fs::read_to_string(wt.join("source")).unwrap(), "ca00");
2842 let ledger: serde_json::Value = serde_json::from_slice(
2843 &std::fs::read(
2844 f.home
2845 .join("runs")
2846 .join(&state.id)
2847 .join("build-outputs.json"),
2848 )
2849 .unwrap(),
2850 )
2851 .unwrap();
2852 assert_eq!(
2853 ledger["identity"],
2854 if precreate {
2855 "rustc-profile-dep-info"
2856 } else {
2857 "cache-tag"
2858 }
2859 );
2860 if precreate {
2861 assert!(!target.join("CACHEDIR.TAG").exists());
2862 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
2863 }
2864 let deleted: Vec<PathBuf> = serde_json::from_value(ledger["files"].clone()).unwrap();
2865 assert!(deleted.contains(&artifact));
2866 assert_eq!(deleted, proven);
2867 for path in deleted {
2868 assert!(!path.exists());
2869 assert!(supported_dirs.contains(&path.parent().unwrap().to_owned()));
2870 }
2871 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
2872 }
2873
2874 fn missing_tag_target(wt: &Path) {
2875 Fx::target(wt);
2876 let target = wt.join("target");
2877 std::fs::remove_file(target.join("CACHEDIR.TAG")).unwrap();
2878 for subdir in ["debug/deps", "debug/.fingerprint"] {
2879 std::fs::create_dir_all(target.join(subdir)).unwrap();
2880 }
2881 std::fs::write(target.join(".rustc_info.json"),
2882 r#"{"rustc_fingerprint":123,"outputs":{"1":{"success":true,"stdout":"rustc fixture","stderr":""}},"successes":{}}"#).unwrap();
2883 }
2884
2885 fn v2_target(wt: &Path) -> PathBuf {
2887 missing_tag_target(wt);
2888 let target = wt.join("target");
2889 std::fs::remove_dir(target.join("debug/deps")).unwrap();
2890 std::fs::remove_dir(target.join("debug/.fingerprint")).unwrap();
2891 let unit = target.join("debug/build/fixture/0123456789abcdef");
2892 let out = unit.join("out");
2893 std::fs::create_dir_all(&out).unwrap();
2894 std::fs::create_dir(unit.join("fingerprint")).unwrap();
2895 for name in [
2896 "lib-fixture",
2897 "lib-fixture.json",
2898 "dep-lib-fixture",
2899 "invoked.timestamp",
2900 ] {
2901 std::fs::write(unit.join("fingerprint").join(name), "fixture").unwrap();
2902 }
2903 let artifact = out.join("libfixture-0123456789abcdef.rlib");
2904 std::fs::copy(target.join("debug/output"), &artifact).unwrap();
2905 std::fs::write(
2906 out.join("fixture-0123456789abcdef.d"),
2907 format!("{}: source\n", artifact.display()),
2908 )
2909 .unwrap();
2910 artifact
2911 }
2912
2913 #[tokio::test]
2914 async fn missing_tag_v2_proves_only_compiler_units_and_preserves_retained_links() {
2915 let f = Fx::new().await;
2916 let state = f.run("bc01", true).await;
2917 let wt = &state.candidates[0].worktree;
2918 let artifact = v2_target(wt);
2919 let unknown = artifact.parent().unwrap().join("operator-note");
2920 std::fs::write(&unknown, "retain").unwrap();
2921 let script_unit = wt.join("target/debug/build/fixture/fedcba9876543210");
2923 let script = script_unit.join("out");
2924 std::fs::create_dir_all(&script).unwrap();
2925 std::fs::create_dir(script_unit.join("fingerprint")).unwrap();
2926 std::fs::write(script_unit.join("fingerprint/invoked.timestamp"), "fixture").unwrap();
2927 let decoy = script.join("libfixture-fedcba9876543210.rlib");
2928 std::fs::copy(&artifact, &decoy).unwrap();
2929 std::fs::write(
2930 script.join("fixture-fedcba9876543210.d"),
2931 format!("{}: source\n", decoy.display()),
2932 )
2933 .unwrap();
2934 let uplift = wt.join("target/debug/libfixture.rlib");
2936 std::fs::hard_link(&artifact, &uplift).unwrap();
2937 std::fs::write(
2938 wt.join("target/debug/fixture.d"),
2939 format!("{}: source\n", uplift.display()),
2940 )
2941 .unwrap();
2942 let bin_unit = wt.join("target/debug/build/fixture/1111111111111111");
2943 std::fs::create_dir_all(bin_unit.join("out")).unwrap();
2944 std::fs::create_dir(bin_unit.join("fingerprint")).unwrap();
2945 for name in [
2946 "bin-fixture",
2947 "bin-fixture.json",
2948 "dep-bin-fixture",
2949 "invoked.timestamp",
2950 ] {
2951 std::fs::write(bin_unit.join("fingerprint").join(name), "fixture").unwrap();
2952 }
2953 let bin = bin_unit.join("out/fixture");
2954 std::fs::hard_link(wt.join("target/debug/output"), &bin).unwrap();
2955 std::fs::write(
2956 bin.with_extension("d"),
2957 format!("{}: source\n", bin.display()),
2958 )
2959 .unwrap();
2960 let retained = wt.join("retained-link");
2961 std::fs::hard_link(&artifact, &retained).unwrap();
2962 use std::os::unix::fs::MetadataExt;
2963 let expected_allocation = std::fs::metadata(wt.join("target/debug/output"))
2964 .unwrap()
2965 .blocks()
2966 * 512;
2967 let sentinel = std::fs::read(&retained).unwrap();
2968 let first = sweep(&f.home, &Fx::disk(), Fx::now()).await;
2969 assert_eq!(first.freed_bytes, 800, "{first:?}");
2970 assert!(!bin.exists());
2971 assert_eq!(
2972 first.estimated_released_allocated_bytes,
2973 Some(expected_allocation)
2974 );
2975 assert_eq!(std::fs::read(&retained).unwrap(), sentinel);
2976 assert!(retained.is_file());
2977 assert!(unknown.is_file());
2978 assert!(decoy.is_file());
2979 assert!(!artifact.exists());
2980 assert_eq!(std::fs::metadata(retained).unwrap().len(), 200);
2982 }
2983
2984 #[tokio::test]
2985 async fn missing_tag_v2_accepts_completed_build_script_run_units() {
2986 let f = Fx::new().await;
2987 let state = f.run("bc02", true).await;
2988 let wt = &state.candidates[0].worktree;
2989 let artifact = v2_target(wt);
2990 let run = wt.join("target/debug/build/fixture/2222222222222222");
2991 std::fs::create_dir_all(run.join("out")).unwrap();
2992 std::fs::create_dir(run.join("fingerprint")).unwrap();
2993 for name in [
2994 "run-build-script-build-script-build",
2995 "run-build-script-build-script-build.json",
2996 ] {
2997 std::fs::write(run.join("fingerprint").join(name), "fixture").unwrap();
2998 }
2999 let decoy = run.join("out/libdecoy-2222222222222222.rlib");
3001 std::fs::copy(&artifact, &decoy).unwrap();
3002 std::fs::write(
3003 run.join("out/decoy-2222222222222222.d"),
3004 format!("{}: source\n", decoy.display()),
3005 )
3006 .unwrap();
3007 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3008 assert!(report.freed_bytes > 0, "{report:?}");
3009 assert!(!artifact.exists());
3010 assert!(decoy.is_file());
3011 }
3012
3013 #[tokio::test]
3014 async fn missing_tag_v2_rejects_incomplete_mixed_and_symlinked_layouts() {
3015 for invalid in [
3016 "build",
3017 "deps",
3018 ".fingerprint",
3019 "out",
3020 "fingerprint",
3021 "hash",
3022 "mixed",
3023 "unit-fingerprint",
3024 "compiler-fingerprint",
3025 "absent-out",
3026 ] {
3027 let f = Fx::new().await;
3028 let state = f.run("bc02", true).await;
3029 let wt = &state.candidates[0].worktree;
3030 let artifact = v2_target(wt);
3031 let unit = artifact.parent().unwrap().parent().unwrap();
3032 let profile = wt.join("target/debug");
3033 match invalid {
3034 "mixed" => {
3035 std::fs::create_dir(profile.join("deps")).unwrap();
3036 std::fs::create_dir(profile.join(".fingerprint")).unwrap();
3037 }
3038 "hash" => {
3039 std::fs::rename(unit, unit.with_file_name("unknown")).unwrap();
3040 }
3041 "unit-fingerprint" => {
3042 std::fs::remove_file(unit.join("fingerprint/invoked.timestamp")).unwrap();
3043 }
3044 "compiler-fingerprint" => {
3045 std::fs::remove_file(unit.join("fingerprint/lib-fixture.json")).unwrap();
3046 }
3047 "absent-out" => {
3048 std::fs::remove_dir_all(unit.join("out")).unwrap();
3049 }
3050 name => {
3051 let path = if matches!(name, "out" | "fingerprint") {
3052 unit.join(name)
3053 } else {
3054 profile.join(name)
3055 };
3056 if path.exists() {
3057 std::fs::remove_dir_all(&path).unwrap();
3058 }
3059 std::os::unix::fs::symlink(&f.repo, &path).unwrap();
3060 }
3061 }
3062 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3063 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
3064 assert!(!report.retained.is_empty(), "{invalid}: {report:?}");
3065 }
3066 }
3067
3068 #[tokio::test]
3069 async fn bookkeeping_flushes_do_not_extend_artifact_grace_including_legacy_records() {
3070 for legacy in [false, true] {
3071 let f = Fx::new().await;
3072 let mut state = f.run("ac00", true).await;
3073 Fx::target(&state.candidates[0].worktree);
3074 let original = state.updated_at;
3075 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
3076 if legacy {
3077 let path = f.home.join("runs").join(&state.id).join("run.json");
3078 let mut value = serde_json::to_value(&state).unwrap();
3079 value.as_object_mut().unwrap().remove("output_activity_at");
3080 std::fs::write(path, serde_json::to_vec(&value).unwrap()).unwrap();
3081 state = RunState::load_under(&state.id, &f.home).unwrap();
3082 assert_eq!(state.output_activity(), original);
3083 }
3084 let age = state.output_activity();
3085 for offset in [30, 20, 10] {
3086 state
3087 .save_under_at(&f.home, now - jiff::SignedDuration::from_secs(offset))
3088 .unwrap();
3089 state = RunState::load_under(&state.id, &f.home).unwrap();
3090 assert_eq!(state.output_activity(), age);
3091 }
3092 let mut disk = Fx::disk();
3093 disk.fold_grace_secs = 3600;
3094 let report = sweep(&f.home, &disk, now).await;
3095 assert_eq!(report.freed_bytes, 200, "legacy={legacy}: {report:?}");
3096 }
3097 }
3098
3099 #[tokio::test]
3100 async fn recent_execution_outputs_and_released_output_leases_retain_artifacts() {
3101 for evidence in [
3102 "execution",
3103 "artifact",
3104 "dep-info",
3105 "fingerprint",
3106 "lease",
3107 "future",
3108 ] {
3109 let f = Fx::new().await;
3110 let mut state = f.run("ac01", true).await;
3111 let wt = state.candidates[0].worktree.clone();
3112 missing_tag_target(&wt);
3113 let now = Fx::now() + jiff::SignedDuration::from_secs(7200);
3114 let recent = now - jiff::SignedDuration::from_secs(10);
3115 match evidence {
3116 "execution" => {
3117 state.note_output_activity(recent);
3118 state.save_under_at(&f.home, recent).unwrap();
3119 }
3120 "lease" => {
3121 let lease = OutputLease::borrow(&wt).unwrap().unwrap();
3122 let file = lease._file.try_clone().unwrap();
3123 drop(lease);
3124 file.set_modified(std::time::SystemTime::from(recent))
3125 .unwrap();
3126 }
3127 _ => {
3128 let relative = match evidence {
3129 "dep-info" => "debug/output.d",
3130 "fingerprint" => "debug/.fingerprint/invoked.timestamp",
3131 _ => "debug/output",
3132 };
3133 let path = wt.join("target").join(relative);
3134 let file = OpenOptions::new()
3135 .write(true)
3136 .create(true)
3137 .truncate(false)
3138 .open(path)
3139 .unwrap();
3140 let at = if evidence == "future" {
3141 now + jiff::SignedDuration::from_secs(10)
3142 } else {
3143 recent
3144 };
3145 file.set_modified(std::time::SystemTime::from(at)).unwrap();
3146 }
3147 }
3148 let mut disk = Fx::disk();
3149 disk.fold_grace_secs = 3600;
3150 let report = sweep(&f.home, &disk, now).await;
3151 assert_eq!(report.freed_bytes, 0, "{evidence}: {report:?}");
3152 assert!(
3153 report.retained.iter().any(|s| s.contains("grace period")
3154 || s.contains("execution activity after inventory")),
3155 "{evidence}: {report:?}"
3156 );
3157 let aged = now + jiff::SignedDuration::from_secs(7200);
3158 let report = sweep(&f.home, &disk, aged).await;
3159 assert_eq!(report.freed_bytes, 200, "{evidence}: {report:?}");
3160 }
3161 }
3162
3163 #[tokio::test]
3164 async fn missing_tag_proof_preserves_locks_unknown_files_and_retirement_boundary() {
3165 let f = Fx::new().await;
3166 let state = f.run("ac02", true).await;
3167 let wt = &state.candidates[0].worktree;
3168 missing_tag_target(wt);
3169 let target = wt.join("target");
3170 let unknown = target.join("debug/notes");
3171 std::fs::write(&unknown, "operator data").unwrap();
3172 for lock in [".cargo-lock", ".cargo-build-lock", ".cargo-artifact-lock"] {
3173 let path = target.join("debug").join(lock);
3174 let file = OpenOptions::new()
3175 .create(true)
3176 .truncate(false)
3177 .read(true)
3178 .write(true)
3179 .open(&path)
3180 .unwrap();
3181 crate::pr_head::try_lock_file(&file).unwrap();
3182 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3183 assert_eq!(report.freed_bytes, 0, "{lock}: {report:?}");
3184 assert!(
3185 report
3186 .retained
3187 .iter()
3188 .any(|s| s.contains(path.to_str().unwrap())),
3189 "{report:?}"
3190 );
3191 }
3192 let owner = crate::pr_head::owner_lock(&f.home, &state.id).unwrap();
3193 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3194 drop(owner);
3195 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
3196 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3197 drop(driver);
3198 let output = OutputLease::borrow(wt).unwrap().unwrap();
3199 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3200 drop(output);
3201 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3202 assert_eq!(report.freed_bytes, 200, "{report:?}");
3203 assert_eq!(std::fs::read_to_string(unknown).unwrap(), "operator data");
3204 assert!(!target.join("CACHEDIR.TAG").exists());
3205 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
3206 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
3207 assert_eq!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes, 0);
3208 }
3209
3210 #[tokio::test]
3211 async fn missing_tag_partial_reclamation_can_resume_without_adopting_residue() {
3212 let f = Fx::new().await;
3213 let state = f.run("ac04", true).await;
3214 let wt = &state.candidates[0].worktree;
3215 missing_tag_target(wt);
3216 let target = wt.join("target");
3217 let first = target.join("debug/output");
3218 let second = target.join("debug/second");
3219 std::fs::copy(&first, &second).unwrap();
3220 std::fs::write(
3221 second.with_extension("d"),
3222 format!("{}: source\n", second.display()),
3223 )
3224 .unwrap();
3225 let outputs = declared_outputs(&target).unwrap();
3226 assert_eq!(outputs.len(), 2);
3227 std::fs::write(f.home.join("runs").join(&state.id).join("build-outputs.json"),
3229 serde_json::to_vec(&serde_json::json!({"target":target,"files":outputs,"action":"reclaim-declared-outputs"})).unwrap()).unwrap();
3230 std::fs::remove_file(first).unwrap();
3231 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3232 assert_eq!(report.freed_bytes, 200, "{report:?}");
3233 assert!(!second.exists());
3234 assert!(second.with_extension("d").exists());
3235 assert!(!residue_proof(&f.home, &state.id, wt).unwrap().exists());
3236 }
3237
3238 #[tokio::test]
3239 async fn missing_tag_proof_rejects_invalid_identity_structure_and_symlinks() {
3240 for invalid in [
3241 "identity",
3242 "deps",
3243 "fingerprint",
3244 "lock",
3245 "dep-info",
3246 "tag-symlink",
3247 "bad-tag",
3248 ] {
3249 let f = Fx::new().await;
3250 let state = f.run("ac03", true).await;
3251 let wt = &state.candidates[0].worktree;
3252 missing_tag_target(wt);
3253 let target = wt.join("target");
3254 match invalid {
3255 "identity" => std::fs::write(
3256 target.join(".rustc_info.json"),
3257 r#"{"rustc_fingerprint":123}"#,
3258 )
3259 .unwrap(),
3260 "deps" | "fingerprint" => {
3261 let path = target.join("debug").join(if invalid == "deps" {
3262 "deps"
3263 } else {
3264 ".fingerprint"
3265 });
3266 std::fs::remove_dir(&path).unwrap();
3267 std::os::unix::fs::symlink(&f.repo, path).unwrap();
3268 }
3269 "lock" => {
3270 std::fs::remove_file(target.join("debug/.cargo-lock")).unwrap();
3271 }
3272 "dep-info" => std::fs::write(target.join("debug/output.d"), "").unwrap(),
3273 "tag-symlink" => {
3274 std::os::unix::fs::symlink(target.join("absent"), target.join("CACHEDIR.TAG"))
3275 .unwrap()
3276 }
3277 "bad-tag" => std::fs::write(target.join("CACHEDIR.TAG"), "unknown").unwrap(),
3278 _ => unreachable!(),
3279 }
3280 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3281 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
3282 assert!(target.join("debug/output").is_file());
3283 }
3284 }
3285
3286 #[tokio::test]
3287 async fn sweep_budget_oversized_target_makes_progress_without_a_second_deletion() {
3288 let f = Fx::new().await;
3289 let first = f.run("ba00", true).await;
3290 let second = f.run("ba01", true).await;
3291 for state in [&first, &second] {
3292 Fx::target(&state.candidates[0].worktree);
3293 }
3294 let mut disk = Fx::disk();
3295 disk.worktree_reclaim_bytes = 100;
3296 let out = sweep(&f.home, &disk, Fx::now()).await;
3297 assert_eq!(out.freed_bytes, 200, "{out:?}");
3298 assert!(
3299 out.retained
3300 .contains(&format!("{}: per-pass byte budget spent", second.id)),
3301 "{out:?}"
3302 );
3303 assert!(
3304 !first.candidates[0]
3305 .worktree
3306 .join("target/debug/output")
3307 .exists()
3308 );
3309 assert!(
3310 second.candidates[0]
3311 .worktree
3312 .join("target/debug/output")
3313 .exists()
3314 );
3315 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
3316 }
3317
3318 #[tokio::test]
3319 async fn sweep_budget_pressure_prioritizes_largest_over_oldest() {
3320 let f = Fx::new().await;
3321 let small = f.run("bb00", true).await;
3322 let large = f.run("bb01", true).await;
3323 for state in [&small, &large] {
3324 Fx::target(&state.candidates[0].worktree);
3325 }
3326 let output = large.candidates[0].worktree.join("target/debug/output");
3327 std::fs::OpenOptions::new()
3328 .write(true)
3329 .open(&output)
3330 .unwrap()
3331 .set_len(1000)
3332 .unwrap();
3333 let mut disk = Fx::disk();
3334 disk.worktree_reclaim_bytes = 300;
3335 disk.min_free_bytes = u64::MAX;
3336 disk.worktree_limit_bytes = u64::MAX;
3338 let out = sweep(&f.home, &disk, Fx::now()).await;
3339 assert_eq!(out.freed_bytes, 1000, "{out:?}");
3340 assert!(!output.exists());
3341 assert!(
3342 small.candidates[0]
3343 .worktree
3344 .join("target/debug/output")
3345 .exists()
3346 );
3347 }
3348
3349 #[tokio::test]
3350 async fn sweep_budget_zero_remaining_and_inventory_are_distinct() {
3351 let f = Fx::new().await;
3352 let first = f.run("bc00", true).await;
3353 let second = f.run("bc01", true).await;
3354 for state in [&first, &second] {
3355 Fx::target(&state.candidates[0].worktree);
3356 }
3357 std::fs::create_dir_all(
3359 first.candidates[0]
3360 .worktree
3361 .join("target/debug/incremental"),
3362 )
3363 .unwrap();
3364 std::fs::write(
3365 first.candidates[0]
3366 .worktree
3367 .join("target/debug/incremental/cache"),
3368 [0; 1000],
3369 )
3370 .unwrap();
3371 let mut disk = Fx::disk();
3372 disk.worktree_reclaim_bytes = 0;
3373 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3374 disk.worktree_reclaim_bytes = 300;
3375 let out = sweep(&f.home, &disk, Fx::now()).await;
3376 assert_eq!(out.freed_bytes, 200, "{out:?}");
3377 assert!(
3378 !first.candidates[0]
3379 .worktree
3380 .join("target/debug/output")
3381 .exists()
3382 );
3383 assert!(
3384 second.candidates[0]
3385 .worktree
3386 .join("target/debug/output")
3387 .exists()
3388 );
3389 assert!(
3390 out.retained
3391 .iter()
3392 .any(|r| r.contains("declared outputs exceed"))
3393 );
3394 }
3395
3396 #[tokio::test]
3397 async fn sweep_budget_exception_preserves_grace_liveness_and_locks() {
3398 let f = Fx::new().await;
3399 let mut protected = f.run("bd00", true).await;
3400 let later = f.run("bd01", true).await;
3401 for state in [&protected, &later] {
3402 Fx::target(&state.candidates[0].worktree);
3403 }
3404 let mut disk = Fx::disk();
3405 disk.worktree_reclaim_bytes = 100;
3406 disk.fold_grace_secs = 3600;
3407 let out = sweep(&f.home, &disk, Fx::now()).await;
3408 assert_eq!(out.freed_bytes, 0);
3409 assert!(
3410 out.retained
3411 .iter()
3412 .any(|r| r.contains("within grace period"))
3413 );
3414 disk.fold_grace_secs = 0;
3415 protected.driver_exited = false;
3416 protected.driver_pid = Some(std::process::id());
3417 protected.driver_started_at = crate::proc::process_started_at(std::process::id());
3418 protected
3419 .save_under_at(&f.home, protected.updated_at)
3420 .unwrap();
3421 let out = sweep(&f.home, &disk, Fx::now()).await;
3422 assert!(
3423 out.retained
3424 .iter()
3425 .any(|r| r.contains("run live or ownership unknown"))
3426 );
3427 assert_eq!(
3428 out.freed_bytes, 200,
3429 "live target must not consume exception: {out:?}"
3430 );
3431 let output = protected.candidates[0].worktree.join("target/debug/output");
3432 assert!(output.exists());
3433 protected.driver_exited = true;
3434 protected.save_under(&f.home).unwrap();
3435 let owner = crate::pr_head::owner_lock(&f.home, &protected.id).unwrap();
3436 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3437 assert!(output.exists());
3438 drop(owner);
3439 let cargo = cargo_locks(&protected.candidates[0].worktree.join("target")).unwrap();
3440 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3441 assert!(output.exists());
3442 drop(cargo);
3443 let lease = OutputLease::take(&protected.candidates[0].worktree, true).unwrap();
3444 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 0);
3445 assert!(output.exists());
3446 drop(lease);
3447 assert_eq!(sweep(&f.home, &disk, Fx::now()).await.freed_bytes, 200);
3448 assert!(!output.exists());
3449 }
3450
3451 #[tokio::test]
3452 async fn sweep_skips_missing_run_paths_and_reclaims_other_runs() {
3453 for remove_root in [true, false] {
3454 let f = Fx::new().await;
3455 let stale = Fx::new().await;
3456 let mut missing = stale.run("aa00", false).await;
3457 Fx::target(&missing.candidates[0].worktree);
3458 missing.save_under(&f.home).unwrap();
3459 if remove_root {
3460 std::fs::remove_dir_all(missing.worktree_root()).unwrap();
3461 std::fs::remove_dir_all(stale.repo.join(".git")).unwrap();
3464 } else {
3465 std::fs::remove_dir_all(&stale.repo).unwrap();
3466 }
3467 let healthy = f.run("aa01", false).await;
3468 Fx::target(&healthy.candidates[0].worktree);
3469
3470 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3471 assert!(report.failures.is_empty(), "{report:?}");
3472 assert_eq!(report.freed_bytes, 200, "{report:?}");
3473 assert!(
3474 !healthy.candidates[0]
3475 .worktree
3476 .join("target/debug/output")
3477 .exists()
3478 );
3479 if !remove_root {
3480 assert!(
3482 missing.candidates[0]
3483 .worktree
3484 .join("target/debug/output")
3485 .exists()
3486 );
3487 }
3488 }
3489 }
3490
3491 #[tokio::test]
3492 async fn sweep_skips_missing_candidate_checkout_and_reclaims_later_runs() {
3493 let f = Fx::new().await;
3494 let mut missing = f.run("aa00", false).await;
3495 Fx::target(&missing.candidates[0].worktree);
3496 missing.repo = missing.candidates[0].worktree.clone();
3497 missing.save_under(&f.home).unwrap();
3498 std::fs::remove_dir_all(&missing.repo).unwrap();
3499 assert!(missing.worktree_root().is_dir());
3500 assert!(f.repo.is_dir());
3501
3502 let healthy = f.run("aa01", false).await;
3503 Fx::target(&healthy.candidates[0].worktree);
3504
3505 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3506 assert!(report.failures.is_empty(), "{report:?}");
3507 assert_eq!(report.freed_bytes, 200, "{report:?}");
3508 assert!(
3509 !healthy.candidates[0]
3510 .worktree
3511 .join("target/debug/output")
3512 .exists()
3513 );
3514 }
3515
3516 #[tokio::test]
3517 async fn sweep_reports_invalid_repository_and_still_reclaims_later_runs() {
3518 let f = Fx::new().await;
3519 let broken = Fx::new().await;
3520 let mut invalid = broken.run("aa00", false).await;
3521 Fx::target(&invalid.candidates[0].worktree);
3522 invalid.save_under(&f.home).unwrap();
3523 std::fs::remove_dir_all(broken.repo.join(".git")).unwrap();
3524 let healthy = f.run("aa01", false).await;
3525 Fx::target(&healthy.candidates[0].worktree);
3526
3527 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
3528 assert_eq!(report.failures.len(), 1, "{report:?}");
3529 assert!(report.failures[0].starts_with(&format!("{}:", invalid.id)));
3530 assert_eq!(report.freed_bytes, 200, "{report:?}");
3531 assert!(
3532 !healthy.candidates[0]
3533 .worktree
3534 .join("target/debug/output")
3535 .exists()
3536 );
3537 assert!(
3538 invalid.candidates[0]
3539 .worktree
3540 .join("target/debug/output")
3541 .exists()
3542 );
3543 }
3544
3545 async fn reconciliation_plan(f: &Fx, task: &Task, source: &RunState) -> serde_json::Value {
3546 let path = reconcile::run(&f.home, &task.id, &source.id, None)
3547 .await
3548 .unwrap();
3549 serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap()
3550 }
3551
3552 #[tokio::test]
3553 async fn reconciliation_standing_chat_applies_the_owners_request_without_a_phrase() {
3554 use crate::{
3555 chat_control::{Operation, dispatch},
3556 talk::{self, Talks},
3557 };
3558 let f = Fx::new().await;
3559 let source = f.run("ae00", true).await;
3560 let clean = f.run("ae01", false).await;
3561 let queue = crate::queue::Queue::at(f.home.join("queue"));
3562 let talks = Talks::at(f.home.join("talks"));
3563 let open_talk = |id: &str| -> talk::Talk {
3564 let mut conversation = serde_json::from_value(serde_json::json!({
3565 "schema":1,"id":id,"repo":f.repo,"agent":"worker","status":"open","turns":[],
3566 "created_at":Timestamp::now(),"updated_at":Timestamp::now(),
3567 "seat":crate::agent::SeatState::new("talk","worker",1)
3568 }))
3569 .unwrap();
3570 talks.put(&mut conversation).unwrap();
3571 conversation
3572 };
3573 let conversation = open_talk("talk-owner");
3574 let other = open_talk("talk-other");
3575 let task = f.task(&[source.clone(), clean.clone()]);
3577 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3578 let operation = |apply: Option<&str>| Operation::ReconcileRetention {
3579 task: task.id.clone(),
3580 authority: source.id.clone(),
3581 apply: apply.map(str::to_owned),
3582 };
3583 let legacy_path = reconcile::run(&f.home, &task.id, &source.id, None)
3585 .await
3586 .unwrap();
3587 let legacy: serde_json::Value =
3588 serde_json::from_slice(&std::fs::read(legacy_path).unwrap()).unwrap();
3589 let legacy_id = legacy["id"].as_str().unwrap();
3590 assert!(
3591 dispatch(
3592 &queue,
3593 &f.home,
3594 &talks,
3595 &conversation.id,
3596 operation(Some(legacy_id))
3597 )
3598 .await
3599 .is_err()
3600 );
3601 let summary = dispatch(&queue, &f.home, &talks, &conversation.id, operation(None))
3602 .await
3603 .unwrap();
3604 assert!(summary.contains("Complete review summary"));
3605 assert!(summary.contains(&source.id) && summary.contains(&clean.id));
3606 let root = f.home.join("retention-reconciliations");
3607 let displayed: serde_json::Value =
3608 serde_json::from_str(summary.split_once('\n').unwrap().1).unwrap();
3609 let id = displayed["plan"].as_str().unwrap();
3610 let dir = root.join(id);
3611 let plan: serde_json::Value =
3612 serde_json::from_slice(&std::fs::read(dir.join("plan.json")).unwrap()).unwrap();
3613 assert_eq!(plan["version"], 2);
3614 assert!(clean.candidates[0].worktree.exists());
3615 assert!(!dir.join("approved.json").exists());
3616 assert!(
3618 dispatch(&queue, &f.home, &talks, &other.id, operation(Some(id)))
3619 .await
3620 .is_err()
3621 );
3622 assert!(!dir.join("approved.json").exists());
3623 let error = reconcile::interrupt_chat_after_fence(
3626 &f.home,
3627 &task,
3628 id,
3629 reconcile::ChatRequest {
3630 talks: &talks,
3631 conversation: &conversation.id,
3632 },
3633 )
3634 .await
3635 .unwrap_err();
3636 assert!(
3637 error.to_string().contains("simulated interruption"),
3638 "{error:#}"
3639 );
3640 assert!(clean.candidates[0].worktree.exists());
3641 assert!(dir.join("approved.json").exists());
3642 dispatch(
3643 &queue,
3644 &f.home,
3645 &talks,
3646 &conversation.id,
3647 operation(Some(id)),
3648 )
3649 .await
3650 .unwrap();
3651 assert!(!clean.candidates[0].worktree.exists());
3652 assert!(source.candidates[0].worktree.exists());
3653 let audit: serde_json::Value =
3654 serde_json::from_slice(&std::fs::read(dir.join("approved.json")).unwrap()).unwrap();
3655 assert_eq!(audit["version"], 2);
3656 assert_eq!(
3657 audit["owner"],
3658 serde_json::json!({"conversation": conversation.id})
3659 );
3660 dispatch(
3661 &queue,
3662 &f.home,
3663 &talks,
3664 &conversation.id,
3665 operation(Some(id)),
3666 )
3667 .await
3668 .unwrap();
3669 }
3670
3671 #[tokio::test]
3672 async fn reconciliation_proves_unmeasured_startup_candidates_empty() {
3673 let f = Fx::new().await;
3674 let source = f.run("af00", true).await;
3675 let mut clean = f.run("af01", false).await;
3676 let mut dirty = f.run("af02", false).await;
3677 for state in [&mut clean, &mut dirty] {
3678 state.status = RunStatus::Prep;
3679 state.prep_complete = Some(false);
3680 state.candidates[0].empty = false;
3681 state.save_under(&f.home).unwrap();
3682 }
3683 std::fs::write(dirty.candidates[0].worktree.join("source"), "unsaved work").unwrap();
3684 let task = f.task(&[source.clone(), clean.clone(), dirty.clone()]);
3685 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3686 let plan = reconciliation_plan(&f, &task, &source).await;
3687 let selected: Vec<_> = plan["candidates"]
3688 .as_array()
3689 .unwrap()
3690 .iter()
3691 .filter(|e| e["selected"] == true)
3692 .collect();
3693 assert_eq!(selected.len(), 1, "{plan:#}");
3694 assert_eq!(selected[0]["run"], clean.id);
3695 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3696 .await
3697 .unwrap();
3698 assert!(!clean.candidates[0].worktree.exists());
3699 assert!(dirty.candidates[0].worktree.exists());
3700 assert!(
3701 RunState::load_under(&dirty.id, &f.home)
3702 .unwrap()
3703 .retired_to
3704 .is_none()
3705 );
3706 }
3707
3708 #[tokio::test]
3709 async fn reconciliation_blanket_retention_selected_source_and_repeated_batches() {
3710 let f = Fx::new().await;
3711 let mut source = f.run("aa00", true).await;
3712 let alternative = f.run("aa01", true).await;
3713 let head = git::rev_parse(&f.repo, &source.candidates[0].branch)
3714 .await
3715 .unwrap();
3716 source.reviewed_commits = Some(vec![head.clone()]);
3717 source.save_under(&f.home).unwrap();
3718 let mut runs = vec![source.clone(), alternative.clone()];
3719 for i in 2..14 {
3720 runs.push(f.run(&format!("aa{i:02}"), false).await);
3721 }
3722 let mut task = f.task(&runs);
3723 task.pr_recovery = Some(crate::pr_recovery::Selection {
3724 url: "https://example.invalid/pull/1".into(),
3725 branch: "selected".into(),
3726 head: head.clone(),
3727 source_head: Some(head.clone()),
3728 source_run: Some(source.id.clone()),
3729 source_proof: None,
3730 disposition: Some(crate::pr_recovery::Disposition::Land),
3731 instruction: task.instruction.clone(),
3732 approved_at: Timestamp::now(),
3733 });
3734 source.pr_recovery = task.pr_recovery.clone();
3735 source.save_under(&f.home).unwrap();
3736 runs[0] = source.clone();
3737 crate::queue::Queue::at(f.home.join("queue"))
3738 .put(&mut task)
3739 .unwrap();
3740 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3741 let task_before = serde_json::to_value(&task).unwrap();
3742 let mut report = Report::default();
3743 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
3744 assert_eq!(
3745 report.retired, 0,
3746 "automatic cleanup must still refuse blanket markers"
3747 );
3748 assert!(
3749 report
3750 .retained
3751 .iter()
3752 .any(|reason| reason.contains("explicitly retained"))
3753 );
3754 let mut total = 0;
3755 for expected in [5, 5, 2, 0] {
3756 let plan = reconciliation_plan(&f, &task, &source).await;
3757 let selected: Vec<_> = plan["candidates"]
3758 .as_array()
3759 .unwrap()
3760 .iter()
3761 .filter(|e| e["selected"] == true)
3762 .collect();
3763 assert_eq!(selected.len(), expected, "{plan:#}");
3764 for e in &selected {
3765 assert!(
3766 RunState::load_under(e["run"].as_str().unwrap(), &f.home)
3767 .unwrap()
3768 .retired_to
3769 .is_none()
3770 );
3771 }
3772 let operation = plan["id"].as_str().unwrap();
3773 for _ in 0..2 {
3774 reconcile::run(&f.home, &task.id, &source.id, Some(operation))
3775 .await
3776 .unwrap();
3777 }
3778 total += expected;
3779 }
3780 assert_eq!(total, 12);
3781 for run in &runs {
3782 let dir = f.home.join("runs").join(&run.id);
3783 assert!(dir.join("run.json").exists());
3784 assert!(dir.join("retain-recovery-sources").exists());
3785 assert!(
3786 git::branch_exists(&f.repo, &run.candidates[0].branch)
3787 .await
3788 .unwrap()
3789 );
3790 }
3791 assert!(source.candidates[0].worktree.exists());
3792 assert!(alternative.candidates[0].worktree.exists());
3793 assert_eq!(
3794 git::rev_parse(&f.repo, &source.candidates[0].branch)
3795 .await
3796 .unwrap(),
3797 head
3798 );
3799 assert_eq!(
3800 serde_json::to_value(
3801 crate::queue::Queue::at(f.home.join("queue"))
3802 .get(&task.id)
3803 .unwrap()
3804 )
3805 .unwrap(),
3806 task_before
3807 );
3808 assert_eq!(
3809 runs.iter()
3810 .filter(|r| RunState::load_under(&r.id, &f.home)
3811 .unwrap()
3812 .retired_to
3813 .is_none())
3814 .count(),
3815 2
3816 );
3817 }
3818
3819 #[tokio::test]
3820 async fn reconciliation_refuses_live_dirty_ignored_unknown_and_git_checkpoints() {
3821 let f = Fx::new().await;
3822 let source = f.run("ab00", true).await;
3823 let mut runs = vec![source.clone()];
3824 for i in 1..9 {
3825 runs.push(f.run(&format!("ab{i:02}"), false).await);
3826 }
3827 std::fs::write(runs[1].candidates[0].worktree.join("source"), "dirty").unwrap();
3828 std::fs::write(runs[2].candidates[0].worktree.join("untracked"), "source").unwrap();
3829 std::fs::create_dir(runs[3].candidates[0].worktree.join("scratch")).unwrap();
3830 std::fs::write(
3831 runs[3].candidates[0].worktree.join("scratch/source"),
3832 "ignored source",
3833 )
3834 .unwrap();
3835 std::fs::write(
3836 runs[4].worktree_root().join("unknown"),
3837 "unregistered source",
3838 )
3839 .unwrap();
3840 let checkpoint = git::git(
3841 &runs[5].candidates[0].worktree,
3842 &[
3843 "rev-parse",
3844 "--path-format=absolute",
3845 "--git-path",
3846 "sequencer",
3847 ],
3848 )
3849 .await
3850 .unwrap();
3851 std::fs::create_dir(checkpoint.trim()).unwrap();
3852 runs[6].driver_exited = false;
3853 runs[6].driver_pid = Some(std::process::id());
3854 runs[6].driver_started_at = crate::proc::process_started_at(std::process::id());
3855 runs[6].save_under(&f.home).unwrap();
3856 runs[7].driver_exited = false;
3857 runs[7].driver_pid = None;
3858 runs[7].driver_claimed = None; runs[7].save_under(&f.home).unwrap();
3860 assert_eq!(runs[7].liveness_under(&f.home, false), Liveness::Unknown);
3861
3862 let hidden = &runs[8].candidates[0].worktree;
3863 git::git(hidden, &["update-index", "--assume-unchanged", "source"])
3864 .await
3865 .unwrap();
3866 std::fs::write(hidden.join("source"), "hidden dirty implementation").unwrap();
3867 assert!(
3868 git::git(hidden, &["status", "--porcelain"])
3869 .await
3870 .unwrap()
3871 .is_empty()
3872 );
3873 let task = f.task(&runs);
3874 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3875 let plan = reconciliation_plan(&f, &task, &source).await;
3876 assert!(
3877 plan["candidates"]
3878 .as_array()
3879 .unwrap()
3880 .iter()
3881 .all(|e| e["selected"] == false),
3882 "{plan:#}"
3883 );
3884 for run in &runs {
3885 assert!(run.candidates[0].worktree.exists());
3886 }
3887 }
3888
3889 #[tokio::test]
3890 async fn reconciliation_stale_plans_and_changed_markers_fail_closed() {
3891 let f = Fx::new().await;
3892 let source = f.run("ac00", true).await;
3893 let empty = f.run("ac01", false).await;
3894 let task = f.task(&[source.clone(), empty.clone()]);
3895 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3896 let plan = reconciliation_plan(&f, &task, &source).await;
3897 std::fs::write(
3898 f.home.join("runs").join(&source.id).join("new-report"),
3899 "new review checkpoint",
3900 )
3901 .unwrap();
3902 assert!(
3903 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
3904 .await
3905 .is_err()
3906 );
3907 assert!(empty.candidates[0].worktree.exists());
3908 assert!(
3909 RunState::load_under(&empty.id, &f.home)
3910 .unwrap()
3911 .retired_to
3912 .is_none()
3913 );
3914 let marker = f
3915 .home
3916 .join("runs")
3917 .join(&empty.id)
3918 .join("retain-recovery-sources");
3919 std::fs::write(&marker, "different owner").unwrap();
3920 let plan = reconciliation_plan(&f, &task, &source).await;
3921 assert!(
3922 plan["candidates"]
3923 .as_array()
3924 .unwrap()
3925 .iter()
3926 .all(|e| e["selected"] == false)
3927 );
3928 }
3929
3930 #[tokio::test]
3931 async fn reconciliation_recovers_interrupted_state_staging_in_a_batch() {
3932 for partial in [false, true] {
3933 let f = Fx::new().await;
3934 let source = f.run("ab10", true).await;
3935 let first = f.run("ab11", false).await;
3936 let second = f.run("ab12", false).await;
3937 let task = f.task(&[source.clone(), first.clone(), second.clone()]);
3938 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
3939 let plan = reconciliation_plan(&f, &task, &source).await;
3940 let id = plan["id"].as_str().unwrap();
3941 let staging = f.home.join("runs").join(&second.id).join("run.json.tmp");
3942 std::fs::write(&staging, b"").unwrap();
3944 assert!(
3945 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3946 .await
3947 .is_err()
3948 );
3949 assert!(staging.exists());
3950 std::fs::remove_file(&staging).unwrap();
3951 let error = reconcile::interrupt_during_state_save(&f.home, &task, id, partial)
3952 .await
3953 .unwrap_err();
3954 assert!(
3955 error.to_string().contains("before run state rename"),
3956 "{error:#}"
3957 );
3958 assert!(
3959 RunState::load_under(&first.id, &f.home)
3960 .unwrap()
3961 .retired_to
3962 .is_some()
3963 );
3964 assert!(
3965 RunState::load_under(&second.id, &f.home)
3966 .unwrap()
3967 .retired_to
3968 .is_none()
3969 );
3970 assert!(first.candidates[0].worktree.exists());
3971 assert!(second.candidates[0].worktree.exists());
3972 let interrupted = std::fs::read(&staging).unwrap();
3973 std::fs::write(&staging, b"unrelated recovery source").unwrap();
3975 assert!(
3976 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3977 .await
3978 .is_err()
3979 );
3980 assert_eq!(
3981 std::fs::read(&staging).unwrap(),
3982 b"unrelated recovery source"
3983 );
3984 std::fs::write(&staging, interrupted).unwrap();
3985 reconcile::run(&f.home, &task.id, &source.id, Some(id))
3986 .await
3987 .unwrap();
3988 assert!(!staging.exists());
3989 assert!(!first.candidates[0].worktree.exists());
3990 assert!(!second.candidates[0].worktree.exists());
3991 assert_eq!(
3992 RunState::load_under(&second.id, &f.home)
3993 .unwrap()
3994 .retired_to,
3995 Some(source.id)
3996 );
3997 }
3998 }
3999
4000 #[tokio::test]
4001 async fn reconciliation_resumes_only_its_exact_durable_fence() {
4002 let f = Fx::new().await;
4003 let source = f.run("ad00", true).await;
4004 let empty = f.run("ad01", false).await;
4005 let task = f.task(&[source.clone(), empty.clone()]);
4006 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4007 let plan = reconciliation_plan(&f, &task, &source).await;
4008 let id = plan["id"].as_str().unwrap();
4009 assert!(
4010 reconcile::interrupt_after_fence(&f.home, &task, id)
4011 .await
4012 .is_err()
4013 );
4014 assert!(empty.candidates[0].worktree.exists());
4015 assert_eq!(
4016 RunState::load_under(&empty.id, &f.home)
4017 .unwrap()
4018 .retired_to
4019 .as_ref(),
4020 Some(&source.id)
4021 );
4022 reconcile::run(&f.home, &task.id, &source.id, Some(id))
4023 .await
4024 .unwrap();
4025 assert!(!empty.candidates[0].worktree.exists());
4026 git::worktree_add_branch(
4028 &f.repo,
4029 &empty.candidates[0].worktree,
4030 "restore-empty",
4031 &f.base,
4032 )
4033 .await
4034 .unwrap();
4035 assert!(
4037 reconcile::run(&f.home, &task.id, &source.id, Some(id))
4038 .await
4039 .is_err()
4040 );
4041 assert!(empty.candidates[0].worktree.exists());
4042 let mut changed = RunState::load_under(&empty.id, &f.home).unwrap();
4043 changed.prep_failure = Some("new owner edit".into());
4044 changed.save_under(&f.home).unwrap();
4045 assert!(
4046 reconcile::run(&f.home, &task.id, &source.id, Some(id))
4047 .await
4048 .is_err()
4049 );
4050 }
4051
4052 #[tokio::test]
4053 async fn reconciliation_checks_prep_intent_reflogs_and_unknown_run_fields() {
4054 let f = Fx::new().await;
4055 let source = f.run("ae00", true).await;
4056 let mut prep = f.run("ae01", false).await;
4057 let candidate = prep.candidates.pop().unwrap();
4058 prep.event("prep-intent", serde_json::json!({"branch":candidate.branch,"worktree":candidate.worktree,"start":f.base,"picks":[]}).to_string());
4059 prep.save_under(&f.home).unwrap();
4060 let reset = f.run("ae02", false).await;
4061 let wt = &reset.candidates[0].worktree;
4062 std::fs::write(wt.join("source"), "unique hidden in reflog").unwrap();
4063 git::git(wt, &["add", "."]).await.unwrap();
4064 git::git(wt, &["commit", "-m", "unique implementation"])
4065 .await
4066 .unwrap();
4067 git::git(wt, &["reset", "--hard", &f.base]).await.unwrap();
4068 let unknown = f.run("ae03", false).await;
4069 let unknown_path = f.home.join("runs").join(&unknown.id).join("run.json");
4070 let mut raw: serde_json::Value =
4071 serde_json::from_slice(&std::fs::read(&unknown_path).unwrap()).unwrap();
4072 raw["unknown_checkpoint"] = serde_json::json!({"source":"retain"});
4073 std::fs::write(unknown_path, serde_json::to_vec_pretty(&raw).unwrap()).unwrap();
4074 let task = f.task(&[source.clone(), prep.clone(), reset.clone(), unknown.clone()]);
4075 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4076 let plan = reconciliation_plan(&f, &task, &source).await;
4077 let selected: Vec<_> = plan["candidates"]
4078 .as_array()
4079 .unwrap()
4080 .iter()
4081 .filter(|e| e["selected"] == true)
4082 .collect();
4083 assert_eq!(selected.len(), 1, "{plan:#}");
4084 assert_eq!(selected[0]["run"], prep.id);
4085 reconcile::run(&f.home, &task.id, &source.id, plan["id"].as_str())
4086 .await
4087 .unwrap();
4088 assert!(!candidate.worktree.exists());
4089 assert!(reset.candidates[0].worktree.exists());
4090 assert!(unknown.candidates[0].worktree.exists());
4091 }
4092
4093 #[tokio::test]
4094 async fn cap_reclaims_blocked_candidates_and_landing_reviewers_across_repositories() {
4095 let f = Fx::new().await;
4096 let mut state = f.run("aaaa", true).await;
4097 let candidate = state.candidates[0].worktree.clone();
4098 std::fs::write(candidate.join("source"), "uncommitted work").unwrap();
4099 Fx::target(&candidate);
4100 for seat in ["review-1", "review-2"] {
4101 let wt = state.worktree_root().join(seat);
4102 git::worktree_add_detached(&f.repo, &wt, &f.base)
4103 .await
4104 .unwrap();
4105 Fx::target(&wt);
4106 }
4107 state.status = RunStatus::Landing;
4108 state.save_under(&f.home).unwrap();
4109 let second = Fx::new().await;
4110 let other = second.run("bbbb", true).await;
4111 let mut imported = other.clone();
4113 imported.save_under(&f.home).unwrap();
4114 Fx::target(&other.candidates[0].worktree);
4115 let guard = OutputLease::borrow(&candidate).unwrap().unwrap();
4116 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4117 assert!(
4118 candidate.join("target/debug/output").exists(),
4119 "active writer protected"
4120 );
4121 assert!(
4122 !state
4123 .worktree_root()
4124 .join("review-1/target/debug/output")
4125 .exists()
4126 );
4127 assert!(
4128 !state
4129 .worktree_root()
4130 .join("review-2/target/debug/output")
4131 .exists()
4132 );
4133 assert!(
4134 !other.candidates[0]
4135 .worktree
4136 .join("target/debug/output")
4137 .exists()
4138 );
4139 assert!(out.freed_bytes >= 600);
4140 drop(guard);
4141 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4142 assert!(out.freed_bytes >= 200);
4143 assert_eq!(
4144 std::fs::read_to_string(candidate.join("source")).unwrap(),
4145 "uncommitted work"
4146 );
4147 assert!(
4148 git::branch_exists(&f.repo, &state.candidates[0].branch)
4149 .await
4150 .unwrap()
4151 );
4152 }
4153
4154 #[tokio::test]
4155 async fn dep_info_reclamation_preserves_unmanifested_nested_files() {
4156 let f = Fx::new().await;
4157 let state = f.run("cc01", false).await;
4158 let wt = &state.candidates[0].worktree;
4159 Fx::target(wt);
4160 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
4161 let dir = wt.join("target").join(dir);
4162 std::fs::create_dir_all(&dir).unwrap();
4163 std::fs::write(dir.join("notes.txt"), "user evidence").unwrap();
4164 }
4165 let manifest = wt.join("target/debug/output.d");
4167 let output = wt.join("target/debug/output");
4168 std::fs::write(
4169 &manifest,
4170 format!(
4171 "{}: {}\n",
4172 output.display(),
4173 wt.join("target/debug/notes.txt").display()
4174 ),
4175 )
4176 .unwrap();
4177 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4178 assert_eq!(report.freed_bytes, 200);
4179 assert!(!output.exists());
4180 for dir in ["debug", "release", "debug/deps", "debug/build/script/out"] {
4181 assert_eq!(
4182 std::fs::read_to_string(wt.join("target").join(dir).join("notes.txt")).unwrap(),
4183 "user evidence"
4184 );
4185 }
4186 assert!(manifest.exists());
4188 assert!(report.retained.iter().any(|s| s.contains("unmanifested")));
4189 std::fs::write(&output, "user note replacing an old binary").unwrap();
4190 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4191 assert_eq!(report.freed_bytes, 0);
4192 assert_eq!(
4193 std::fs::read_to_string(&output).unwrap(),
4194 "user note replacing an old binary"
4195 );
4196 }
4197
4198 #[tokio::test]
4199 async fn measurement_cache_expires_and_disabled_eviction_does_not_walk_outputs() {
4200 let f = Fx::new().await;
4201 let state = f.run("cc02", false).await;
4202 let wt = &state.candidates[0].worktree;
4203 Fx::target(wt);
4204 let mut disk = Fx::disk();
4205 disk.worktree_limit_bytes = u64::MAX;
4206 let now = Fx::now();
4207 let first = sweep(&f.home, &disk, now).await;
4208 std::fs::write(wt.join("target/debug/output"), [1; 1000]).unwrap();
4209 assert_eq!(
4210 sweep(&f.home, &disk, now).await.measured_bytes,
4211 first.measured_bytes
4212 );
4213 assert!(
4214 sweep(&f.home, &disk, now + jiff::SignedDuration::from_secs(301))
4215 .await
4216 .measured_bytes
4217 > first.measured_bytes
4218 );
4219 disk.worktree_limit_bytes = 0;
4220 assert_eq!(sweep(&f.home, &disk, now).await.measured_bytes, 0);
4221 }
4222
4223 #[tokio::test]
4224 async fn unfinished_source_fold_reports_its_retention_reason() {
4225 let f = Fx::new().await;
4226 let mut source = f.run("cc03", true).await;
4227 let task = f.task(&[source.clone()]);
4228 let error = crate::graph::fold_run(&mut source, true, &f.home)
4229 .await
4230 .unwrap_err();
4231 assert!(error.to_string().contains(&task.id));
4232 assert!(source.candidates[0].worktree.exists());
4233 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4234 let error = crate::graph::fold_run(&mut source, true, &f.home)
4235 .await
4236 .unwrap_err();
4237 assert!(error.to_string().contains("explicitly retained"));
4238 }
4239
4240 #[tokio::test]
4241 async fn confirmed_merged_source_folds_while_task_is_held_but_retention_still_wins() {
4242 let f = Fx::new().await;
4243 let mut source = f.run("cc04", true).await;
4244 let mut task = f.task(&[source.clone()]);
4245 task.hold_manual(Some("queue settlement pending".into()));
4246 let queue = crate::queue::Queue::at(f.home.join("queue"));
4247 queue.put(&mut task).unwrap();
4248 source.merge = Some(crate::run::MergeOutcome {
4249 mode: crate::config::MergeMode::Pr,
4250 ok: true,
4251 detail: "confirmed merged".into(),
4252 empty: false,
4253 });
4254 for status in [RunStatus::Ready, RunStatus::Failed, RunStatus::Superseded] {
4256 source.status = status;
4257 assert!(
4258 crate::graph::fold_run(&mut source, true, &f.home)
4259 .await
4260 .is_err()
4261 );
4262 assert!(source.candidates[0].worktree.exists());
4263 }
4264 git::git(
4265 &f.repo,
4266 &["merge", "--ff-only", &source.candidates[0].branch],
4267 )
4268 .await
4269 .unwrap();
4270 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
4271 .await
4272 .unwrap();
4273 source.status = RunStatus::Merged;
4274 source.save_under(&f.home).unwrap();
4275 assert!(crate::pr_recovery::source_retention_reason(&f.home, &source).is_none());
4276 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
4277 assert!(
4278 crate::graph::fold_run(&mut source, true, &f.home)
4279 .await
4280 .unwrap_err()
4281 .to_string()
4282 .contains("explicitly retained")
4283 );
4284 std::fs::remove_file(
4287 f.home
4288 .join("runs")
4289 .join(&source.id)
4290 .join("retain-recovery-sources"),
4291 )
4292 .unwrap();
4293 let (folded, unreadable) = crate::clean::fold_due(
4294 &f.home.join("runs"),
4295 &f.home,
4296 &f.bay,
4297 &Fx::disk(),
4298 Fx::now(),
4299 )
4300 .await
4301 .unwrap();
4302 assert_eq!((folded, unreadable), (1, 0));
4303 assert!(!source.candidates[0].worktree.exists());
4304 let evidence = f
4305 .home
4306 .join("runs")
4307 .join(&source.id)
4308 .join("artifacts/fold-evidence");
4309 let names: Vec<_> = std::fs::read_dir(evidence)
4310 .unwrap()
4311 .map(|entry| entry.unwrap().file_name())
4312 .collect();
4313 assert_eq!(names, [std::ffi::OsString::from("manifest.json")]);
4314 assert_eq!(
4315 queue.get(&task.id).unwrap().status,
4316 crate::queue::TaskStatus::Held
4317 );
4318 }
4319
4320 #[tokio::test]
4321 async fn merged_fold_skips_proven_squash_and_archives_unpublished_sources() {
4322 for variant in ["exact", "partial", "checkpoint", "loser", "reviewer"] {
4323 let f = Fx::new().await;
4324 let mut state = f.run("cc06", true).await;
4325 let candidate = state.candidates[0].clone();
4326 let intermediate = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
4327 std::fs::write(candidate.worktree.join("second"), [0, 1, 255, 42]).unwrap();
4328 git::rescue_commit(&candidate.worktree, "second candidate change")
4329 .await
4330 .unwrap();
4331 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
4332 state.tally = Some(crate::run::Tally {
4333 first_choice: Default::default(),
4334 borda: Default::default(),
4335 winner: 'A',
4336 rankings: 0,
4337 unanimous_initial: true,
4338 deliberated: false,
4339 changed_votes: 0,
4340 unanimous_final: true,
4341 tie_break: None,
4342 judges: 0,
4343 present: 0,
4344 quorum: 0,
4345 met_quorum: true,
4346 uncontested: Some("fixture".into()),
4347 });
4348 let reviewer = state.worktree_root().join("review-1");
4351 git::worktree_add_detached(&f.repo, &reviewer, &intermediate)
4352 .await
4353 .unwrap();
4354 std::fs::write(f.repo.join("unrelated-before"), "base advance").unwrap();
4357 git::rescue_commit(&f.repo, "advance before squash")
4358 .await
4359 .unwrap();
4360 let published = if variant == "partial" {
4361 &intermediate
4362 } else {
4363 &head
4364 };
4365 git::git(&f.repo, &["merge", "--squash", published])
4366 .await
4367 .unwrap();
4368 git::git(&f.repo, &["commit", "-m", "squash candidate"])
4369 .await
4370 .unwrap();
4371 let squash = git::rev_parse(&f.repo, "HEAD").await.unwrap();
4372 assert!(
4373 !git::git_raw(&f.repo, &["merge-base", "--is-ancestor", &head, &squash])
4374 .await
4375 .unwrap()
4376 .ok()
4377 );
4378 std::fs::write(f.repo.join("unrelated-after"), "later base advance").unwrap();
4379 git::rescue_commit(&f.repo, "advance after squash")
4380 .await
4381 .unwrap();
4382 git::git(&f.repo, &["update-ref", "refs/remotes/origin/main", "HEAD"])
4383 .await
4384 .unwrap();
4385 if matches!(variant, "checkpoint" | "reviewer") {
4386 std::fs::write(reviewer.join("unpublished"), "reviewer checkpoint").unwrap();
4388 git::rescue_commit(&reviewer, "unpublished checkpoint")
4389 .await
4390 .unwrap();
4391 if variant == "checkpoint" {
4392 git::git(&reviewer, &["reset", "--hard", &intermediate])
4393 .await
4394 .unwrap();
4395 }
4396 }
4397 if variant == "loser" {
4398 let mut loser = candidate.clone();
4399 loser.label = 'B';
4400 loser.branch = "magi/cc06/B".into();
4401 loser.worktree = state.worktree_root().join("cand-B");
4402 git::worktree_add_branch(&f.repo, &loser.worktree, &loser.branch, &f.base)
4403 .await
4404 .unwrap();
4405 std::fs::write(loser.worktree.join("unique"), "losing candidate").unwrap();
4406 git::rescue_commit(&loser.worktree, "independent loser")
4407 .await
4408 .unwrap();
4409 state.candidates.push(loser);
4410 }
4411 state.status = RunStatus::Merged;
4412 state.merge = Some(crate::run::MergeOutcome {
4413 mode: crate::config::MergeMode::Pr,
4414 ok: true,
4415 detail: "candidate A squash merged".into(),
4416 empty: false,
4417 });
4418 state.save_under(&f.home).unwrap();
4419 let review_head = git::rev_parse(&reviewer, "HEAD").await.unwrap();
4420 let result = crate::graph::fold_run(&mut state, true, &f.home).await;
4421 result.unwrap();
4422 assert!(
4423 !candidate.worktree.exists() && !reviewer.exists(),
4424 "{variant}"
4425 );
4426 assert!(
4427 !git::branch_exists(&f.repo, &candidate.branch)
4428 .await
4429 .unwrap()
4430 );
4431 let artifacts = f
4432 .home
4433 .join("runs")
4434 .join(&state.id)
4435 .join("artifacts/fold-evidence");
4436 assert_eq!(
4437 artifacts.join(format!("{head}.patch")).exists(),
4438 variant == "partial",
4439 "only a partially published winner needs evidence: {variant}"
4440 );
4441 assert_eq!(
4442 artifacts.join(format!("{intermediate}.patch")).exists(),
4443 variant == "partial",
4444 "squash proof also covers intermediate reviewer tips: {variant}"
4445 );
4446 let patches: Vec<_> = std::fs::read_dir(&artifacts)
4447 .unwrap()
4448 .map(|entry| entry.unwrap().path())
4449 .filter(|path| path.extension().is_some_and(|ext| ext == "patch"))
4450 .collect();
4451 let expected = match variant {
4452 "partial" => 2,
4453 "loser" | "reviewer" => 1,
4454 _ => 0,
4455 };
4456 assert_eq!(patches.len(), expected, "{variant}");
4457 if variant == "loser" {
4458 assert!(
4459 std::fs::read_to_string(&patches[0])
4460 .unwrap()
4461 .contains("losing candidate")
4462 );
4463 }
4464 assert!(artifacts.join("manifest.json").exists());
4465 if variant == "reviewer" {
4466 assert!(
4467 std::fs::read_to_string(artifacts.join(format!("{review_head}.patch")))
4468 .unwrap()
4469 .contains("reviewer checkpoint")
4470 );
4471 }
4472 crate::graph::fold_run(&mut state, true, &f.home)
4473 .await
4474 .unwrap();
4475 }
4476 }
4477
4478 #[tokio::test]
4479 async fn merged_fold_missing_review_source_preserves_artifacts_and_retries() {
4480 for missing in [true, false] {
4481 let f = Fx::new().await;
4482 let mut state = f.run("cc05", true).await;
4483 git::git(
4486 &f.repo,
4487 &["update-ref", "refs/remotes/origin/main", &f.base],
4488 )
4489 .await
4490 .unwrap();
4491 let candidate = state.candidates[0].clone();
4492 if !missing {
4493 std::fs::write(f.repo.join("base-advance"), "new base").unwrap();
4494 git::rescue_commit(&f.repo, "advance base before rebase")
4495 .await
4496 .unwrap();
4497 git::git(&candidate.worktree, &["rebase", "main"])
4498 .await
4499 .unwrap();
4500 git::sync_to_head(&candidate.worktree).await.unwrap();
4503 }
4504 let head = git::rev_parse(&candidate.worktree, "HEAD").await.unwrap();
4505 state.reviewed_commits = Some(Vec::new());
4507 state.candidates[0].worktree = state.worktree_root().join("under-review");
4508 git::git(
4509 &f.repo,
4510 &[
4511 "worktree",
4512 "move",
4513 candidate.worktree.to_str().unwrap(),
4514 state.candidates[0].worktree.to_str().unwrap(),
4515 ],
4516 )
4517 .await
4518 .unwrap();
4519 let reviewer = state.worktree_root().join("review-1");
4520 git::worktree_add_detached(&f.repo, &reviewer, &head)
4521 .await
4522 .unwrap();
4523 std::fs::write(reviewer.join("obsolete"), "old history").unwrap();
4526 git::rescue_commit(&reviewer, "old review history")
4527 .await
4528 .unwrap();
4529 git::git(&reviewer, &["reset", "--hard", &head])
4530 .await
4531 .unwrap();
4532 if missing {
4533 std::fs::remove_dir_all(&state.candidates[0].worktree).unwrap();
4534 }
4535 let run_dir = f.home.join("runs").join(&state.id);
4536 let artifacts = run_dir.join("artifacts");
4537 std::fs::create_dir_all(&artifacts).unwrap();
4538 std::fs::write(artifacts.join("review.json"), "review findings").unwrap();
4539 if missing {
4540 std::fs::write(artifacts.join("cand-A.patch"), "original snapshot").unwrap();
4541 }
4542 state.status = RunStatus::Merged;
4543 state.merge = Some(crate::run::MergeOutcome {
4544 mode: crate::config::MergeMode::Pr,
4545 ok: true,
4546 detail: "confirmed squash merge".into(),
4547 empty: false,
4548 });
4549 state.save_under(&f.home).unwrap();
4550 let owned = state.candidates[0].worktree.clone();
4552 state.candidates[0].worktree = f.home.join("missing-external");
4553 assert!(
4554 crate::graph::fold_run(&mut state, true, &f.home)
4555 .await
4556 .unwrap_err()
4557 .to_string()
4558 .contains("outside the inventoried")
4559 );
4560 state.candidates[0].worktree = owned;
4561 let guard = OutputLease::borrow(&reviewer).unwrap().unwrap();
4563 assert!(
4564 crate::graph::fold_run(&mut state, true, &f.home)
4565 .await
4566 .is_err()
4567 );
4568 assert!(reviewer.exists());
4569 drop(guard);
4570 let evidence = artifacts.join("fold-evidence");
4572 std::fs::write(&evidence, "blocked").unwrap();
4573 assert!(
4574 crate::graph::fold_run(&mut state, true, &f.home)
4575 .await
4576 .is_err()
4577 );
4578 assert!(reviewer.exists());
4579 std::fs::remove_file(&evidence).unwrap();
4580 std::fs::create_dir_all(&evidence).unwrap();
4581 let oversized = evidence.join("capacity-fixture");
4582 std::fs::File::create(&oversized)
4583 .unwrap()
4584 .set_len(64 * 1024 * 1024)
4585 .unwrap();
4586 assert!(
4587 crate::graph::fold_run(&mut state, true, &f.home)
4588 .await
4589 .is_err()
4590 );
4591 assert!(reviewer.exists());
4592 std::fs::remove_file(oversized).unwrap();
4593 std::fs::create_dir_all(reviewer.join("scratch")).unwrap();
4596 std::fs::write(reviewer.join("scratch/recovery"), "keep").unwrap();
4597 assert!(
4598 crate::graph::fold_run(&mut state, true, &f.home)
4599 .await
4600 .is_err()
4601 );
4602 std::fs::remove_dir_all(reviewer.join("scratch")).unwrap();
4603 let checkpoint = git::git(&reviewer, &["rev-parse", "--git-path", "CHERRY_PICK_HEAD"])
4604 .await
4605 .unwrap();
4606 std::fs::write(&checkpoint, &head).unwrap();
4607 assert!(
4608 crate::graph::fold_run(&mut state, true, &f.home)
4609 .await
4610 .is_err()
4611 );
4612 std::fs::remove_file(checkpoint).unwrap();
4613 let (folded, unreadable) = crate::clean::fold_due(
4614 &f.home.join("runs"),
4615 &f.home,
4616 &f.bay,
4617 &Fx::disk(),
4618 Fx::now(),
4619 )
4620 .await
4621 .unwrap();
4622 if folded != 1 {
4623 panic!(
4624 "normal cleanup failed: {:#}",
4625 crate::graph::fold_run(&mut state, true, &f.home)
4626 .await
4627 .unwrap_err()
4628 );
4629 }
4630 assert_eq!((folded, unreadable), (1, 0));
4631 assert!(!reviewer.exists() && !state.candidates[0].worktree.exists());
4632 state = RunState::load_under(&state.id, &f.home).unwrap();
4633 assert!(state.candidates[0].folded);
4634 let manifest = std::fs::read(evidence.join("manifest.json")).unwrap();
4635 crate::graph::fold_run(&mut state, true, &f.home)
4636 .await
4637 .unwrap();
4638 assert_eq!(
4639 std::fs::read(evidence.join("manifest.json")).unwrap(),
4640 manifest
4641 );
4642 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
4643 .await
4644 .unwrap();
4645 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
4646 assert!(
4647 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
4648 .await
4649 .unwrap()
4650 .ok()
4651 );
4652 let manifest: serde_json::Value = serde_json::from_slice(&manifest).unwrap();
4655 let old_blob = manifest[format!("head-{head}")]["base_blobs"][0]
4656 .as_str()
4657 .unwrap();
4658 let recovered = f.home.join("recovered");
4659 std::fs::create_dir_all(&recovered).unwrap();
4660 git::git(&recovered, &["init"]).await.unwrap();
4661 std::fs::write(
4662 recovered.join("source"),
4663 std::fs::read(evidence.join(format!("blob-{old_blob}"))).unwrap(),
4664 )
4665 .unwrap();
4666 git::git(
4667 &recovered,
4668 &[
4669 "apply",
4670 evidence.join(format!("{head}.patch")).to_str().unwrap(),
4671 ],
4672 )
4673 .await
4674 .unwrap();
4675 assert_eq!(
4676 std::fs::read_to_string(recovered.join("source")).unwrap(),
4677 "cc05"
4678 );
4679 assert!(
4680 !std::fs::read(evidence.join(format!("{head}.patch")))
4681 .unwrap()
4682 .is_empty()
4683 );
4684 assert_eq!(
4685 std::fs::read_to_string(artifacts.join("review.json")).unwrap(),
4686 "review findings"
4687 );
4688 if missing {
4689 assert_eq!(
4690 std::fs::read_to_string(artifacts.join("cand-A.patch")).unwrap(),
4691 "original snapshot"
4692 );
4693 } else {
4694 assert!(!artifacts.join("cand-A.patch").exists());
4695 }
4696 assert!(run_dir.join("run.json").exists());
4697 }
4698 }
4699
4700 #[tokio::test]
4701 async fn slow_retirement_proof_allows_task_updates_and_aborts_on_new_owner_pin() {
4702 use std::os::unix::fs::PermissionsExt as _;
4703 let f = Fx::new().await;
4704 let source = f.run("dd03", true).await;
4705 let empty = f.run("dd04", false).await;
4706 let task = f.task(&[source, empty.clone()]);
4707 let entered = f.repo.join("proof-entered");
4708 let release = f.repo.join("proof-release");
4709 let hook = f.repo.join("proof-monitor");
4710 std::fs::write(&hook, format!(
4711 "#!/bin/sh\n: > '{}'\nwhile ! test -f '{}'; do sleep 0.01; done\nprintf 'probe\\0'\n",
4712 entered.display(), release.display())).unwrap();
4713 std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
4714 git::git(
4715 &f.repo,
4716 &["config", "core.fsmonitor", hook.to_str().unwrap()],
4717 )
4718 .await
4719 .unwrap();
4720 struct ReleaseProof(PathBuf);
4721 impl Drop for ReleaseProof {
4722 fn drop(&mut self) {
4723 let _ = std::fs::write(&self.0, "continue");
4724 }
4725 }
4726 let _release_on_panic = ReleaseProof(release.clone());
4727 let home = f.home.clone();
4728 let sweep = tokio::spawn(async move {
4729 let mut report = Report::default();
4730 retire_empty(&home, &Fx::disk(), Fx::now(), &mut report).await;
4731 report
4732 });
4733 let watchdog = std::time::Duration::from_secs(60);
4736 tokio::time::timeout(watchdog, async {
4737 while !entered.exists() {
4738 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
4739 }
4740 })
4741 .await
4742 .expect("Git retirement proof did not enter the blocking hook");
4743 let queue = crate::queue::Queue::at(f.home.join("queue"));
4744 let mut selected = queue.get(&task.id).unwrap();
4745 selected.resume_override = Some(crate::queue::OperatorResume {
4746 question_id: "during-slow-proof".into(),
4747 at: Fx::now(),
4748 conductor_rehold: None,
4749 forced: false,
4750 pinned_run: Some(empty.id.clone()),
4751 });
4752 let mut update = tokio::task::spawn_blocking(move || queue.put(&mut selected));
4753 let updated_while_blocked = match tokio::time::timeout(watchdog, &mut update).await {
4757 Ok(result) => {
4758 result.unwrap().unwrap();
4759 true
4760 }
4761 Err(_) => {
4762 std::fs::write(&release, "continue").unwrap();
4763 update.await.unwrap().unwrap();
4764 false
4765 }
4766 };
4767 std::fs::write(release, "continue").unwrap();
4768 let report = tokio::time::timeout(watchdog, sweep)
4769 .await
4770 .expect("Git retirement proof did not finish after release")
4771 .unwrap();
4772 assert!(
4773 updated_while_blocked,
4774 "task writers must not wait behind Git retirement proof"
4775 );
4776 assert_eq!(report.retired, 0);
4777 assert!(report.retained.iter().any(|r| r.contains("task changed")));
4778 assert!(empty.candidates[0].worktree.exists());
4779 assert!(
4780 RunState::load_under(&empty.id, &f.home)
4781 .unwrap()
4782 .retired_to
4783 .is_none()
4784 );
4785 }
4786
4787 #[tokio::test]
4788 async fn retirement_commit_rechecks_task_after_unlocked_proof_and_preserves_new_pin() {
4789 let f = Fx::new().await;
4790 let source = f.run("dd01", true).await;
4791 let mut empty = f.run("dd02", false).await;
4792 let expected = f.task(&[source.clone(), empty.clone()]);
4793 let queue = crate::queue::Queue::at(f.home.join("queue"));
4794 let _claim = queue.claim(&expected.id).unwrap();
4795 let _owner = crate::pr_head::owner_lock(&f.home, &empty.id).unwrap();
4797 let _driver = crate::driver::maintenance(&f.home, &empty.id).unwrap();
4798 let _outputs = OutputLease::take(&empty.candidates[0].worktree, true).unwrap();
4799 let mut selected = queue.get(&expected.id).unwrap();
4800 selected.resume_override = Some(crate::queue::OperatorResume {
4801 question_id: "new-owner-decision".into(),
4802 at: Fx::now(),
4803 conductor_rehold: None,
4804 forced: false,
4805 pinned_run: Some(empty.id.clone()),
4806 });
4807 queue.put(&mut selected).unwrap();
4808 let error = commit_retirement(
4809 &f.home,
4810 &queue,
4811 &expected,
4812 &mut empty,
4813 &source.id,
4814 &expected.runs,
4815 Fx::now(),
4816 )
4817 .unwrap_err();
4818 assert!(error.to_string().contains("task changed"));
4819 assert!(
4820 RunState::load_under(&empty.id, &f.home)
4821 .unwrap()
4822 .retired_to
4823 .is_none()
4824 );
4825 assert!(empty.candidates[0].worktree.exists());
4826 assert_eq!(
4827 queue
4828 .get(&expected.id)
4829 .unwrap()
4830 .resume_override
4831 .unwrap()
4832 .pinned_run,
4833 Some(empty.id.clone())
4834 );
4835 assert!(
4836 !f.home
4837 .join("recovery-lineages")
4838 .join(format!("{}.json", expected.id))
4839 .exists()
4840 );
4841 assert!(
4843 commit_retirement(
4844 &f.home,
4845 &queue,
4846 &selected,
4847 &mut empty,
4848 &source.id,
4849 &selected.runs,
4850 Fx::now()
4851 )
4852 .unwrap_err()
4853 .to_string()
4854 .contains("selected this recovery source")
4855 );
4856 }
4857
4858 #[tokio::test]
4859 async fn cargo_owners_and_driver_resumes_are_protected_with_retained_root_data() {
4860 let f = Fx::new().await;
4861 let state = f.run("cccc", false).await;
4862 let wt = &state.candidates[0].worktree;
4863 Fx::target(wt);
4864 std::fs::write(wt.join("target/notes"), "user file").unwrap();
4865 std::os::unix::fs::symlink(f.repo.join("source"), wt.join("target/debug/link")).unwrap();
4866 assert!(
4867 !declared_outputs(&wt.join("target"))
4868 .unwrap()
4869 .contains(&wt.join("target/debug/link"))
4870 );
4871 std::fs::remove_file(wt.join("target/debug/link")).unwrap();
4872 let cargo = cargo_locks(&wt.join("target")).unwrap();
4873 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4874 assert_eq!(out.freed_bytes, 0);
4875 assert!(out.retained.iter().any(|s| s.contains("Cargo build")));
4876 drop(cargo);
4877 let driver = crate::driver::maintenance(&f.home, &state.id).unwrap();
4878 let out = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4879 assert_eq!(out.freed_bytes, 0);
4880 drop(driver);
4881 let exclusive = OutputLease::take(wt, true).unwrap();
4882 assert!(
4883 OutputLease::borrow(wt).is_err(),
4884 "resume cannot start a writer during cleanup"
4885 );
4886 drop(exclusive);
4887 assert!(sweep(&f.home, &Fx::disk(), Fx::now()).await.freed_bytes > 0);
4888 assert_eq!(
4889 std::fs::read(wt.join("target/notes")).unwrap(),
4890 b"user file"
4891 );
4892 }
4893
4894 #[tokio::test]
4895 async fn root_leftovers_allow_eviction_but_keep_empty_attempts_registered() {
4896 for leftover in ["tmp", "magi-tools", "lifecycle-client-check.log"] {
4897 let f = Fx::new().await;
4898 let source = f.run("aa10", true).await;
4899 let empty = f.run("aa11", false).await;
4900 f.task(&[source, empty.clone()]);
4901 let wt = &empty.candidates[0].worktree;
4902 Fx::target(wt);
4903 let path = wt.join("target").join(leftover);
4904 if leftover.ends_with(".log") {
4905 std::fs::write(&path, "keep log\n").unwrap();
4906 } else {
4907 std::fs::create_dir(&path).unwrap();
4909 }
4910 let mut disk = Fx::disk();
4911 disk.auto_fold = true;
4912 let report = sweep(&f.home, &disk, Fx::now()).await;
4913 assert_eq!(report.freed_bytes, 200, "{report:?}");
4914 assert_eq!(report.retired, 0, "{report:?}");
4915 assert!(!wt.join("target/debug/output").exists());
4916 assert!(!residue_proof(&f.home, &empty.id, wt).unwrap().exists());
4917 assert!(path.exists());
4918 if leftover.ends_with(".log") {
4919 assert_eq!(std::fs::read(&path).unwrap(), b"keep log\n");
4920 }
4921 assert_eq!(std::fs::read(wt.join("source")).unwrap(), b"base\n");
4922 assert!(worktrees(&empty).await.unwrap().contains(wt));
4923 assert!(
4924 git::branch_exists(&f.repo, &empty.candidates[0].branch)
4925 .await
4926 .unwrap()
4927 );
4928 assert!(
4929 RunState::load_under(&empty.id, &f.home)
4930 .unwrap()
4931 .retired_to
4932 .is_none()
4933 );
4934 }
4935 }
4936
4937 #[tokio::test]
4938 async fn residue_proof_rejects_new_unknown_root_entries_including_empty_directories() {
4939 let f = Fx::new().await;
4940 let state = f.run("aa12", false).await;
4941 let wt = &state.candidates[0].worktree;
4942 Fx::target(wt);
4943 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4944 assert_eq!(report.freed_bytes, 200, "{report:?}");
4945 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4946 for name in ["tmp", "magi-tools", "log"] {
4947 let path = wt.join("target").join(name);
4948 std::fs::create_dir(&path).unwrap();
4949 assert!(verify_residue(&f.home, &state.id, wt).await.is_err());
4950 std::fs::remove_dir(path).unwrap();
4951 }
4952 assert!(verify_residue(&f.home, &state.id, wt).await.is_ok());
4953 }
4954
4955 #[tokio::test]
4956 async fn target_identity_and_known_profiles_still_fail_closed_with_root_leftovers() {
4957 for invalid in [
4958 "tag",
4959 "identity",
4960 "tracked",
4961 "profile-file",
4962 "profile-symlink",
4963 "identity-symlink",
4964 ] {
4965 let f = Fx::new().await;
4966 let state = f.run("aa13", false).await;
4967 let wt = &state.candidates[0].worktree;
4968 Fx::target(wt);
4969 std::fs::create_dir(wt.join("target/tmp")).unwrap();
4970 let target = wt.join("target");
4971 match invalid {
4972 "tag" => std::fs::write(target.join("CACHEDIR.TAG"), "not Cargo").unwrap(),
4973 "identity" => std::fs::write(target.join(".rustc_info.json"), "{}").unwrap(),
4974 "tracked" => {
4975 git::git(wt, &["add", "-f", "target/debug/output"])
4976 .await
4977 .unwrap();
4978 }
4979 "profile-file" => {
4980 std::fs::write(target.join("release"), "keep").unwrap();
4981 }
4982 "profile-symlink" => {
4983 std::os::unix::fs::symlink(target.join("debug"), target.join("release"))
4984 .unwrap();
4985 }
4986 "identity-symlink" => {
4987 std::fs::rename(target.join(".rustc_info.json"), target.join("saved-info"))
4988 .unwrap();
4989 std::os::unix::fs::symlink(
4990 target.join("saved-info"),
4991 target.join(".rustc_info.json"),
4992 )
4993 .unwrap();
4994 }
4995 _ => unreachable!(),
4996 }
4997 let before = std::fs::read(target.join("debug/output")).unwrap();
4998 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
4999 assert_eq!(report.freed_bytes, 0, "{invalid}: {report:?}");
5000 assert_eq!(std::fs::read(target.join("debug/output")).unwrap(), before);
5001 assert!(target.join("tmp").is_dir());
5002 }
5003 }
5004
5005 #[tokio::test]
5006 async fn reclaimed_cargo_residue_allows_retirement_only_while_exact_and_unowned() {
5007 let f = Fx::new().await;
5008 let source = f.run("0000", true).await;
5009 let empty = f.run("0001", false).await;
5010 let changed = f.run("0002", false).await;
5011 let unknown = f.run("0003", false).await;
5012 let unproven = f.run("0004", false).await;
5013 f.task(&[
5014 source.clone(),
5015 empty.clone(),
5016 changed.clone(),
5017 unknown.clone(),
5018 unproven.clone(),
5019 ]);
5020 for run in [&empty, &changed, &unknown] {
5021 Fx::target(&run.candidates[0].worktree);
5022 }
5023 let reviewer = empty.worktree_root().join("review-1");
5024 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
5025 .await
5026 .unwrap();
5027 Fx::target(&reviewer);
5028 let mut disk = Fx::disk();
5029 disk.auto_fold = false;
5030 let report = sweep(&f.home, &disk, Fx::now()).await;
5031 assert_eq!(report.freed_bytes, 800, "{report:?}");
5032 let wt = &empty.candidates[0].worktree;
5033 assert!(wt.join("target/debug/output.d").exists());
5034 assert!(!wt.join("target/debug/output").exists());
5035 let proof_path = residue_proof(&f.home, &empty.id, wt).unwrap();
5036 let proof: CargoResidue =
5037 serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap();
5038 assert_eq!(
5039 proof.files[Path::new("debug/output.d")],
5040 std::fs::read(wt.join("target/debug/output.d")).unwrap()
5041 );
5042 std::fs::write(
5043 changed.candidates[0].worktree.join("target/debug/output.d"),
5044 "owner notes",
5045 )
5046 .unwrap();
5047 std::fs::write(
5048 unknown.candidates[0].worktree.join("target/debug/notes"),
5049 "keep",
5050 )
5051 .unwrap();
5052 Fx::target(&unproven.candidates[0].worktree);
5053 std::fs::remove_file(unproven.candidates[0].worktree.join("target/debug/output")).unwrap();
5054 disk.auto_fold = true;
5055 let locks = cargo_locks(&wt.join("target")).unwrap();
5056 let mut report = Report::default();
5057 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
5058 assert_eq!(report.retired, 0, "{report:?}");
5059 assert!(
5060 report
5061 .retained
5062 .iter()
5063 .any(|reason| reason.contains("Cargo build"))
5064 );
5065 drop(locks);
5066 retire_empty(&f.home, &disk, Fx::now(), &mut report).await;
5067 assert_eq!(report.retired, 1, "{report:?}");
5068 assert!(!wt.exists());
5069 assert!(!reviewer.exists());
5070 assert!(
5071 residue_proof(&f.home, &empty.id, &reviewer)
5072 .unwrap()
5073 .exists()
5074 );
5075 assert!(
5076 proof_path.exists(),
5077 "metadata evidence outlives the worktree"
5078 );
5079 assert_eq!(
5080 RunState::load_under(&empty.id, &f.home).unwrap().retired_to,
5081 Some(source.id)
5082 );
5083 for run in [&changed, &unknown, &unproven] {
5084 assert!(run.candidates[0].worktree.exists());
5085 assert!(
5086 RunState::load_under(&run.id, &f.home)
5087 .unwrap()
5088 .retired_to
5089 .is_none()
5090 );
5091 }
5092 }
5093
5094 #[tokio::test]
5095 async fn empty_backlog_retires_to_useful_older_source_in_bounded_idempotent_passes() {
5096 let f = Fx::new().await;
5097 let source = f.run("0000", true).await;
5098 let source_head = git::rev_parse(&f.repo, &source.candidates[0].branch)
5099 .await
5100 .unwrap();
5101 let mut runs = vec![source.clone()];
5102 for n in 1..=12 {
5103 runs.push(f.run(&format!("{n:04}"), false).await);
5104 }
5105 let dirty = runs[1].candidates[0].worktree.clone();
5106 std::fs::write(dirty.join("unrecorded"), "preparation changes").unwrap();
5107 let ignored = runs[2].candidates[0].worktree.clone();
5108 std::fs::create_dir(ignored.join("scratch")).unwrap();
5109 std::fs::write(ignored.join("scratch/note"), "keep").unwrap();
5110 runs[1].candidates.clear();
5113 runs[1].save_under(&f.home).unwrap();
5114 let task = f.task(&runs);
5115 let mut total = 0;
5116 for _ in 0..4 {
5117 let mut report = Report::default();
5118 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5119 assert!(report.retired <= 5);
5120 total += report.retired;
5121 }
5122 assert_eq!(total, 10);
5123 assert!(dirty.exists());
5124 assert!(ignored.exists());
5125 assert_eq!(
5126 git::rev_parse(&f.repo, &source.candidates[0].branch)
5127 .await
5128 .unwrap(),
5129 source_head
5130 );
5131 let retired = RunState::load_under(&runs[3].id, &f.home).unwrap();
5132 assert_eq!(retired.retired_to.as_ref(), Some(&source.id));
5133 assert!(crate::pr_head::ensure_owner_record(&retired, Some(&retired)).is_err());
5134 let mut released = task.clone();
5135 released.release();
5136 assert_eq!(
5137 crate::daemon::unfinished_run_with(&released.runs, released.short(), |id| {
5138 RunState::load_under(id, &f.home)
5139 }),
5140 Some(source.id.clone())
5141 );
5142
5143 assert!(
5144 f.home
5145 .join("runs")
5146 .join(&retired.id)
5147 .join("run.json")
5148 .exists()
5149 );
5150 assert!(
5151 f.home
5152 .join("recovery-lineages")
5153 .join(format!("{}.json", task.id))
5154 .exists()
5155 );
5156 }
5157
5158 #[tokio::test]
5159 async fn distinct_candidates_and_explicitly_retained_sources_are_never_discarded() {
5160 let f = Fx::new().await;
5161 let a = f.run("dddd", true).await;
5162 let b = f.run("eeee", true).await;
5163 let empty = f.run("ffff", false).await;
5164 let task = f.task(&[a.clone(), b.clone(), empty.clone()]);
5165 assert!(
5166 crate::pr_recovery::source_retention_reason(&f.home, &a).is_some(),
5167 "unfinished queue history protects older source branches from ordinary folding"
5168 );
5169 let mut prior = a.clone();
5170 assert!(
5171 crate::graph::fold_run(&mut prior, true, &f.home)
5172 .await
5173 .unwrap_err()
5174 .to_string()
5175 .contains(&task.id)
5176 );
5177 let mut report = Report::default();
5178 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5179 assert_eq!(report.retired, 0);
5180 assert!(report.retained.iter().any(|r| r.contains("ambiguous")));
5181 crate::pr_recovery::retain_sources(&task, &f.home).unwrap();
5182 Fx::target(&a.candidates[0].worktree);
5183 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
5184 assert!(
5185 report.freed_bytes > 0,
5186 "source retention does not retain regenerable outputs"
5187 );
5188 assert!(a.candidates[0].worktree.exists());
5189 assert!(b.candidates[0].worktree.exists());
5190 assert!(empty.candidates[0].worktree.exists());
5191 }
5192
5193 #[tokio::test]
5194 async fn decisions_checkpoints_and_partial_retirement_remain_safe() {
5195 let f = Fx::new().await;
5196 let authority = f.run("p000", true).await;
5197 let question_run = f.run("p001", false).await;
5198 let mut checkpoint = f.run("p002", false).await;
5199 checkpoint.land_approval = Some(crate::run::LandApproval {
5200 question: "pending".into(),
5201 head: f.base.clone(),
5202 });
5203 checkpoint.save_under(&f.home).unwrap();
5204 let retained = f.run("p003", false).await;
5205 std::fs::write(
5206 f.home
5207 .join("runs")
5208 .join(&retained.id)
5209 .join("retain-recovery-sources"),
5210 "owner",
5211 )
5212 .unwrap();
5213 let mut partial = f.run("p004", false).await;
5214 let reviewer = partial.worktree_root().join("review-1");
5215 git::worktree_add_detached(&f.repo, &reviewer, &f.base)
5216 .await
5217 .unwrap();
5218 partial.retired_to = Some(authority.id.clone());
5220 partial.save_under(&f.home).unwrap();
5221 assert!(
5222 git::worktree_remove_clean(&f.repo, &partial.candidates[0].worktree)
5223 .await
5224 .unwrap()
5225 );
5226 let git_checkpoint = f.run("p005", false).await;
5227 let location = git::git(
5228 &git_checkpoint.candidates[0].worktree,
5229 &[
5230 "rev-parse",
5231 "--path-format=absolute",
5232 "--git-path",
5233 "sequencer",
5234 ],
5235 )
5236 .await
5237 .unwrap();
5238 std::fs::create_dir(location.trim()).unwrap();
5239 let mut preservation = f.run("p006", false).await;
5240 preservation
5241 .candidate_preservation
5242 .insert('A', "pending preservation".into());
5243 preservation.save_under(&f.home).unwrap();
5244 let task = f.task(&[
5245 preservation.clone(),
5246 git_checkpoint.clone(),
5247 authority.clone(),
5248 question_run.clone(),
5249 checkpoint.clone(),
5250 retained.clone(),
5251 partial.clone(),
5252 ]);
5253 let questions = crate::ask::Questions::at(f.home.join("questions"));
5254 let mut question = crate::ask::Question::new(
5255 question_run.id.clone(),
5256 "implement".into(),
5257 "fixture".into(),
5258 "owner decision".into(),
5259 String::new(),
5260 vec!["keep".into()],
5261 );
5262 questions.put(&mut question).unwrap();
5263 let mut report = Report::default();
5264 retire_empty(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5265 assert_eq!(report.retired, 1, "{report:?}");
5266 assert!(!reviewer.exists());
5267 for state in [
5268 &question_run,
5269 &checkpoint,
5270 &retained,
5271 &git_checkpoint,
5272 &preservation,
5273 ] {
5274 assert!(state.candidates[0].worktree.exists());
5275 assert!(
5276 RunState::load_under(&state.id, &f.home)
5277 .unwrap()
5278 .retired_to
5279 .is_none()
5280 );
5281 }
5282 assert!(questions.get(&question.id).unwrap().status.open());
5283 assert_eq!(
5284 crate::queue::Queue::at(f.home.join("queue"))
5285 .get(&task.id)
5286 .unwrap()
5287 .status,
5288 task.status
5289 );
5290 }
5291
5292 #[tokio::test]
5293 async fn preservation_schema_migration_remains_distinct_from_retirement() {
5294 let f = Fx::new().await;
5295 let mut state = f.run("p007", false).await;
5296 state.schema = 15;
5297 state.save_under(&f.home).unwrap();
5298 let native = RunState::load_under(&state.id, &f.home).unwrap();
5299 assert_eq!(native.schema, crate::run::SCHEMA);
5300 assert!(native.candidate_preservation.is_empty());
5301 assert!(native.retired_to.is_none());
5302 state.schema = 14;
5303 state.save_under(&f.home).unwrap();
5304 let mut legacy = RunState::load_under(&state.id, &f.home).unwrap();
5305 assert!(legacy.candidate_preservation.contains_key(&'A'));
5306 legacy.save_under(&f.home).unwrap();
5307 let restored = RunState::load_under(&state.id, &f.home).unwrap();
5308 assert_eq!(
5309 restored.candidate_preservation,
5310 legacy.candidate_preservation
5311 );
5312 }
5313
5314 #[tokio::test]
5315 async fn cache_cleanup_excludes_process_cleanup_and_missing_path_aliases() {
5316 let f = Fx::new().await;
5317 let cache = f.repo.join("cache");
5318 let guard = OutputLease::cache(&f.home, &cache, false).unwrap();
5319 std::fs::create_dir_all(&cache).unwrap();
5320 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
5321 assert!(
5322 crate::cache::maintenance_prune(&f.home, &cache, 1)
5323 .unwrap()
5324 .is_none()
5325 );
5326 drop(guard);
5327 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
5328 }
5329
5330 #[tokio::test]
5331 async fn launcher_death_does_not_release_live_tool_output_reservations() {
5332 let f = Fx::new().await;
5333 let run = f.run("abcd", false).await;
5334 let wt = &run.candidates[0].worktree;
5335 let mut helper = tokio::process::Command::new(std::env::current_exe().unwrap());
5336 helper
5337 .quiet()
5338 .args(["--exact", "seat_process::tests::crashed_launcher_helper"])
5339 .env("MAGI_TEST_CRASHED_LAUNCHER", &f.home)
5340 .env("MAGI_TEST_WORKTREE", wt)
5341 .stdin(std::process::Stdio::null())
5342 .stdout(std::process::Stdio::null())
5343 .stderr(std::process::Stdio::null())
5344 .kill_on_drop(true);
5345 assert!(
5346 tokio::time::timeout(std::time::Duration::from_secs(8), helper.status())
5347 .await
5348 .unwrap()
5349 .unwrap()
5350 .success()
5351 );
5352 let pid: i32 = std::fs::read_to_string(f.home.join("tool.pid"))
5353 .unwrap()
5354 .trim()
5355 .parse()
5356 .unwrap();
5357 struct KillTool(i32);
5358 impl Drop for KillTool {
5359 fn drop(&mut self) {
5360 unsafe { libc::kill(self.0, libc::SIGKILL) };
5362 }
5363 }
5364 let tool = KillTool(pid);
5365 assert!(OutputLease::take(wt, true).is_err());
5366 assert!(OutputLease::cache(&f.home, &f.home.join("cache"), true).is_err());
5367 drop(tool);
5368 tokio::time::timeout(std::time::Duration::from_secs(5), async {
5369 loop {
5370 if OutputLease::take(wt, true).is_ok()
5371 && OutputLease::cache(&f.home, &f.home.join("cache"), true).is_ok()
5372 {
5373 break;
5374 }
5375 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5376 }
5377 })
5378 .await
5379 .unwrap();
5380 }
5381
5382 #[tokio::test]
5383 async fn orphaned_tool_keeps_output_and_cache_leases_after_cli_exit() {
5384 let f = Fx::new().await;
5385 let run = f.run("abcd", false).await;
5386 let wt = &run.candidates[0].worktree;
5387 let cache = f.repo.join("cache");
5388 std::fs::create_dir(&cache).unwrap();
5389 for (cancel, clear_owner) in [(false, true), (true, true), (false, false), (true, false)] {
5390 let pid_file = f.repo.join(format!("orphan-{cancel}-{clear_owner}"));
5391 let mut cmd = tokio::process::Command::new(std::env::current_exe().unwrap());
5392 cmd.quiet()
5393 .args(["--exact", "seat_process::tests::orphan_tool_helper"])
5394 .env("MAGI_TEST_ORPHAN_PID", &pid_file)
5395 .stdin(std::process::Stdio::null())
5396 .stdout(std::process::Stdio::null())
5397 .stderr(std::process::Stdio::null())
5398 .kill_on_drop(true);
5399 if clear_owner {
5400 cmd.env("MAGI_TEST_CLEAR_OWNER", "1");
5401 } else {
5402 cmd.env("MAGI_TEST_CLOSE_FDS", "1");
5403 }
5404 let gib = 1024 * 1024 * 1024;
5405 let mut leases = vec![
5406 OutputLease::borrow(wt).unwrap().unwrap(),
5407 OutputLease::cache(&f.home, &cache, false).unwrap(),
5408 ];
5409 leases.extend(
5410 OutputLease::admit_with(
5411 &f.home,
5412 std::slice::from_ref(wt),
5413 8 * gib,
5414 |_| Ok(9 * gib),
5415 )
5416 .unwrap(),
5417 );
5418 let mut running = crate::seat_process::Running::spawn(cmd, None, leases).unwrap();
5419 assert!(
5420 tokio::time::timeout(std::time::Duration::from_secs(5), running.child().wait())
5421 .await
5422 .unwrap()
5423 .unwrap()
5424 .success()
5425 );
5426 let pid: i32 = std::fs::read_to_string(pid_file).unwrap().parse().unwrap();
5427 struct KillTool(Option<i32>);
5428 impl Drop for KillTool {
5429 fn drop(&mut self) {
5430 if let Some(pid) = self.0 {
5432 unsafe { libc::kill(pid, libc::SIGKILL) };
5433 }
5434 }
5435 }
5436 let mut tool = KillTool(Some(pid));
5437 assert!(OutputLease::take(wt, true).is_err());
5441 let finish = if cancel {
5442 drop(running);
5443 None
5444 } else {
5445 Some(tokio::spawn(async move { running.finish().await }))
5446 };
5447 if clear_owner {
5448 tokio::time::sleep(std::time::Duration::from_millis(150)).await;
5449 assert!(crate::proc::pid_alive(pid as u32));
5450 assert!(OutputLease::take(wt, true).is_err(), "cancel={cancel}");
5451 assert!(OutputLease::cache(&f.home, &cache, true).is_err());
5452 assert!(
5453 OutputLease::admit_with(&f.home, std::slice::from_ref(wt), 8 * gib, |_| Ok(
5454 9 * gib
5455 ))
5456 .is_err(),
5457 "orphan still owns its disk reservation"
5458 );
5459 assert!(
5460 crate::cache::maintenance_prune(&f.home, &cache, 0)
5461 .unwrap()
5462 .is_none()
5463 );
5464 if let Some(finish) = &finish {
5465 assert!(!finish.is_finished());
5466 }
5467 unsafe { libc::kill(pid, libc::SIGKILL) };
5469 }
5470 if let Some(finish) = finish {
5471 tokio::time::timeout(std::time::Duration::from_secs(5), finish)
5472 .await
5473 .unwrap()
5474 .unwrap();
5475 }
5476 tokio::time::timeout(std::time::Duration::from_secs(5), async {
5477 loop {
5478 if OutputLease::take(wt, true).is_ok()
5479 && OutputLease::cache(&f.home, &cache, true).is_ok()
5480 && OutputLease::admit_with(
5481 &f.home,
5482 std::slice::from_ref(wt),
5483 8 * gib,
5484 |_| Ok(9 * gib),
5485 )
5486 .is_ok()
5487 {
5488 if !clear_owner {
5490 assert!(!crate::proc::pid_alive(pid as u32));
5491 }
5492 break;
5493 }
5494 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5495 }
5496 })
5497 .await
5498 .unwrap();
5499 tool.0 = None;
5500 }
5501 }
5502
5503 #[tokio::test]
5504 async fn cancellation_keeps_output_and_cache_ownership_until_tools_terminate() {
5505 let f = Fx::new().await;
5506 let run = f.run("abcd", false).await;
5507 let wt = &run.candidates[0].worktree;
5508 let cache = f.repo.join("cache");
5509 std::fs::create_dir(&cache).unwrap();
5510 let pid_file = f.repo.join("child-pid");
5511 let mut cmd = tokio::process::Command::new("sh");
5512 cmd.args(["-c", "sleep 30 & echo $! > \"$CHILD_PID_FILE\"; wait"])
5513 .quiet()
5514 .env("CHILD_PID_FILE", &pid_file)
5515 .process_group(0)
5516 .kill_on_drop(true);
5517 let running = crate::seat_process::Running::spawn(
5518 cmd,
5519 None,
5520 vec![
5521 OutputLease::borrow(wt).unwrap().unwrap(),
5522 OutputLease::cache(&f.home, &cache, false).unwrap(),
5523 ],
5524 )
5525 .unwrap();
5526 tokio::time::timeout(std::time::Duration::from_secs(5), async {
5527 while !pid_file.exists() {
5528 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5529 }
5530 })
5531 .await
5532 .unwrap();
5533 assert!(OutputLease::take(wt, true).is_err());
5534 assert!(
5535 crate::cache::maintenance_prune(&f.home, &cache, 0)
5536 .unwrap()
5537 .is_none()
5538 );
5539 let pid: u32 = std::fs::read_to_string(pid_file)
5540 .unwrap()
5541 .trim()
5542 .parse()
5543 .unwrap();
5544 drop(running);
5545 tokio::time::timeout(std::time::Duration::from_secs(10), async {
5546 loop {
5547 if let Ok(guard) = OutputLease::take(wt, true) {
5548 assert!(
5549 !crate::proc::pid_alive(pid),
5550 "output lock released before tool death"
5551 );
5552 drop(guard);
5553 break;
5554 }
5555 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
5556 }
5557 })
5558 .await
5559 .unwrap();
5560 assert!(OutputLease::cache(&f.home, &cache, true).is_ok());
5561 }
5562
5563 #[tokio::test]
5564 async fn emergency_pressure_closes_writes_and_zero_floor_opts_out() {
5565 let f = Fx::new().await;
5566 let reason = tokio::time::timeout(
5567 std::time::Duration::from_secs(2),
5568 emergency(vec![f.repo.clone()], u64::MAX, false),
5569 )
5570 .await
5571 .unwrap();
5572 assert!(reason.contains("disk pressure"));
5573 assert!(
5574 tokio::time::timeout(
5575 std::time::Duration::from_millis(10),
5576 emergency(vec![f.repo.clone()], 0, false)
5577 )
5578 .await
5579 .is_err()
5580 );
5581 assert_eq!(
5582 std::fs::read_to_string(f.repo.join("source")).unwrap(),
5583 "base\n"
5584 );
5585 }
5586
5587 #[tokio::test]
5588 async fn automatic_requeues_cannot_refund_failed_input_generation() {
5589 let f = Fx::new().await;
5590 let mut run = f.run("1234", false).await;
5591 run.prep_failure = Some("cherry-pick conflict".into());
5592 let mut task = f.task(std::slice::from_ref(&run));
5593 let inputs = crate::queue::recovery_inputs(&run.sources);
5594 for _ in 0..108 {
5595 task.recovery_failed(&run);
5596 task.requeue();
5597 assert_eq!(task.attempts, 0);
5598 }
5599 let bytes = serde_json::to_vec(&task).unwrap();
5600 let mut restored: Task = serde_json::from_slice(&bytes).unwrap();
5601 assert!(
5602 restored
5603 .recovery_exhausted(&run.base_commit, &inputs, 2)
5604 .is_some()
5605 );
5606 assert!(
5607 restored
5608 .recovery_exhausted("different-base", &inputs, 2)
5609 .is_none()
5610 );
5611 let original_base = run.base_commit.clone();
5612 run.base_commit = "different-base".into();
5613 restored.recovery_failed(&run);
5614 assert!(
5615 restored
5616 .recovery_exhausted(&original_base, &inputs, 2)
5617 .is_some(),
5618 "returning to old inputs does not replenish their budget"
5619 );
5620 assert!(
5621 restored
5622 .recovery_exhausted(&run.base_commit, &inputs, 2)
5623 .is_none()
5624 );
5625 restored.release_operator("test-owner", None);
5626 assert!(
5627 restored
5628 .recovery_exhausted(&run.base_commit, &inputs, 2)
5629 .is_none()
5630 );
5631 }
5632 async fn inactive_source(f: &Fx, tag: &str, useful: bool) -> RunState {
5633 let mut state = f.run(tag, useful).await;
5634 state.status = RunStatus::Superseded;
5635 state.save_under(&f.home).unwrap();
5636 state
5637 }
5638
5639 fn historical_review(state: &mut RunState, head: &str) {
5640 state.reviews.push(
5641 serde_json::from_value(serde_json::json!({
5642 "round": state.reviews.len() + 1, "head": head, "reviews": []
5643 }))
5644 .unwrap(),
5645 );
5646 }
5647
5648 async fn reviewer(f: &Fx, state: &RunState, head: &str) -> PathBuf {
5649 let path = state.worktree_root().join("review-1");
5650 git::worktree_add_detached(&f.repo, &path, head)
5651 .await
5652 .unwrap();
5653 path
5654 }
5655
5656 #[tokio::test]
5657 async fn source_eviction_separates_run_authority_from_read_seats() {
5658 for status in [RunStatus::Superseded, RunStatus::Ready, RunStatus::Merged] {
5659 let f = Fx::new().await;
5660 let mut state = inactive_source(&f, "ef01", true).await;
5661 state.status = status;
5662 state.pr = Some(
5663 serde_json::from_value(serde_json::json!({
5664 "url":"https://example.invalid/pull/1", "number":1, "state":"merged",
5665 "checks":"green", "round":1, "rounds":1
5666 }))
5667 .unwrap(),
5668 );
5669 state.pr_head = Some(state.candidates[0].branch.clone());
5670 state.pr_merged_head = Some(f.base.clone());
5671 state.recovery = Some(
5672 serde_json::from_value(serde_json::json!({
5673 "from_run": "20260101-000000-abcd", "generation": 1,
5674 "mode": "full-review"
5675 }))
5676 .unwrap(),
5677 );
5678 state.pr_recovery = Some(
5679 serde_json::from_value(serde_json::json!({
5680 "url": "https://example.invalid/pull/1", "branch": "public",
5681 "head": f.base, "source_run": state.id,
5682 "instruction": "keep implementation", "approved_at": Fx::now()
5683 }))
5684 .unwrap(),
5685 );
5686 state.save_under(&f.home).unwrap();
5687 let review = reviewer(&f, &state, &f.base).await;
5688 let mut other = inactive_source(&f, "ef02", true).await;
5689 other.pr_recovery = state.pr_recovery.clone();
5690 other.save_under(&f.home).unwrap();
5691 let mut task = f.task(&[state.clone(), other]);
5692 task.pr_recovery = state.pr_recovery.clone();
5693 crate::queue::Queue::at(f.home.join("queue"))
5694 .put(&mut task)
5695 .unwrap();
5696 let run_path = f.home.join("runs").join(&state.id).join("run.json");
5697 let before = std::fs::read(&run_path).unwrap();
5698 let task_before = serde_json::to_value(&task).unwrap();
5699 let mut report = Report::default();
5700 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5701 assert_eq!(report.evicted_copies, 1, "{status:?}: {report:?}");
5702 assert!(!review.exists());
5703 assert!(state.candidates[0].worktree.exists());
5704 assert!(
5705 git::branch_exists(&f.repo, &state.candidates[0].branch)
5706 .await
5707 .unwrap()
5708 );
5709 assert_eq!(std::fs::read(&run_path).unwrap(), before);
5710 assert_eq!(
5711 serde_json::to_value(
5712 crate::queue::Queue::at(f.home.join("queue"))
5713 .get(&task.id)
5714 .unwrap()
5715 )
5716 .unwrap(),
5717 task_before
5718 );
5719 }
5720 }
5721
5722 #[tokio::test]
5723 async fn source_eviction_retains_own_paths_and_unfinished_continuations() {
5724 for kind in ["own-path", "active-seat", "paused-session"] {
5725 let f = Fx::new().await;
5726 let mut state = inactive_source(&f, "ef03", true).await;
5727 let review = reviewer(&f, &state, &f.base).await;
5728 let expected = match kind {
5729 "own-path" => {
5730 state.attachments.push(review.join("source"));
5731 "run.attachments[0]"
5732 }
5733 "active-seat" => {
5734 state.active.insert(
5735 "review-1".into(),
5736 serde_json::from_value(serde_json::json!({
5737 "node":"review", "started_at":Fx::now(), "timeout_secs":60
5738 }))
5739 .unwrap(),
5740 );
5741 "continuation lease"
5742 }
5743 _ => {
5744 state.status = RunStatus::Blocked;
5745 state.parked = true;
5746 let mut seat = crate::agent::SeatState::new("review-1", "fixture", 1);
5747 seat.turns = 2;
5748 seat.captured_session = Some("saved-session".into());
5749 state.seats.insert("review-1".into(), seat);
5750 "resume contract"
5751 }
5752 };
5753 state.save_under(&f.home).unwrap();
5754 let mut report = Report::default();
5755 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5756 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5757 assert!(review.exists());
5758 assert!(
5759 report.retained.iter().any(|r| r.contains(expected)),
5760 "{kind}: {report:?}"
5761 );
5762 restore_execution_seats(&f.home, &state, None)
5765 .await
5766 .unwrap();
5767 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), f.base);
5768 }
5769 }
5770
5771 #[tokio::test]
5772 async fn source_eviction_distinguishes_historical_cwd_from_question_dependencies() {
5773 for kind in ["waiting", "lease", "historical", "unknown"] {
5774 let f = Fx::new().await;
5775 let state = inactive_source(&f, "ef06", true).await;
5776 let review = reviewer(&f, &state, &f.base).await;
5777 let store = crate::ask::Questions::at(f.home.join("questions"));
5778 let mut question = crate::ask::Question::new(
5779 "20260101-000000-ffff".into(),
5780 "review".into(),
5781 "review-1".into(),
5782 "fixture".into(),
5783 "fixture".into(),
5784 Vec::new(),
5785 );
5786 question.cwd = Some(review.to_string_lossy().into_owned());
5787 if kind != "waiting" {
5788 question.status = crate::ask::QuestionStatus::Abandoned;
5789 }
5790 store.put(&mut question).unwrap();
5791 if kind == "lease" {
5792 store.beat(&question.id, crate::ask::WaiterKind::Daemon);
5793 }
5794 if kind == "unknown" {
5795 let path = store.path_of(&question.id);
5796 let mut raw: serde_json::Value =
5797 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
5798 raw["future_cwd_authority"] = serde_json::Value::Null;
5799 std::fs::write(path, serde_json::to_vec(&raw).unwrap()).unwrap();
5800 }
5801 let mut report = Report::default();
5802 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5803 assert_eq!(
5804 report.evicted_copies,
5805 usize::from(kind == "historical"),
5806 "{kind}: {report:?}"
5807 );
5808 assert_eq!(review.exists(), kind != "historical");
5809 if kind == "waiting" || kind == "lease" {
5810 assert!(
5811 report
5812 .retained
5813 .iter()
5814 .any(|r| r.contains("question") && r.contains("cwd")),
5815 "{report:?}"
5816 );
5817 }
5818 }
5819 }
5820
5821 #[tokio::test]
5822 async fn archived_seat_setup_reuses_repository_coordination() {
5823 crate::run::pin_test_home();
5824 let f = Fx::new().await;
5825 let mut state = inactive_source(&f, "ef07", true).await;
5826 let review = reviewer(&f, &state, &f.base).await;
5827 let mut report = Report::default();
5828 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5829 assert_eq!(report.evicted_copies, 1, "{report:?}");
5830 git::acquire_worktree_config(&f.repo).await.unwrap();
5831 state.config.verify.build_cache = crate::config::BuildCache::Isolated;
5832 state.config.verify.shell = Some(vec!["sh".into(), "-c".into()]);
5833 state.config.disk.min_free_bytes = 0;
5834 state.config.worktree.setup = vec![crate::config::SetupStep {
5835 run: Some("touch setup-completed".into()),
5836 ..Default::default()
5837 }];
5838 std::fs::write(f.repo.join(".git/fixture-contract"), "fixture-v1").unwrap();
5839 state.config.coordination = Some(crate::config::Coordination {
5840 contract_file: "fixture-contract".into(),
5841 contract_token: "fixture-v1".into(),
5842 lock_file: "fixture-lock".into(),
5843 });
5844 let timeout = std::time::Duration::from_secs(15);
5845 let wait = crate::repo_lock::WaitPolicy::new(timeout, || false);
5846 let lease = crate::repo_lock::SeatLease::operation_with(
5847 &f.repo,
5848 state.config.coordination.as_ref(),
5849 &wait,
5850 )
5851 .await
5852 .unwrap()
5853 .unwrap();
5854 for _ in 0..2 {
5855 tokio::time::timeout(
5856 timeout,
5857 restore_execution_seats(&f.home, &state, Some(&lease)),
5858 )
5859 .await
5860 .expect("restoration must not wait for its own repository operation")
5861 .unwrap();
5862 }
5863 assert!(review.join("setup-completed").is_file());
5864 }
5865
5866 #[tokio::test]
5867 async fn source_eviction_failed_execution_setup_keeps_its_checkpoint() {
5868 let f = Fx::new().await;
5869 let mut state = inactive_source(&f, "ef07", true).await;
5870 let review = reviewer(&f, &state, &f.base).await;
5871 let mut report = Report::default();
5872 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5873 assert_eq!(report.evicted_copies, 1, "{report:?}");
5874 state.config.worktree.setup = vec![crate::config::SetupStep {
5875 run: Some("printf preserved > source; exit 1".into()),
5876 ..Default::default()
5877 }];
5878 state.save_under(&f.home).unwrap();
5879 assert!(
5880 restore_execution_seats(&f.home, &state, None)
5881 .await
5882 .is_err()
5883 );
5884 let marker = f
5885 .home
5886 .join("runs")
5887 .join(&state.id)
5888 .join("artifacts/restore-review-1.pending");
5889 assert!(marker.exists());
5890 assert_eq!(
5891 std::fs::read_to_string(review.join("source")).unwrap(),
5892 "preserved"
5893 );
5894 assert!(
5895 restore_execution_seats(&f.home, &state, None)
5896 .await
5897 .unwrap_err()
5898 .to_string()
5899 .contains("interrupted execution restoration")
5900 );
5901 let mut retry = Report::default();
5902 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
5903 assert_eq!(retry.evicted_copies, 0, "{retry:?}");
5904 assert!(
5905 retry
5906 .retained
5907 .iter()
5908 .any(|r| r.contains("restoration checkpoint")),
5909 "{retry:?}"
5910 );
5911 assert!(marker.exists());
5912 assert_eq!(
5913 std::fs::read_to_string(review.join("source")).unwrap(),
5914 "preserved"
5915 );
5916 }
5917
5918 #[tokio::test]
5919 async fn source_eviction_retains_borrowed_or_promisor_anchor_stores() {
5920 for kind in [
5921 "alternates",
5922 "promisor-config",
5923 "promisor-pack",
5924 "objects-symlink",
5925 ] {
5926 let f = Fx::new().await;
5927 let state = inactive_source(&f, "ef11", true).await;
5928 let review = reviewer(&f, &state, &f.base).await;
5929 let common = crate::repo_lock::common_dir(&f.repo)
5930 .await
5931 .unwrap()
5932 .unwrap();
5933 match kind {
5934 "alternates" => {
5935 let borrowed = f._temp.path().join("borrowed-objects");
5936 std::fs::create_dir(&borrowed).unwrap();
5937 std::fs::write(
5938 common.join("objects/info/alternates"),
5939 format!("{}\n", borrowed.display()),
5940 )
5941 .unwrap();
5942 }
5943 "promisor-config" => {
5944 git::git(&f.repo, &["config", "remote.origin.promisor", "true"])
5945 .await
5946 .unwrap();
5947 }
5948 "promisor-pack" => {
5949 std::fs::write(common.join("objects/pack/fixture.promisor"), "").unwrap();
5950 }
5951 _ => {
5952 let borrowed = f._temp.path().join("borrowed-objects");
5953 std::fs::rename(common.join("objects"), &borrowed).unwrap();
5954 std::os::unix::fs::symlink(borrowed, common.join("objects")).unwrap();
5955 }
5956 }
5957 let mut report = Report::default();
5958 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5959 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
5960 assert!(review.exists());
5961 assert!(
5962 report.retained.iter().any(|r| r.contains("source anchor")),
5963 "{kind}: {report:?}"
5964 );
5965 }
5966 }
5967
5968 #[tokio::test]
5969 async fn source_eviction_bounds_long_shared_history_and_requires_its_retained_anchor() {
5970 let mut f = Fx::new().await;
5971 crate::fold_evidence::fixtures::shared_history(&f.repo, &f.base, 5000).await;
5972 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
5973 assert_eq!(
5974 git::git(&f.repo, &["rev-list", "--count", &f.base])
5975 .await
5976 .unwrap(),
5977 "5001"
5978 );
5979 let state = inactive_source(&f, "ef10", true).await;
5980 let review = reviewer(&f, &state, &f.base).await;
5981 std::fs::write(review.join("source"), "bounded review delta").unwrap();
5982 git::git(&review, &["add", "."]).await.unwrap();
5983 let tree = git::git(&review, &["write-tree"]).await.unwrap();
5984 let head = git::git(
5985 &review,
5986 &["commit-tree", &tree, "-p", &f.base, "-m", "review delta"],
5987 )
5988 .await
5989 .unwrap();
5990 git::git(&review, &["checkout", "--detach", &head])
5991 .await
5992 .unwrap();
5993 let run_file = f.home.join("runs").join(&state.id).join("run.json");
5994 let run_before = std::fs::read(&run_file).unwrap();
5995 let mut report = Report::default();
5996 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
5997 assert_eq!(report.evicted_copies, 1, "{report:?}");
5998 assert!(!review.exists());
5999 assert!(state.candidates[0].worktree.exists());
6000 assert!(
6001 git::branch_exists(&f.repo, &state.candidates[0].branch)
6002 .await
6003 .unwrap()
6004 );
6005 assert_eq!(std::fs::read(&run_file).unwrap(), run_before);
6006 let archive = f
6007 .home
6008 .join("runs")
6009 .join(&state.id)
6010 .join("artifacts/source-evidence")
6011 .join(format!("{head}-{}", f.base));
6012 let manifest: serde_json::Value =
6013 serde_json::from_slice(&std::fs::read(archive.join("manifest.json")).unwrap()).unwrap();
6014 assert_eq!(manifest["anchor"], f.base);
6015 assert!(
6016 std::fs::metadata(archive.join("objects.pack"))
6017 .unwrap()
6018 .len()
6019 < 16 * 1024
6020 );
6021 let pin = format!("refs/magi/source-anchors/{}", f.base);
6022 assert_eq!(git::rev_parse(&f.repo, &pin).await.unwrap(), f.base);
6023 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6024 .await
6025 .unwrap();
6026 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6027 assert!(
6028 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
6029 .await
6030 .unwrap()
6031 .ok()
6032 );
6033 let other = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6034 .await
6035 .unwrap();
6036 for kind in ["missing", "moved", "symbolic", "symlink"] {
6038 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
6039 .await
6040 .unwrap();
6041 match kind {
6042 "moved" => {
6043 git::git(&f.repo, &["update-ref", &pin, &other])
6044 .await
6045 .unwrap();
6046 }
6047 "symbolic" => {
6048 git::git(&f.repo, &["symbolic-ref", &pin, "refs/heads/main"])
6049 .await
6050 .unwrap();
6051 }
6052 "symlink" => {
6053 let common = crate::repo_lock::common_dir(&f.repo)
6054 .await
6055 .unwrap()
6056 .unwrap();
6057 let target = f._temp.path().join("operator-anchor");
6058 std::fs::write(&target, format!("{}\n", f.base)).unwrap();
6059 let file = common.join(&pin);
6060 std::fs::create_dir_all(file.parent().unwrap()).unwrap();
6061 std::os::unix::fs::symlink(target, file).unwrap();
6062 }
6063 _ => {}
6064 }
6065 assert!(
6066 restore_execution_seats(&f.home, &state, None)
6067 .await
6068 .is_err(),
6069 "{kind}"
6070 );
6071 assert!(!review.exists());
6072 assert!(
6073 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
6074 .await
6075 .unwrap()
6076 .ok()
6077 );
6078 if kind == "symlink" {
6079 let common = crate::repo_lock::common_dir(&f.repo)
6080 .await
6081 .unwrap()
6082 .unwrap();
6083 std::fs::remove_file(common.join(&pin)).unwrap();
6084 } else {
6085 git::git(&f.repo, &["update-ref", "--no-deref", "-d", &pin])
6086 .await
6087 .unwrap();
6088 }
6089 git::git(&f.repo, &["update-ref", &pin, &f.base])
6090 .await
6091 .unwrap();
6092 }
6093 assert!(
6096 git::worktree_remove_clean(&f.repo, &state.candidates[0].worktree)
6097 .await
6098 .unwrap()
6099 );
6100 git::git(
6101 &f.repo,
6102 &[
6103 "update-ref",
6104 "-d",
6105 &format!("refs/heads/{}", state.candidates[0].branch),
6106 ],
6107 )
6108 .await
6109 .unwrap();
6110 git::git(&f.repo, &["update-ref", "-d", "refs/heads/main"])
6111 .await
6112 .unwrap();
6113 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6114 .await
6115 .unwrap();
6116 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6117 assert_eq!(
6118 git::git(&f.repo, &["for-each-ref", "--format=%(refname)"])
6119 .await
6120 .unwrap(),
6121 pin
6122 );
6123 assert_eq!(
6124 git::git(&f.repo, &["rev-list", "--count", &f.base])
6125 .await
6126 .unwrap(),
6127 "5001"
6128 );
6129 restore_execution_seats(&f.home, &state, None)
6130 .await
6131 .unwrap();
6132 assert_eq!(
6133 git::git(&review, &["rev-list", "--count", "HEAD"])
6134 .await
6135 .unwrap(),
6136 "5002"
6137 );
6138 assert_eq!(
6139 git::git(&f.repo, &["rev-parse", "--is-shallow-repository"])
6140 .await
6141 .unwrap(),
6142 "false"
6143 );
6144 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
6145 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6146 let patch = seat_diff(&f.home, &state.id, &head).await.unwrap();
6147 assert!(String::from_utf8_lossy(&patch).contains("bounded review delta"));
6148 std::fs::remove_dir_all(&f.repo).unwrap();
6149 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), patch);
6150 let inspection = f._temp.path().join("shared-history-inspection");
6151 restore_seat(&f.home, &state.id, &head, &inspection)
6152 .await
6153 .unwrap();
6154 assert_eq!(git::rev_parse(&inspection, "HEAD").await.unwrap(), head);
6155 assert_eq!(
6156 git::git(&inspection, &["rev-parse", "--is-shallow-repository"])
6157 .await
6158 .unwrap(),
6159 "true"
6160 );
6161 assert_eq!(
6162 git::git(&inspection, &["log", "--format=%H"])
6163 .await
6164 .unwrap(),
6165 format!("{head}\n{}", f.base)
6166 );
6167 assert!(!inspection.join(".git/objects/info/alternates").exists());
6168 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
6169 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
6170 }
6171
6172 #[tokio::test]
6173 async fn source_eviction_retains_tree_only_archives_and_rejects_import_before_writes() {
6174 let f = Fx::new().await;
6175 let state = inactive_source(&f, "ef08", true).await;
6176 let review = reviewer(&f, &state, &f.base).await;
6177 let tree = git::rev_parse(&review, "HEAD^{tree}").await.unwrap();
6178 let parent = git::git(&review, &["commit-tree", &tree, "-m", "U0"])
6179 .await
6180 .unwrap();
6181 let head = git::git(&review, &["commit-tree", &tree, "-p", &parent, "-m", "U1"])
6182 .await
6183 .unwrap();
6184 git::git(&review, &["checkout", "--detach", &head])
6185 .await
6186 .unwrap();
6187 let archive =
6188 crate::fold_evidence::fixtures::tree_only_archive(&state, &f.home, &head).await;
6189 let mut report = Report::default();
6190 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6191 assert_eq!(report.evicted_copies, 0, "{report:?}");
6192 assert!(review.exists());
6193 assert!(
6194 report
6195 .retained
6196 .iter()
6197 .any(|r| r.contains("incomplete archived commit ancestry")),
6198 "{report:?}"
6199 );
6200 assert!(git::worktree_remove_clean(&f.repo, &review).await.unwrap());
6202 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6203 .await
6204 .unwrap();
6205 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6206 for oid in [&parent, &head] {
6207 assert!(
6208 !git::git_raw(&f.repo, &["cat-file", "-e", oid])
6209 .await
6210 .unwrap()
6211 .ok()
6212 );
6213 }
6214 let patch = source_diff(&archive).await.unwrap();
6215 assert!(
6216 crate::fold_evidence::import_snapshot(&archive, &f.repo)
6217 .await
6218 .is_err()
6219 );
6220 assert!(
6221 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
6222 .await
6223 .unwrap()
6224 .ok()
6225 );
6226 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
6227 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6228 let inspection = f._temp.path().join("legacy-tree-inspection");
6229 restore_source(&archive, &inspection).await.unwrap();
6230 assert_eq!(git::git(&inspection, &["write-tree"]).await.unwrap(), tree);
6231 assert!(
6232 !git::git_raw(&inspection, &["rev-parse", "--verify", "HEAD"])
6233 .await
6234 .unwrap()
6235 .ok()
6236 );
6237 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
6238 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
6239 assert_eq!(source_diff(&archive).await.unwrap(), patch);
6240 assert!(state.candidates[0].worktree.exists());
6241 }
6242
6243 #[tokio::test]
6244 async fn source_eviction_retains_ancestry_above_expanded_budget() {
6245 let f = Fx::new().await;
6246 let state = inactive_source(&f, "ef09", true).await;
6247 let review = reviewer(&f, &state, &f.base).await;
6248 std::fs::File::create(review.join("ancestor-only"))
6249 .unwrap()
6250 .set_len(64 * 1024 * 1024 + 1)
6251 .unwrap();
6252 git::git(&review, &["add", "."]).await.unwrap();
6253 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
6254 let parent = git::git(
6255 &review,
6256 &["commit-tree", &parent_tree, "-m", "large ancestor"],
6257 )
6258 .await
6259 .unwrap();
6260 std::fs::remove_file(review.join("ancestor-only")).unwrap();
6261 git::git(&review, &["add", "."]).await.unwrap();
6262 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6263 let head = git::git(
6264 &review,
6265 &[
6266 "commit-tree",
6267 &tree,
6268 "-p",
6269 &parent,
6270 "-m",
6271 "small final tree",
6272 ],
6273 )
6274 .await
6275 .unwrap();
6276 git::git(&review, &["checkout", "--detach", &head])
6277 .await
6278 .unwrap();
6279 let mut report = Report::default();
6280 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6281 assert_eq!(report.evicted_copies, 0, "{report:?}");
6282 assert!(
6283 report
6284 .retained
6285 .iter()
6286 .any(|r| r.contains("expanded object limit exceeded")),
6287 "{report:?}"
6288 );
6289 assert!(review.exists());
6290 assert!(state.candidates[0].worktree.exists());
6291 }
6292
6293 #[tokio::test]
6294 async fn source_eviction_restores_execution_after_gc_and_preserves_generations() {
6295 let f = Fx::new().await;
6296 let state = inactive_source(&f, "ef04", true).await;
6297 let review = reviewer(&f, &state, &f.base).await;
6298 std::fs::write(review.join("ancestor-only"), "ancestral content").unwrap();
6299 git::git(&review, &["add", "."]).await.unwrap();
6300 let parent_tree = git::git(&review, &["write-tree"]).await.unwrap();
6301 let parent = git::git(
6302 &review,
6303 &["commit-tree", &parent_tree, "-m", "unrelated parent"],
6304 )
6305 .await
6306 .unwrap();
6307 std::fs::remove_file(review.join("ancestor-only")).unwrap();
6308 std::fs::write(review.join("source"), "unrelated review checkpoint").unwrap();
6309 git::git(&review, &["add", "."]).await.unwrap();
6310 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6311 let head = git::git(
6312 &review,
6313 &[
6314 "commit-tree",
6315 &tree,
6316 "-p",
6317 &parent,
6318 "-m",
6319 "unrelated review checkpoint",
6320 ],
6321 )
6322 .await
6323 .unwrap();
6324 git::git(&review, &["checkout", "--detach", &head])
6325 .await
6326 .unwrap();
6327 let mut report = Report::default();
6328 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6329 assert_eq!(report.evicted_copies, 1, "{report:?}");
6330 let ledger = f
6331 .home
6332 .join("runs")
6333 .join(&state.id)
6334 .join("artifacts/eviction-review-1.json");
6335 let original = std::fs::read(&ledger).unwrap();
6336 let diff = seat_diff(&f.home, &state.id, &head).await.unwrap();
6337 assert!(String::from_utf8_lossy(&diff).contains("unrelated review checkpoint"));
6338 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6339 .await
6340 .unwrap();
6341 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6342 assert!(
6343 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
6344 .await
6345 .unwrap()
6346 .ok()
6347 );
6348 assert!(
6349 !git::git_raw(&f.repo, &["cat-file", "-e", &parent])
6350 .await
6351 .unwrap()
6352 .ok()
6353 );
6354 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
6355 assert!(
6356 seat_heads(&f.home, &state.id)
6357 .await
6358 .unwrap()
6359 .contains_key(&head)
6360 );
6361 restore_execution_seats(&f.home, &state, None)
6362 .await
6363 .unwrap();
6364 assert_eq!(git::rev_parse(&review, "HEAD").await.unwrap(), head);
6365 assert_eq!(
6366 git::git(&review, &["log", "--format=%H"]).await.unwrap(),
6367 format!("{head}\n{parent}")
6368 );
6369 git::git(&f.repo, &["fsck", "--full"]).await.unwrap();
6370 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6371 let inspection = f._temp.path().join("complete-history-inspection");
6372 restore_seat(&f.home, &state.id, &head, &inspection)
6373 .await
6374 .unwrap();
6375 assert_eq!(
6376 git::git(&inspection, &["log", "--format=%H"])
6377 .await
6378 .unwrap(),
6379 format!("{head}\n{parent}")
6380 );
6381 git::git(&inspection, &["fsck", "--full"]).await.unwrap();
6382 git::git(&inspection, &["gc", "--prune=now"]).await.unwrap();
6383
6384 assert_eq!(git::git(&review, &["write-tree"]).await.unwrap(), tree);
6385 assert!(
6386 !git::git_raw(&review, &["symbolic-ref", "-q", "HEAD"])
6387 .await
6388 .unwrap()
6389 .ok()
6390 );
6391 assert_eq!(
6392 std::fs::read_to_string(review.join("source")).unwrap(),
6393 "unrelated review checkpoint"
6394 );
6395 std::fs::write(review.join("operator-notes"), "keep").unwrap();
6396 assert!(
6397 restore_execution_seats(&f.home, &state, None)
6398 .await
6399 .is_err()
6400 );
6401 assert_eq!(
6402 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
6403 "keep"
6404 );
6405 std::fs::remove_file(review.join("operator-notes")).unwrap();
6406 restore_execution_seats(&f.home, &state, None)
6407 .await
6408 .unwrap();
6409 let mut second = Report::default();
6410 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
6411 assert_eq!(second.evicted_copies, 1, "{second:?}");
6412 assert_eq!(std::fs::read(&ledger).unwrap(), original);
6413 assert!(ledger.with_file_name("eviction-review-1-2.json").exists());
6414 let mut legacy: serde_json::Value = serde_json::from_slice(&original).unwrap();
6417 legacy["version"] = 1.into();
6418 legacy.as_object_mut().unwrap().remove("base");
6419 std::fs::write(&ledger, serde_json::to_vec(&legacy).unwrap()).unwrap();
6420 let mut advanced = state.clone();
6421 advanced.base_commit = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6422 .await
6423 .unwrap();
6424 advanced.save_under(&f.home).unwrap();
6425 assert_eq!(seat_diff(&f.home, &state.id, &head).await.unwrap(), diff);
6426 let raw: serde_json::Value = serde_json::from_slice(&original).unwrap();
6428 let archive = PathBuf::from(raw["evidence"].as_str().unwrap());
6429 let patch = std::fs::read(archive.join("change.patch")).unwrap();
6430 std::fs::write(archive.join("change.patch"), "corrupt").unwrap();
6431 assert!(
6432 restore_execution_seats(&f.home, &state, None)
6433 .await
6434 .is_err()
6435 );
6436 assert!(!review.exists());
6437 std::fs::write(archive.join("change.patch"), patch).unwrap();
6438 std::fs::create_dir(&review).unwrap();
6439 std::fs::write(review.join("operator-notes"), "keep").unwrap();
6440 assert!(
6441 restore_execution_seats(&f.home, &state, None)
6442 .await
6443 .is_err()
6444 );
6445 assert_eq!(
6446 std::fs::read_to_string(review.join("operator-notes")).unwrap(),
6447 "keep"
6448 );
6449 }
6450
6451 #[tokio::test]
6452 async fn source_eviction_preserves_history_and_restores_binary_modes_after_gc() {
6453 use std::os::unix::fs::PermissionsExt;
6454 let mut f = Fx::new().await;
6455 std::fs::write(f.repo.join("binary"), b"old\0binary\xff").unwrap();
6456 git::git(&f.repo, &["add", "."]).await.unwrap();
6457 git::git(&f.repo, &["commit", "-m", "binary base"])
6458 .await
6459 .unwrap();
6460 f.base = git::rev_parse(&f.repo, "HEAD").await.unwrap();
6461 let mut state = inactive_source(&f, "ec01", true).await;
6462 let candidate = state.candidates[0].worktree.clone();
6463 let earlier = git::rev_parse(&candidate, "HEAD").await.unwrap();
6464 historical_review(&mut state, &earlier);
6465 let review = reviewer(&f, &state, &earlier).await;
6466 std::fs::remove_file(candidate.join("source")).unwrap();
6467 std::fs::write(candidate.join("binary"), b"new\0binary\xfe").unwrap();
6468 std::fs::write(candidate.join("script"), "run\n").unwrap();
6469 std::fs::set_permissions(
6470 candidate.join("script"),
6471 std::fs::Permissions::from_mode(0o755),
6472 )
6473 .unwrap();
6474 git::git(&candidate, &["add", "."]).await.unwrap();
6475 git::git(&candidate, &["commit", "-m", "binary deletion and mode"])
6476 .await
6477 .unwrap();
6478 let head = git::rev_parse(&candidate, "HEAD").await.unwrap();
6479 let tree = git::rev_parse(&candidate, "HEAD^{tree}").await.unwrap();
6480 let base_tree = git::rev_parse(&candidate, &format!("{}^{{tree}}", f.base))
6481 .await
6482 .unwrap();
6483 historical_review(&mut state, &head);
6484 historical_review(&mut state, &head);
6485 state.status = RunStatus::Superseded;
6486 state.save_under(&f.home).unwrap();
6487 let task = f.task(std::slice::from_ref(&state));
6488 let before = serde_json::to_value(&state).unwrap();
6489 git::git(&review, &["reset", "--hard", &head])
6490 .await
6491 .unwrap();
6492 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
6493 assert_eq!(report.evicted_copies, 1, "{report:?}");
6494 assert!(!review.exists());
6495 assert!(candidate.exists(), "selected implementation survives");
6496 assert!(report.source_unlinked_logical_bytes > 0);
6497 assert!(report.source_unlinked_allocated_bytes.is_some());
6498 assert!(report.source_evidence_added_allocated_bytes.is_some());
6499 assert_eq!(
6500 serde_json::to_value(RunState::load_under(&state.id, &f.home).unwrap()).unwrap(),
6501 before
6502 );
6503 assert_eq!(
6504 crate::queue::Queue::at(f.home.join("queue"))
6505 .get(&task.id)
6506 .unwrap()
6507 .runs,
6508 task.runs
6509 );
6510 let archive = f
6511 .home
6512 .join("runs")
6513 .join(&state.id)
6514 .join("artifacts/source-evidence")
6515 .join(format!("{head}-{}", f.base));
6516 let patch = source_diff(&archive).await.unwrap();
6517 assert!(String::from_utf8_lossy(&patch).contains("GIT binary patch"));
6518 assert!(String::from_utf8_lossy(&patch).contains("100755"));
6519 assert!(
6520 git::worktree_remove_clean(&f.repo, &candidate)
6521 .await
6522 .unwrap()
6523 );
6524 git::git(&f.repo, &["branch", "-D", &state.candidates[0].branch])
6525 .await
6526 .unwrap();
6527 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6528 .await
6529 .unwrap();
6530 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6531 assert!(
6532 !git::git_raw(&f.repo, &["cat-file", "-e", &head])
6533 .await
6534 .unwrap()
6535 .ok()
6536 );
6537 std::fs::remove_dir_all(&f.repo).unwrap();
6539 assert_eq!(source_diff(&archive).await.unwrap(), patch);
6540 let restored = f._temp.path().join("restored");
6541 restore_source(&archive, &restored).await.unwrap();
6542 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
6543 assert_eq!(
6544 std::fs::read(restored.join("binary")).unwrap(),
6545 b"new\0binary\xfe"
6546 );
6547 assert!(!restored.join("source").exists());
6548 assert_ne!(
6549 std::fs::metadata(restored.join("script"))
6550 .unwrap()
6551 .permissions()
6552 .mode()
6553 & 0o111,
6554 0
6555 );
6556 assert!(restore_source(&archive, &restored).await.is_err());
6557 git::git(&restored, &["read-tree", "--reset", "-u", &base_tree])
6560 .await
6561 .unwrap();
6562 let patch_file = f._temp.path().join("saved.patch");
6563 std::fs::write(&patch_file, &patch).unwrap();
6564 git::git(
6565 &restored,
6566 &["apply", "--index", "--binary", patch_file.to_str().unwrap()],
6567 )
6568 .await
6569 .unwrap();
6570 assert_eq!(git::git(&restored, &["write-tree"]).await.unwrap(), tree);
6571 assert_eq!(
6572 std::fs::read(restored.join("binary")).unwrap(),
6573 b"new\0binary\xfe"
6574 );
6575 }
6576
6577 #[tokio::test]
6578 async fn source_eviction_archives_unrelated_tips_and_allows_lineage_references() {
6579 let f = Fx::new().await;
6580 let state = inactive_source(&f, "ec10", true).await;
6581 let review = reviewer(&f, &state, &f.base).await;
6582 let mut successor = inactive_source(&f, "ec11", true).await;
6583 successor.retired_to = Some(state.id.clone());
6584 successor.save_under(&f.home).unwrap();
6585 f.task(&[state.clone(), successor]);
6586
6587 let mut tips = Vec::new();
6588 for contents in ["reflog-only evidence", "ORIG_HEAD-only evidence"] {
6589 std::fs::write(review.join("source"), contents).unwrap();
6590 git::git(&review, &["add", "."]).await.unwrap();
6591 let tree = git::git(&review, &["write-tree"]).await.unwrap();
6592 let tip = git::git(
6593 &review,
6594 &["commit-tree", &tree, "-p", &f.base, "-m", contents],
6595 )
6596 .await
6597 .unwrap();
6598 git::git(&review, &["reset", "--hard", &tip]).await.unwrap();
6599 tips.push(tip);
6600 git::git(&review, &["reset", "--hard", &f.base])
6601 .await
6602 .unwrap();
6603 }
6604 git::git(&review, &["reflog", "delete", "--rewrite", "HEAD@{1}"])
6606 .await
6607 .unwrap();
6608 git::git(&review, &["update-ref", "ORIG_HEAD", &tips[1]])
6609 .await
6610 .unwrap();
6611 let mut report = Report::default();
6612 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6613 assert_eq!(report.evicted_copies, 1, "{report:?}");
6614 assert!(!review.exists());
6615 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
6616 .await
6617 .unwrap();
6618 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
6619 for tip in &tips {
6620 assert!(
6621 !git::git_raw(&f.repo, &["cat-file", "-e", tip])
6622 .await
6623 .unwrap()
6624 .ok()
6625 );
6626 }
6627 std::fs::remove_dir_all(&f.repo).unwrap();
6629 let shown = seat_evidence(&f.home, &state.id).await.unwrap();
6630 for (i, tip) in tips.iter().enumerate() {
6631 assert!(
6632 shown
6633 .iter()
6634 .any(|row| row.contains(tip) && row.contains("patch_bytes="))
6635 );
6636 let destination = f._temp.path().join(format!("restore-history-{i}"));
6637 restore_seat(&f.home, &state.id, tip, &destination)
6638 .await
6639 .unwrap();
6640 assert_eq!(
6641 std::fs::read_to_string(destination.join("source")).unwrap(),
6642 ["reflog-only evidence", "ORIG_HEAD-only evidence"][i]
6643 );
6644 }
6645 assert!(
6646 restore_seat(
6647 &f.home,
6648 &state.id,
6649 "unknown",
6650 &f._temp.path().join("missing")
6651 )
6652 .await
6653 .is_err()
6654 );
6655 }
6656
6657 #[tokio::test]
6658 async fn source_eviction_revalidates_archive_immediately_before_unlink() {
6659 fn corrupt(path: &Path) -> Result<()> {
6660 let root = path.parent().unwrap();
6661 let marker = root.join("archive-to-corrupt");
6664 let archive = std::fs::read_to_string(&marker)?;
6665 std::fs::write(Path::new(&archive).join("change.patch"), "corrupt")?;
6666 Ok(())
6667 }
6668 let f = Fx::new().await;
6669 let state = inactive_source(&f, "ec12", true).await;
6670 let review = reviewer(&f, &state, &f.base).await;
6671 let archive = f
6672 .home
6673 .join("runs")
6674 .join(&state.id)
6675 .join("artifacts/source-evidence")
6676 .join(format!("{}-{}", f.base, f.base));
6677 std::fs::write(
6678 state.worktree_root().join("archive-to-corrupt"),
6679 archive.to_str().unwrap(),
6680 )
6681 .unwrap();
6682 let mut report = Report::default();
6683 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut report, corrupt).await;
6684 assert_eq!(report.evicted_copies, 0, "{report:?}");
6685 assert!(review.exists());
6686 assert!(
6687 report
6688 .retained
6689 .iter()
6690 .any(|r| r.contains("corrupt snapshot")),
6691 "{report:?}"
6692 );
6693 }
6694
6695 #[tokio::test]
6696 async fn source_eviction_fails_closed_for_dirty_ignored_borrowed_and_authoritative_copies() {
6697 for kind in [
6698 "dirty",
6699 "hidden",
6700 "ignored",
6701 "empty-directory",
6702 "untracked",
6703 "borrowed",
6704 "borrowed-symlink",
6705 "lease",
6706 "owner",
6707 "driver",
6708 "task-pin",
6709 "task-replacement",
6710 "malformed-orig-head",
6711 "grace",
6712 "resumable",
6713 "archive-symlink",
6714 "root-symlink",
6715 "checkpoint",
6716 "symlink",
6717 "retained",
6718 "unreadable",
6719 "unknown-field",
6720 "unknown-task-field",
6721 ] {
6722 let f = Fx::new().await;
6723 let mut state = inactive_source(&f, "ec02", true).await;
6724 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
6725 .await
6726 .unwrap();
6727 historical_review(&mut state, &head);
6728 let review = reviewer(&f, &state, &head).await;
6729 let mut output = None;
6730 let mut owner = None;
6731 let mut driver = None;
6732 let mut disk = Fx::disk();
6733 match kind {
6734 "dirty" => std::fs::write(review.join("source"), "notes").unwrap(),
6735 "hidden" => {
6736 git::git(&review, &["update-index", "--assume-unchanged", "source"])
6737 .await
6738 .unwrap();
6739 std::fs::write(review.join("source"), "notes").unwrap();
6740 }
6741 "ignored" => {
6742 std::fs::create_dir(review.join("scratch")).unwrap();
6743 std::fs::write(review.join("scratch/notes"), "notes").unwrap();
6744 }
6745 "empty-directory" => std::fs::create_dir(review.join("owner-notes")).unwrap(),
6746 "untracked" => std::fs::write(review.join("notes"), "notes").unwrap(),
6747 "borrowed" => {
6748 let mut other = f.run("ec03", true).await;
6749 other.candidates[0].worktree = review.clone();
6750 other.save_under(&f.home).unwrap();
6751 }
6752 "borrowed-symlink" => {
6753 let alias = f._temp.path().join("borrowed-alias");
6754 std::os::unix::fs::symlink(&review, &alias).unwrap();
6755 let mut other = f.run("ec03", true).await;
6756 other.candidates[0].worktree = alias;
6757 other.save_under(&f.home).unwrap();
6758 }
6759 "lease" => output = OutputLease::borrow(&review).unwrap(),
6760 "owner" => owner = Some(crate::pr_head::owner_lock(&f.home, &state.id).unwrap()),
6761 "driver" => driver = Some(crate::driver::maintenance(&f.home, &state.id).unwrap()),
6762 "task-pin" => {
6763 let mut task = f.task(std::slice::from_ref(&state));
6764 task.resume_override = Some(crate::queue::OperatorResume {
6765 question_id: "fixture-pin".into(),
6766 at: Fx::now(),
6767 conductor_rehold: None,
6768 forced: true,
6769 pinned_run: Some(state.id.clone()),
6770 });
6771 crate::queue::Queue::at(f.home.join("queue"))
6772 .put(&mut task)
6773 .unwrap();
6774 }
6775 "task-replacement" => {
6776 let mut task = f.task(std::slice::from_ref(&state));
6777 task.replacement = Some(crate::merge_decision::Replacement {
6778 question: "fixture-replacement".into(),
6779 run: state.id.clone(),
6780 pr: "https://example.invalid/pull/1".into(),
6781 agent: "worker".into(),
6782 context: String::new(),
6783 });
6784 crate::queue::Queue::at(f.home.join("queue"))
6785 .put(&mut task)
6786 .unwrap();
6787 }
6788 "malformed-orig-head" => {
6789 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6790 .await
6791 .unwrap();
6792 std::fs::write(Path::new(&admin).join("ORIG_HEAD"), "not-an-object-id")
6793 .unwrap();
6794 }
6795 "grace" => disk.fold_grace_secs = 3600,
6796 "resumable" => state.status = RunStatus::Blocked,
6797 "archive-symlink" => {
6798 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6799 std::fs::create_dir_all(&artifacts).unwrap();
6800 let outside = f._temp.path().join("outside");
6801 std::fs::create_dir(&outside).unwrap();
6802 std::os::unix::fs::symlink(&outside, artifacts.join("source-evidence"))
6803 .unwrap();
6804 }
6805 "root-symlink" => {
6806 let root = state.worktree_root();
6807 let outside = f._temp.path().join("outside-root");
6808 std::fs::rename(&root, &outside).unwrap();
6809 std::os::unix::fs::symlink(&outside, &root).unwrap();
6810 }
6811 "checkpoint" => {
6812 let admin = git::git(&review, &["rev-parse", "--absolute-git-dir"])
6813 .await
6814 .unwrap();
6815 std::fs::write(Path::new(&admin).join("MERGE_MSG"), "checkpoint").unwrap();
6816 }
6817 "symlink" => {
6818 std::fs::remove_file(review.join("source")).unwrap();
6819 std::os::unix::fs::symlink(".gitignore", review.join("source")).unwrap();
6820 }
6821 "retained" => std::fs::write(
6822 f.home
6823 .join("runs")
6824 .join(&state.id)
6825 .join("retain-recovery-sources"),
6826 "owner",
6827 )
6828 .unwrap(),
6829 "unreadable" => {
6830 std::fs::create_dir_all(f.home.join("runs/20260101-000000-ffff")).unwrap();
6831 }
6832 "unknown-field" | "unknown-task-field" => {}
6833 _ => unreachable!(),
6834 }
6835 state.save_under(&f.home).unwrap();
6836 if kind == "unknown-field" {
6837 let path = f.home.join("runs").join(&state.id).join("run.json");
6838 let mut raw: serde_json::Value =
6839 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6840 raw["future_source_owner"] = serde_json::json!({"path": review});
6841 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6842 }
6843 if kind == "unknown-task-field" {
6844 let task = f.task(std::slice::from_ref(&state));
6845 let path = crate::queue::Queue::at(f.home.join("queue")).path_of(&task.id);
6846 let mut raw: serde_json::Value =
6847 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
6848 raw["future_source_owner"] = serde_json::json!({"path": review});
6849 std::fs::write(&path, serde_json::to_vec(&raw).unwrap()).unwrap();
6850 }
6851 let mut report = Report::default();
6852 copies::evict(&f.home, &disk, Fx::now(), &mut report).await;
6853 assert_eq!(report.evicted_copies, 0, "{kind}: {report:?}");
6854 assert!(review.exists(), "{kind}: {report:?}");
6855 assert!(!report.retained.is_empty(), "{kind}: {report:?}");
6856 drop(output);
6857 drop(owner);
6858 drop(driver);
6859 }
6860 }
6861
6862 #[tokio::test]
6863 async fn source_eviction_reconciles_interruption_and_refuses_corrupt_archives() {
6864 let f = Fx::new().await;
6865 let state = inactive_source(&f, "ec04", true).await;
6866 let review = reviewer(&f, &state, &f.base).await;
6867 let mut report = Report::default();
6868 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6869 assert_eq!(report.evicted_copies, 1, "{report:?}");
6870 let artifacts = f.home.join("runs").join(&state.id).join("artifacts");
6871 let ledger = artifacts.join("eviction-review-1.json");
6872 let mut record: serde_json::Value =
6873 serde_json::from_slice(&std::fs::read(&ledger).unwrap()).unwrap();
6874 record["phase"] = "pending".into();
6875 std::fs::write(&ledger, serde_json::to_vec(&record).unwrap()).unwrap();
6876 let archive = PathBuf::from(record["evidence"].as_str().unwrap());
6877 let pack = std::fs::read(archive.join("objects.pack")).unwrap();
6878 std::fs::write(archive.join("objects.pack"), b"corrupt").unwrap();
6879 let mut blocked = Report::default();
6880 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
6881 assert!(
6882 blocked
6883 .retained
6884 .iter()
6885 .any(|r| r.contains("corrupt snapshot")),
6886 "{blocked:?}"
6887 );
6888 assert_eq!(
6889 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6890 ["phase"],
6891 "pending"
6892 );
6893 assert!(
6894 restore_source(&archive, &f._temp.path().join("bad-restore"))
6895 .await
6896 .is_err()
6897 );
6898 std::fs::write(archive.join("objects.pack"), &pack).unwrap();
6899 let mut retried = Report::default();
6900 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
6901 assert_eq!(
6902 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap()).unwrap()
6903 ["phase"],
6904 "evicted"
6905 );
6906 assert_eq!(retried.evicted_copies, 0);
6907 assert!(!review.exists());
6908 let pending = archive.with_extension("pending");
6910 std::fs::rename(&archive, &pending).unwrap();
6911 assert_eq!(
6912 crate::fold_evidence::snapshot(&state, &f.home, &f.base, 64 * 1024 * 1024)
6913 .await
6914 .unwrap(),
6915 archive
6916 );
6917 assert!(!pending.exists());
6918 }
6919
6920 #[tokio::test]
6921 async fn source_eviction_reclaims_superseded_empty_attempt_without_retiring_lineage() {
6922 let f = Fx::new().await;
6923 let mut empty = inactive_source(&f, "ec05", false).await;
6924 historical_review(&mut empty, &f.base);
6925 empty.status = RunStatus::Superseded;
6926 empty.save_under(&f.home).unwrap();
6927 let source = inactive_source(&f, "ec06", true).await;
6928 let task = f.task(&[empty.clone(), source.clone()]);
6929 let before = serde_json::to_value(&empty).unwrap();
6930 let mut report = Report::default();
6931 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
6932 assert_eq!(report.evicted_copies, 1, "{report:?}");
6933 assert!(!empty.candidates[0].worktree.exists());
6934 assert!(source.candidates[0].worktree.exists());
6935 assert!(
6936 git::branch_exists(&f.repo, &empty.candidates[0].branch)
6937 .await
6938 .unwrap()
6939 );
6940 assert_eq!(
6941 serde_json::to_value(RunState::load_under(&empty.id, &f.home).unwrap()).unwrap(),
6942 before
6943 );
6944 let mut retry = Report::default();
6945 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
6946 assert_eq!(retry.removed_empty_roots, 1, "{retry:?}");
6947 assert!(!empty.worktree_root().exists());
6948 assert_eq!(
6949 crate::queue::Queue::at(f.home.join("queue"))
6950 .get(&task.id)
6951 .unwrap()
6952 .runs,
6953 task.runs
6954 );
6955 }
6956 #[tokio::test]
6957 async fn source_eviction_retries_pending_intent_and_exact_missing_registration() {
6958 fn interrupt(_: &Path) -> Result<()> {
6959 anyhow::bail!("fixture interruption before unlink")
6960 }
6961 fn vanish(path: &Path) -> Result<()> {
6962 std::fs::remove_dir_all(path)?;
6963 anyhow::bail!("fixture interruption after checkout unlink")
6964 }
6965 for stage in [
6966 "intent",
6967 "missing-registration",
6968 "changed-admin",
6969 "missing-archive",
6970 "wrong-identity",
6971 ] {
6972 let f = Fx::new().await;
6973 let state = inactive_source(&f, "ec07", true).await;
6974 let review = reviewer(&f, &state, &f.base).await;
6975 let admin = PathBuf::from(
6976 git::git(&review, &["rev-parse", "--absolute-git-dir"])
6977 .await
6978 .unwrap(),
6979 );
6980 let mut interrupted = Report::default();
6981 copies::evict_with(
6982 &f.home,
6983 &Fx::disk(),
6984 Fx::now(),
6985 &mut interrupted,
6986 if stage == "missing-registration" || stage == "changed-admin" {
6987 vanish
6988 } else {
6989 interrupt
6990 },
6991 )
6992 .await;
6993 assert_eq!(interrupted.evicted_copies, 0, "{stage}: {interrupted:?}");
6994 let ledger = f
6995 .home
6996 .join("runs")
6997 .join(&state.id)
6998 .join("artifacts/eviction-review-1.json");
6999 let record: serde_json::Value =
7000 serde_json::from_slice(&std::fs::read(&ledger).unwrap_or_else(|error| {
7001 panic!("{stage}: missing pending ledger: {error}; {interrupted:?}")
7002 }))
7003 .unwrap();
7004 assert_eq!(record["phase"], "pending");
7005 if stage == "changed-admin" {
7006 std::fs::write(admin.join("owner-note"), "keep").unwrap();
7007 }
7008 if stage == "missing-archive" {
7009 std::fs::remove_file(
7010 Path::new(record["evidence"].as_str().unwrap()).join("objects.pack"),
7011 )
7012 .unwrap();
7013 }
7014 if stage == "wrong-identity" {
7015 let head = git::rev_parse(&state.candidates[0].worktree, "HEAD")
7016 .await
7017 .unwrap();
7018 let other =
7019 crate::fold_evidence::snapshot(&state, &f.home, &head, 64 * 1024 * 1024)
7020 .await
7021 .unwrap();
7022 let original = Path::new(record["evidence"].as_str().unwrap());
7023 for name in [
7024 "manifest.json",
7025 "objects.pack",
7026 "change.patch",
7027 "change.index",
7028 ] {
7029 std::fs::copy(other.join(name), original.join(name)).unwrap();
7030 }
7031 }
7032 let mut retried = Report::default();
7033 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retried).await;
7034 let phase =
7035 serde_json::from_slice::<serde_json::Value>(&std::fs::read(&ledger).unwrap())
7036 .unwrap()["phase"]
7037 .clone();
7038 if stage == "changed-admin" || stage == "missing-archive" || stage == "wrong-identity" {
7039 assert_eq!(phase, "pending", "{stage}: {retried:?}");
7040 assert!(!retried.retained.is_empty());
7041 if stage == "changed-admin" {
7042 assert!(admin.join("owner-note").exists());
7043 } else {
7044 assert!(review.exists());
7045 }
7046 } else {
7047 assert_eq!(phase, "evicted", "{stage}: {retried:?}");
7048 assert!(!review.exists());
7049 assert!(!admin.exists());
7050 }
7051 }
7052 }
7053 fn source_census(home: &Path) -> serde_json::Value {
7054 serde_json::from_slice(&std::fs::read(home.join("source-eviction-cursor")).unwrap())
7055 .unwrap()
7056 }
7057
7058 #[tokio::test]
7059 async fn source_eviction_scan_bounds_missing_root_checkpoints_and_eventually_reclaims() {
7060 let f = Fx::new().await;
7061 let state = inactive_source(&f, "ffff", true).await;
7062 let review = reviewer(&f, &state, &f.base).await;
7063 for i in 0..130 {
7064 let mut old = state.clone();
7065 old.id = format!("20260101-000000-{i:04x}");
7066 old.candidates.clear();
7067 old.save_under(&f.home).unwrap();
7068 if i == 0 {
7070 let artifacts = f.home.join("runs").join(&old.id).join("artifacts");
7071 std::fs::create_dir_all(&artifacts).unwrap();
7072 std::fs::write(artifacts.join("eviction-review-1.json"), "{}").unwrap();
7073 }
7074 }
7075 for cycle in 1..=2 {
7080 for (pass, expected_next) in [65, 129, 131].into_iter().enumerate() {
7081 let mut report = Report::default();
7082 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7083 assert!(report.failures.is_empty(), "{report:?}");
7084 assert_eq!(report.evicted_copies, usize::from(cycle == 1 && pass == 2));
7085 assert_eq!(review.exists(), cycle == 1 && pass < 2);
7086 let census = source_census(&f.home);
7087 assert_eq!(census["current"]["next"], expected_next);
7088 assert_eq!(census["current"]["completed_at"].is_null(), pass < 2);
7089 assert_eq!(census["current"]["number"], cycle);
7090 assert_eq!(
7091 census["current"]["runs"]["20260101-000000-0000"]["root"]["outcome"],
7092 "error"
7093 );
7094 assert_eq!(
7095 census["current"]["runs"]["20260101-000000-0001"]["root"]["detail"],
7096 "root absent and no eviction records; nothing to reclaim"
7097 );
7098 if pass < 2 {
7099 let next_id = format!("20260101-000000-{expected_next:04x}");
7100 assert_eq!(
7101 census["current"]["runs"][next_id]["root"]["outcome"],
7102 "not-yet-inspected"
7103 );
7104 }
7105 }
7106 }
7107 }
7108
7109 #[tokio::test]
7110 async fn source_eviction_scan_reaches_old_seats_despite_empty_roots_and_timestamp_churn() {
7111 let f = Fx::new().await;
7112 let mut state = inactive_source(&f, "ffff", true).await;
7113 let review = reviewer(&f, &state, &f.base).await;
7114 let mut history = Vec::new();
7117 for i in 0..130 {
7118 let mut old = state.clone();
7119 old.id = format!("20260101-000000-{i:04x}");
7120 old.candidates.clear();
7121 old.save_under(&f.home).unwrap();
7122 std::fs::create_dir_all(old.worktree_root()).unwrap();
7123 history.push(old);
7124 }
7125 state
7126 .save_under_at(
7127 &f.home,
7128 Timestamp::now() + jiff::SignedDuration::from_secs(10),
7129 )
7130 .unwrap();
7131 for pass in 0..3 {
7132 let mut report = Report::default();
7133 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7134 assert!(report.failures.is_empty(), "{report:?}");
7135 let census = source_census(&f.home);
7136 assert_eq!(
7137 census["current"]["next"].as_u64().unwrap(),
7138 ((pass + 1) * 64).min(131)
7139 );
7140 if pass < 2 {
7141 assert!(review.exists());
7142 assert!(census["current"]["completed_at"].is_null());
7143 assert_eq!(
7144 census["current"]["runs"][&state.id]["root"]["outcome"],
7145 "not-yet-inspected"
7146 );
7147 }
7148 for old in history.iter_mut().take(64) {
7151 old.updated_at = Timestamp::now();
7152 old.save_under(&f.home).unwrap();
7153 }
7154 if pass == 1 {
7155 let older = state.updated_at - jiff::SignedDuration::from_secs(100);
7162 for old in history.iter_mut().take(128) {
7163 if f.home.join("runs").join(&old.id).exists() {
7164 old.save_under_at(&f.home, older).unwrap();
7165 }
7166 }
7167 history[127]
7168 .save_under_at(&f.home, older - jiff::SignedDuration::from_secs(100))
7169 .unwrap();
7170 let mut legacy: Vec<_> = history
7171 .iter()
7172 .filter(|r| f.home.join("runs").join(&r.id).exists())
7173 .cloned()
7174 .collect();
7175 legacy.push(state.clone());
7176 legacy.sort_by_key(|r| r.updated_at);
7177 let index = legacy.iter().position(|r| r.id == history[127].id).unwrap();
7178 let len = legacy.len();
7179 legacy.rotate_left((index + 1) % len);
7180 assert!(legacy.iter().take(64).all(|r| r.id != state.id));
7181 }
7182 if pass == 0 {
7183 std::fs::remove_dir_all(f.home.join("runs").join(&history[64].id)).unwrap();
7184 let mut added = state.clone();
7185 added.id = "20260101-000000-fffe".into();
7186 added.candidates.clear();
7187 added.save_under(&f.home).unwrap();
7188 }
7189 }
7190 assert!(!review.exists());
7191 let census = source_census(&f.home);
7192 assert!(!census["current"]["completed_at"].is_null());
7193 assert_eq!(
7194 census["current"]["runs"][&state.id]["seats"]["review-1"]["outcome"],
7195 "evicted"
7196 );
7197 assert_eq!(
7198 census["current"]["runs"][&history[64].id]["root"]["outcome"],
7199 "error"
7200 );
7201 let mut next = Report::default();
7202 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
7203 let wrapped = source_census(&f.home);
7204 assert_eq!(wrapped["current"]["number"], 2);
7205 assert_eq!(wrapped["previous"], census["current"]);
7206 assert!(
7207 wrapped["current"]["runs"]
7208 .get("20260101-000000-fffe")
7209 .is_some()
7210 );
7211 }
7212
7213 #[tokio::test]
7214 async fn source_eviction_scan_resumes_within_run_and_bounds_failed_seats() {
7215 fn fail_one(path: &Path) -> Result<()> {
7216 ensure!(
7217 path.file_name().unwrap() != "review-2",
7218 "injected checkpoint failure"
7219 );
7220 Ok(())
7221 }
7222 let f = Fx::new().await;
7223 let state = inactive_source(&f, "ef20", true).await;
7224 let mut paths = Vec::new();
7225 for i in 1..=8 {
7226 let path = state.worktree_root().join(format!("review-{i}"));
7227 git::worktree_add_detached(&f.repo, &path, &f.base)
7228 .await
7229 .unwrap();
7230 paths.push(path);
7231 }
7232 std::fs::write(paths[2].join("owner-data"), "retain me").unwrap();
7233 let mut first = Report::default();
7234 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut first, fail_one).await;
7235 assert_eq!(first.evicted_copies, 3, "{first:?}");
7236 let census = source_census(&f.home);
7237 let seats = &census["current"]["runs"][&state.id]["seats"];
7238 assert_eq!(census["current"]["after_seat"], "review-5");
7239 assert_eq!(seats["review-2"]["outcome"], "error");
7240 assert_eq!(seats["review-2"]["stage"], "unlink");
7241 assert_eq!(seats["review-3"]["outcome"], "protected");
7242 assert_eq!(seats["review-6"]["outcome"], "not-yet-inspected");
7243 assert!(first.source_unlinked_logical_bytes > 0);
7244 assert!(first.source_evidence_added_logical_bytes > 0);
7245 let mut second = Report::default();
7246 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut second).await;
7247 assert_eq!(second.evicted_copies, 3, "{second:?}");
7248 assert!(paths[1].exists(), "failed seat waits until next cycle");
7249 assert!(paths[2].exists());
7250 assert!(!source_census(&f.home)["current"]["completed_at"].is_null());
7251 let mut third = Report::default();
7252 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut third).await;
7253 assert_eq!(third.evicted_copies, 1, "{third:?}");
7254 assert!(!paths[1].exists());
7255 assert_eq!(
7256 std::fs::read_to_string(paths[2].join("owner-data")).unwrap(),
7257 "retain me"
7258 );
7259 let shown = seat_inspections(&f.home, &state.id).unwrap().join("\n");
7260 assert!(
7261 shown.contains("cycle=1 complete=true review-2 error stage=unlink"),
7262 "{shown}"
7263 );
7264 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
7265 let destination = f.bay.join("restored-scan-seat");
7266 restore_seat(&f.home, &state.id, &f.base, &destination)
7267 .await
7268 .unwrap();
7269 assert_eq!(git::rev_parse(&destination, "HEAD").await.unwrap(), f.base);
7270 }
7271
7272 #[tokio::test]
7273 async fn source_eviction_scan_continues_after_archive_failure_and_busy_output() {
7274 let f = Fx::new().await;
7275 let failed = inactive_source(&f, "ef30", true).await;
7276 let failed_seat = reviewer(&f, &failed, &f.base).await;
7277 let evidence = f
7278 .home
7279 .join("runs")
7280 .join(&failed.id)
7281 .join("artifacts/source-evidence");
7282 std::fs::create_dir_all(&evidence).unwrap();
7283 std::fs::File::create(evidence.join("reserved-evidence"))
7284 .unwrap()
7285 .set_len(64 * 1024 * 1024)
7286 .unwrap();
7287 let state = inactive_source(&f, "ef31", true).await;
7288 let busy = reviewer(&f, &state, &f.base).await;
7289 let available = state.worktree_root().join("review-2");
7290 git::worktree_add_detached(&f.repo, &available, &f.base)
7291 .await
7292 .unwrap();
7293 let output = OutputLease::take(&busy, true).unwrap();
7294 let mut report = Report::default();
7295 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7296 assert_eq!(report.evicted_copies, 1, "{report:?}");
7297 assert!(failed_seat.exists());
7298 assert!(busy.exists());
7299 assert!(!available.exists());
7300 let census = source_census(&f.home);
7301 assert!(!census["current"]["completed_at"].is_null());
7302 assert_eq!(
7303 census["current"]["runs"][&failed.id]["seats"]["review-1"]["outcome"],
7304 "error"
7305 );
7306 assert_eq!(
7307 census["current"]["runs"][&state.id]["seats"]["review-1"]["outcome"],
7308 "protected"
7309 );
7310 drop(output);
7311 let mut retry = Report::default();
7312 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
7313 assert_eq!(retry.evicted_copies, 1, "{retry:?}");
7314 assert!(!busy.exists());
7315 assert!(failed_seat.exists());
7316 }
7317
7318 #[tokio::test]
7319 async fn source_eviction_scan_checkpoint_failure_stops_before_later_seats() {
7320 fn block_checkpoint(path: &Path) -> Result<()> {
7321 let home = path
7322 .parent()
7323 .unwrap()
7324 .parent()
7325 .unwrap()
7326 .parent()
7327 .unwrap()
7328 .join("home");
7329 let cursor = home.join("source-eviction-cursor");
7330 std::fs::rename(&cursor, home.join("saved-cursor"))?;
7331 std::fs::create_dir(&cursor)?;
7332 Ok(())
7333 }
7334 let f = Fx::new().await;
7335 let state = inactive_source(&f, "ef32", true).await;
7336 let first = reviewer(&f, &state, &f.base).await;
7337 let second = state.worktree_root().join("review-2");
7338 git::worktree_add_detached(&f.repo, &second, &f.base)
7339 .await
7340 .unwrap();
7341 let mut report = Report::default();
7342 copies::evict_with(
7343 &f.home,
7344 &Fx::disk(),
7345 Fx::now(),
7346 &mut report,
7347 block_checkpoint,
7348 )
7349 .await;
7350 assert_eq!(report.evicted_copies, 1, "{report:?}");
7351 assert!(!first.exists());
7352 assert!(second.exists());
7353 assert!(
7354 report
7355 .failures
7356 .iter()
7357 .any(|e| e.contains("persist source scan census"))
7358 );
7359 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "error");
7360 std::fs::remove_dir(f.home.join("source-eviction-cursor")).unwrap();
7361 std::fs::rename(
7362 f.home.join("saved-cursor"),
7363 f.home.join("source-eviction-cursor"),
7364 )
7365 .unwrap();
7366 let mut retry = Report::default();
7367 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
7368 assert_eq!(retry.evicted_copies, 1, "{retry:?}");
7369 assert!(!second.exists());
7370 }
7371
7372 #[tokio::test]
7373 async fn source_eviction_scan_reports_invalid_cursor_and_global_ownership() {
7374 let f = Fx::new().await;
7375 let state = inactive_source(&f, "ef21", true).await;
7376 let review = reviewer(&f, &state, &f.base).await;
7377 let cursor = f.home.join("source-eviction-cursor");
7378 std::fs::write(&cursor, "broken cursor").unwrap();
7379 let mut report = Report::default();
7380 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7381 assert_eq!(std::fs::read_to_string(&cursor).unwrap(), "broken cursor");
7382 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "error");
7383 assert!(review.exists());
7384 std::fs::write(&cursor, "20260101-000000-abcd").unwrap();
7386 let mut disabled = Fx::disk();
7387 disabled.auto_fold = false;
7388 let mut report = Report::default();
7389 copies::evict(&f.home, &disabled, Fx::now(), &mut report).await;
7390 assert_eq!(report.source_scan.as_ref().unwrap()["outcome"], "disabled");
7391 let record = f.home.join("runs").join(&state.id).join("run.json");
7392 let original = std::fs::read(&record).unwrap();
7393 std::fs::write(&record, "invalid ownership").unwrap();
7394 let mut report = Report::default();
7395 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7396 assert!(
7397 report
7398 .failures
7399 .iter()
7400 .any(|e| e.contains("global run ownership validation"))
7401 );
7402 assert!(review.exists());
7403 std::fs::write(record, original).unwrap();
7404 let mut report = Report::default();
7405 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7406 assert_eq!(report.evicted_copies, 1, "{report:?}");
7407 assert_eq!(source_census(&f.home)["version"], 1);
7408 }
7409
7410 #[tokio::test]
7411 async fn source_eviction_batches_reproducible_seats_without_dropping_task_history() {
7412 let f = Fx::new().await;
7413 let mut runs = Vec::new();
7414 let mut paths = Vec::new();
7415 for tag in ["ec10", "ec11", "ec12", "ec13"] {
7416 let state = inactive_source(&f, tag, true).await;
7417 for name in ["judge-1", "advisor-1"] {
7418 let path = state.worktree_root().join(name);
7419 git::worktree_add_detached(&f.repo, &path, &f.base)
7420 .await
7421 .unwrap();
7422 paths.push(path);
7423 }
7424 runs.push(state);
7425 }
7426 let task = f.task(&runs);
7427 let mut report = Report::default();
7428 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7429 assert_eq!(report.evicted_copies, 5, "{report:?}");
7430 let mut next = Report::default();
7431 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut next).await;
7432 assert_eq!(next.evicted_copies, 3, "{next:?}");
7433 assert!(paths.iter().all(|p| !p.exists()));
7434 assert!(runs.iter().all(|r| r.candidates[0].worktree.exists()));
7435 assert_eq!(
7436 crate::queue::Queue::at(f.home.join("queue"))
7437 .get(&task.id)
7438 .unwrap()
7439 .runs,
7440 task.runs
7441 );
7442 let limited = Fx::new().await;
7443 let state = inactive_source(&limited, "ec15", true).await;
7444 let copy = reviewer(&limited, &state, &limited.base).await;
7445 let evidence = limited
7446 .home
7447 .join("runs")
7448 .join(&state.id)
7449 .join("artifacts/source-evidence");
7450 std::fs::create_dir_all(&evidence).unwrap();
7451 let retained = evidence.join("reserved-evidence");
7452 std::fs::File::create(&retained)
7453 .unwrap()
7454 .set_len(64 * 1024 * 1024)
7455 .unwrap();
7456 let mut blocked = Report::default();
7457 copies::evict(&limited.home, &Fx::disk(), Fx::now(), &mut blocked).await;
7458 assert_eq!(blocked.evicted_copies, 0, "{blocked:?}");
7459 assert!(copy.exists());
7460 assert_eq!(
7461 std::fs::metadata(&retained).unwrap().len(),
7462 64 * 1024 * 1024
7463 );
7464 assert!(
7465 blocked
7466 .retained
7467 .iter()
7468 .any(|r| r.contains("per-run storage limit")),
7469 "{blocked:?}"
7470 );
7471 }
7472 #[tokio::test]
7473 async fn source_eviction_migrates_historical_records_without_rewriting_them() {
7474 let f = Fx::new().await;
7475 let mut state = inactive_source(&f, "ec14", true).await;
7476 historical_review(&mut state, &f.base);
7477 historical_review(&mut state, &f.base);
7478 state.status = RunStatus::Superseded;
7479 state.save_under(&f.home).unwrap();
7480 let review = reviewer(&f, &state, &f.base).await;
7481 let task = f.task(std::slice::from_ref(&state));
7482 let path = f.home.join("runs").join(&state.id).join("run.json");
7483 let mut raw: serde_json::Value =
7484 serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap();
7485 raw["schema"] = 10.into();
7486 raw.as_object_mut().unwrap().remove("output_activity_at");
7487 raw["resource_wait"] = serde_json::Value::Null;
7490 raw["resource_block"] = serde_json::Value::Null;
7491 raw["driver_claimed"] = serde_json::Value::Null;
7492 raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
7493 let graph = raw["config"]["graph"].as_object_mut().unwrap();
7495 let count = graph.remove("implementers").unwrap();
7496 graph.insert("candidates".into(), count);
7497 let queue = crate::queue::Queue::at(f.home.join("queue"));
7498 let task_path = queue.path_of(&task.id);
7499 let mut task_raw: serde_json::Value =
7500 serde_json::from_slice(&std::fs::read(&task_path).unwrap()).unwrap();
7501 task_raw["sources"] = serde_json::json!({"start_from":null,"pick_commits":[]});
7502 let task_historical = serde_json::to_vec_pretty(&task_raw).unwrap();
7503 std::fs::write(&task_path, &task_historical).unwrap();
7504 let historical = serde_json::to_vec_pretty(&raw).unwrap();
7505 std::fs::write(&path, &historical).unwrap();
7506 let report = sweep(&f.home, &Fx::disk(), Fx::now()).await;
7507 assert_eq!(report.evicted_copies, 1, "{report:?}");
7508 assert!(!review.exists());
7509 assert_eq!(std::fs::read(&path).unwrap(), historical);
7510 assert_eq!(std::fs::read(&task_path).unwrap(), task_historical);
7511 assert!(state.candidates[0].worktree.exists());
7512 assert_eq!(
7513 crate::queue::Queue::at(f.home.join("queue"))
7514 .get(&task.id)
7515 .unwrap()
7516 .runs,
7517 task.runs
7518 );
7519 }
7520 #[tokio::test]
7521 async fn source_eviction_archives_old_only_reflog_oid_after_unrewritten_deletion() {
7522 let f = Fx::new().await;
7523 let state = inactive_source(&f, "ed01", true).await;
7524 let review = reviewer(&f, &state, &f.base).await;
7525 std::fs::write(review.join("old-only"), b"old-only\0binary\xff").unwrap();
7526 git::git(&review, &["add", "."]).await.unwrap();
7527 let tree = git::git(&review, &["write-tree"]).await.unwrap();
7528 let tip = git::git(
7529 &review,
7530 &[
7531 "commit-tree",
7532 &tree,
7533 "-p",
7534 &f.base,
7535 "-m",
7536 "temporary detached source",
7537 ],
7538 )
7539 .await
7540 .unwrap();
7541 git::git(&review, &["checkout", "--detach", &tip])
7542 .await
7543 .unwrap();
7544 git::git(&review, &["checkout", "--detach", &f.base])
7545 .await
7546 .unwrap();
7547 git::git(&review, &["reflog", "delete", "HEAD@{1}"])
7548 .await
7549 .unwrap();
7550 git::git(&review, &["update-ref", "-d", "ORIG_HEAD"])
7551 .await
7552 .unwrap();
7553 let shown = git::git(&review, &["reflog", "show", "--format=%H", "HEAD"])
7554 .await
7555 .unwrap();
7556 assert!(!shown.lines().any(|oid| oid == tip));
7557 let admin = PathBuf::from(
7558 git::git(&review, &["rev-parse", "--absolute-git-dir"])
7559 .await
7560 .unwrap(),
7561 );
7562 let raw = std::fs::read_to_string(admin.join("logs/HEAD")).unwrap();
7563 assert!(raw.lines().any(|line| line.starts_with(&format!("{tip} "))));
7564 fn interrupt(_: &Path) -> Result<()> {
7565 anyhow::bail!("fixture interruption before unlink")
7566 }
7567 let mut pending = Report::default();
7568 copies::evict_with(&f.home, &Fx::disk(), Fx::now(), &mut pending, interrupt).await;
7569 assert!(review.exists());
7570 let ledger = f
7571 .home
7572 .join("runs")
7573 .join(&state.id)
7574 .join("artifacts/eviction-review-1.json");
7575 let complete = std::fs::read(&ledger).unwrap();
7576 let mut incomplete: serde_json::Value = serde_json::from_slice(&complete).unwrap();
7577 incomplete["history"]
7578 .as_object_mut()
7579 .unwrap()
7580 .remove(&tip)
7581 .unwrap();
7582 std::fs::write(&ledger, serde_json::to_vec(&incomplete).unwrap()).unwrap();
7583 let mut blocked = Report::default();
7584 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut blocked).await;
7585 assert!(review.exists());
7586 assert!(
7587 blocked
7588 .retained
7589 .iter()
7590 .any(|reason| reason.contains("incomplete reflog checkpoint")),
7591 "{blocked:?}"
7592 );
7593 std::fs::write(&ledger, complete).unwrap();
7594 let mut report = Report::default();
7595 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7596 assert_eq!(report.evicted_copies, 1, "{report:?}");
7597 git::git(&f.repo, &["reflog", "expire", "--expire=now", "--all"])
7598 .await
7599 .unwrap();
7600 git::git(&f.repo, &["gc", "--prune=now"]).await.unwrap();
7601 assert!(
7602 !git::git_raw(&f.repo, &["cat-file", "-e", &tip])
7603 .await
7604 .unwrap()
7605 .ok()
7606 );
7607 std::fs::remove_dir_all(&f.repo).unwrap();
7608 let destination = f._temp.path().join("restore-old-only");
7609 restore_seat(&f.home, &state.id, &tip, &destination)
7610 .await
7611 .unwrap();
7612 assert_eq!(
7613 std::fs::read(destination.join("old-only")).unwrap(),
7614 b"old-only\0binary\xff"
7615 );
7616 }
7617
7618 #[tokio::test]
7619 async fn source_eviction_crashed_driver_lease_does_not_block_cleanup_or_retry() {
7620 fn interrupt(_: &Path) -> Result<()> {
7621 anyhow::bail!("fixture interruption before unlink")
7622 }
7623 fn vanish(path: &Path) -> Result<()> {
7624 std::fs::remove_dir_all(path)?;
7625 anyhow::bail!("fixture interruption before unregistration")
7626 }
7627 for stage in ["ordinary", "intent", "missing-registration", "live-driver"] {
7628 let f = Fx::new().await;
7629 let mut state = inactive_source(&f, "ed02", true).await;
7630 state.driver_exited = false;
7631 state.driver_claimed = Some(true);
7632 state.driver_pid = Some(std::process::id());
7633 state.driver_started_at = crate::proc::process_started_at(std::process::id());
7634 state.begin_pass();
7635 state.status = RunStatus::Superseded;
7636 state.save_under(&f.home).unwrap();
7637 let review = reviewer(&f, &state, &f.base).await;
7638 let identity = serde_json::json!({"pid":state.driver_pid,"started_at":state.driver_started_at,
7641 "pass":state.passes.last().unwrap().started_at,"pass_index":state.passes.last().unwrap().index});
7642 std::fs::write(
7643 f.home.join("runs").join(&state.id).join("driver.json"),
7644 serde_json::to_vec(&identity).unwrap(),
7645 )
7646 .unwrap();
7647 let live = if stage == "live-driver" {
7648 Some(crate::driver::maintenance(&f.home, &state.id).unwrap())
7649 } else {
7650 None
7651 };
7652 assert_eq!(
7653 state.liveness_under(&f.home, false),
7654 if live.is_some() {
7655 Liveness::Live
7656 } else {
7657 Liveness::Dead
7658 }
7659 );
7660 let mut report = Report::default();
7661 if stage == "intent" || stage == "missing-registration" {
7662 copies::evict_with(
7663 &f.home,
7664 &Fx::disk(),
7665 Fx::now(),
7666 &mut report,
7667 if stage == "intent" { interrupt } else { vanish },
7668 )
7669 .await;
7670 assert_eq!(report.evicted_copies, 0, "{stage}: {report:?}");
7671 let mut retry = Report::default();
7672 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut retry).await;
7673 assert!(!review.exists(), "{stage}: {retry:?}");
7674 let ledger = f
7675 .home
7676 .join("runs")
7677 .join(&state.id)
7678 .join("artifacts/eviction-review-1.json");
7679 let receipt: serde_json::Value =
7680 serde_json::from_slice(&std::fs::read(ledger).unwrap()).unwrap();
7681 assert_eq!(receipt["phase"], "evicted", "{stage}: {retry:?}");
7682 } else {
7683 copies::evict(&f.home, &Fx::disk(), Fx::now(), &mut report).await;
7684 assert_eq!(
7685 report.evicted_copies,
7686 usize::from(live.is_none()),
7687 "{stage}: {report:?}"
7688 );
7689 assert_eq!(review.exists(), live.is_some());
7690 }
7691 drop(live);
7692 assert!(
7693 !RunState::load_under(&state.id, &f.home)
7694 .unwrap()
7695 .driver_exited
7696 );
7697 }
7698 }
7699}